From 656e120470556721c9f62fb0bd7c32b54df2c7c0 Mon Sep 17 00:00:00 2001 From: qmhu Date: Sun, 10 Jul 2022 18:06:57 +0800 Subject: [PATCH 1/2] update docs for ehpa and prometheus-adapter --- ...ffective-hpa-with-prometheus-adapter.zh.md | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) rename docs/{ => tutorials}/effective-hpa-with-prometheus-adapter.zh.md (92%) diff --git a/docs/effective-hpa-with-prometheus-adapter.zh.md b/docs/tutorials/effective-hpa-with-prometheus-adapter.zh.md similarity index 92% rename from docs/effective-hpa-with-prometheus-adapter.zh.md rename to docs/tutorials/effective-hpa-with-prometheus-adapter.zh.md index 2546e4091..503dc47d5 100644 --- a/docs/effective-hpa-with-prometheus-adapter.zh.md +++ b/docs/tutorials/effective-hpa-with-prometheus-adapter.zh.md @@ -1,7 +1,7 @@ # 基于 Effective HPA 实现自定义指标的智能弹性实践 Kubernetes HPA 支持了丰富的弹性扩展能力,Kubernetes 平台开发者部署服务实现自定义 Metric 的服务,Kubernetes 用户配置多项内置的资源指标或者自定义 Metric 指标实现自定义水平弹性。 -Crane Effective HPA 兼容社区的 Kubernetes HPA 的能力,提供了更智能的弹性策略,比如基于预测的弹性和基于 Cron 周期的弹性等。 +Effective HPA 兼容社区的 Kubernetes HPA 的能力,提供了更智能的弹性策略,比如基于预测的弹性和基于 Cron 周期的弹性等。 Prometheus 是当下流行的开源监控系统,通过 Prometheus 可以获取到用户的自定义指标配置。 本文将通过一个例子介绍了如何基于 Effective HPA 实现自定义指标的智能弹性。 @@ -11,26 +11,23 @@ Prometheus 是当下流行的开源监控系统,通过 Prometheus 可以获取 - Kubernetes 1.18+ - Helm 3.1.0 - Crane v0.6.0+ +- Prometheus ## 环境搭建 -### 集群版本 +### 安装 PrometheusAdapter -版本:v1.22.9 -注:为了接入阿里云ECI以降低成本,需要指定节点进行副本扩缩,故采用该版本作为基础环境 +镜像及版本:registry.cn-hangzhou.aliyuncs.com/istios/prometheus-adapter-amd64:v0.9.0 +注:EHPA需要兼容目前已有的外部指标,该部分通过prometheus-adapter实现 -```bash -# kubectl version +通过 Helm 安装 PrometheusAdapter -Client Version: version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.9", GitCommit:"6df4433e288edc9c40c2e344eb336f63fad45cd2", GitTreeState:"clean", BuildDate:"2022-04-13T19:57:43Z", GoVersion:"go1.16.15", Compiler:"gc", Platform:"linux/amd64"} -Server Version: version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.9", GitCommit:"6df4433e288edc9c40c2e344eb336f63fad45cd2", GitTreeState:"clean", BuildDate:"2022-04-13T19:52:02Z", GoVersion:"go1.16.15", Compiler:"gc", Platform:"linux/amd64"} +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +helm install my-release prometheus-community/prometheus-adapter ``` -### Prometheus - -镜像及版本:registry.cn-hangzhou.aliyuncs.com/istios/prometheus-adapter-amd64:v0.9.0 -注:EHPA需要兼容目前已有的外部指标,该部分通过prometheus-adapter实现 - #### 指标采集 服务设置 From 82d292ae360ee5163466786a8dba9c8b024464f4 Mon Sep 17 00:00:00 2001 From: qmhu Date: Tue, 12 Jul 2022 12:12:00 +0800 Subject: [PATCH 2/2] fix bug: missing custom metric info --- docs/images/remote-adapter.png | Bin 0 -> 45180 bytes ...ffective-hpa-with-prometheus-adapter.zh.md | 535 ++++++++++-------- mkdocs.yml | 3 + pkg/metricprovider/custom_metric_provider.go | 30 +- .../external_metric_provider.go | 18 +- 5 files changed, 317 insertions(+), 269 deletions(-) create mode 100644 docs/images/remote-adapter.png diff --git a/docs/images/remote-adapter.png b/docs/images/remote-adapter.png new file mode 100644 index 0000000000000000000000000000000000000000..aa70270f939e30c9ca1eae72f2ad8309bdfe18cd GIT binary patch literal 45180 zcmeFYcT`jDw&<;*U_nrkCQYPECv+kxC`wg9M0yK^l2Ap65K!sTqzDNmC{m>NP(qR3 z0#T3}dI?GiCDPlsc=tJbzx$5!{qucy+;Ptx_YcN^WU-#L=A6$gzd4_e6LUJ)TuY8H16Mf=54;5N`ty?Fc{3-8hHBjz5A`F z_$AWQ(*@s$-g=LEB6;c^it*_ah@`bH>cJI|`q^jr^XY{O3(JKg12RiBi-!w!pzI@J zB@X%;MMjfIN!=T~HQrsfXYx;MDd3&G6$MkhAX+N?Pfq#AngjUcSdqWLj5 z58+(5>@IXD#P`Cl^<(V9ljoM*N9$XOjOtz*ll9-;UD|87Bwicp^CvU9&%9D`a~z%q zIozkaGj@k^Bv*g<+pFP~4_QUT2y0NJ>_89=93;9j9KV(#L>ewbx$nC@VWj`3P3~9< z8r+^Fu|~4`TWzo(Rc`5vS4lZ=y_)+vcWl!Z_1NU0?45%HpF)TJpaV(`tQ0ZozL;ug zQdVZ-|3&A}sinI&&AV)=jzT`FHp9W^gE`$&ETF@ySei6(9rA*2{?TC4%? zbNL3DjF^!b@(4*q9=ky;lKLL!UjHveeLE1L{Bmx1P9fXO+n{eE(@Aa;JGQZMc(V$- z@_a4Y(73{;H{d`b3rl7oqN_1A|sy`<<)|cZ*y)<$4ELtWBIfXaMzzg%y zNQ>@_kWv4gr+y=&OD?~-yo@WgVjUjJep~(49IfLcW?bx`FSnH+9=DPciD*NKkCz&Z zDl!|rl=N!V0mb_9w5R?ij`(Ss=UezTaT~lx@&pRf<<1!Jt*dh1_nTT^=EV_OwN!nqCC>dEEM2b1>wT`vzvucmb z+w^#X`c1B|@pt*q`CiVV=D~(CsdnS>q?|P2r+%961S2EyxRzMBTBxzFv0zHt$GEM} zWgOiAX}ub!7EVH_b3C``W}D>j8Yhb5ekBoOZBvJc-=QrEk}s;87wa#V={!%WuTFw? z*Yvk(^aKi3;@IqZ*Nxz##ues+iP-N$c#v_v5y)l7XSF=pKW)7}4)Az_{JGYK|MqqR z17O2!D!M}aw_8V*uR!*nKWD|?n!sw`@a+hN+!-3*d^EbDeQOCeGJIKfLWHjyxnAML zyTs;ZZA-c%O-P}AVUg8Z_A0BbYzrnsu)`>XozF(8CBK?%6Su($UM|_%PxI>*t?fwn zW93i7n)Xm4y$Bk6sV~X%8V&nL>*{@eKjuu3Cia+U#p&{RcwAqr-?bA5P?Gm=4nUQn zy#fbADU$vwJVuysgx#r|{8A!H-dh1KW_h-wJM;0dz@^H@AaL~Wg=K~He0KXkISUAo zQ?xkF&vF6u!SaLj-ObJiHgopJ43ZRMZK!ai4|X`qCo> z>i8_8uWh7tP&0srmQD{AnM9jwjLVjmUe$E`#g??N1O+EyFYDzFKUNK$`E>i0!8P&g zs&h1c=If15a-VTKVr){q26Q?VbM<%jM(`l2)#ddVfs~{z%J5s7Y3ID+=hv z4Ax)mrDWott!Oohw$S@c8|)RQqDVv(QF0mb#c0rlfb7S%JC>Q08@H+S4uGST=f^TE zWLM;t&!3nOh*Tf{|Naz6)B=oZ)4IYUQ%z6rUY%Ix)z4oiGsQ+6b93mi4vX4jX>8Rmes@d+(67TkBkC6ud}8YvypJ;8pQNz5iNo7vY$k>5AC5Fp3x`wAY+RI&b9u*3Uf` ztnFjD&HDDbFuS&h0i}Yt^+O54Xjx~BHPt!`c&O}gLq@~Elc7N{sFe+lztLo4cWt@| z4bAEapQP$*8fm#@Pr1$0<=Ljd)kxKLC24_ITJISw=kl5h26oUN)XoHaewL$} z1eHpYz4@U!zK6CgJxymjxREd5#&%e1%~v?xWr_5RfW8QU$vvqR1(si~d-5&c>Uh-C zQlzLJ!h1kS6uBp1bJ+Req*AYGe=uGI63fP-WV@th7Epo~#S3j- z>*1|{q*nD;eLtS~W-N?+#;vfvIBS@<6^$_)t1)E$|5jn88cBy?4^`ICQ2_NnWHFGZ)amC2-~XS3-c7jo|PpI_5{ zJJTWFPcJ$ZR%fwETHhZt6&F0X$bJ9!O`x_BlxMc^Eiv()4w@ysp3X{k-gsNcPdWJ0 z{}{JD;Un%`$b8)oc+{6b<)HerW2vZt>|DQVx4m3O*lUM)`0vgq#w)Kj9M>l+dgdl8 zcx3Ibw{yj90iR5y`fhZd_!W09za6Wt(hmgXef+lV?w{ZNfRd9{x(xCG9 zg7Q91=b4r;@_eGjRx~|$?e3*FWI!ak!bD3Pv)Rk1l}#6gcRb7PB$~R1Sp>(J!z%_% zDWsl5KO^ffzJpo+BT8B)#VMev=!$AQ9Z;i&=09;R>9`fnJ7OeN4G6S?p4|@9xoj=P z<-2z8m<5FV`Sc;eW_Q5Qf^R7&2S!C1D|gV}UTdb`Zzrf6*UF8ttR|Yj?yOwDQ7ap; zKNZa8yw^A3*Yqz4HNykN(Jtj zj1g>dy)Ilv+p|$hy~*=w{zoI?vq-GdP%j5!E=J>c^wZz;VOH~UePrs2rkpBeEJt5J z&i?zScx9EtKXHVg=C4Lz}d<@OATQV})jvKlOl-SV3CBOgtJI>Vl)4ch!K$Set4T zg2!7$u!8tABlAnq8@35XZHW#oMLkX{xYhdl$FdZ#7kLbldL?PW?{III)<;&5@acZg zrbzW%Y@z@0Q9VEVtg_0#c45cGwzZ5nT}_APz$64eNQ+|w<3s*kShqDQeP?$r?nt3J zX+ff!7PY|e7%Z773MALhw}1Bwta$$T60N+r4*?OdaCLi3ryZ!y%+oR+O2DkBN8Xqp zG=g$8^d4?cs*n>HE;gv1f2w#wF-ldpDh!Lvh2Yzh{{GgO)BRxxI&LLomr(Xjjjdoj z0`MU{W(3R~{REp|?a^GTq@LsjEN7xlQwf8^wLgls-wPM6MN9TVx*W>zAdpLO9VDz_pNvZZlFSP^Bcuu=f%M*LqcS^k z|6>(T8ZO`BG{!(9#Zxvt8q}b-_`f|tZjcaBF#6_w3CcP3q+`j!yW^m?*tfnAC$w#V zJH4hrMCHWm?#lIASL*N&yOM-&TOZalkI*YN3TWqm-DLQFwDR6BVAT&Lq`G2V$C|4Y zZK?n&xe*9M183H^FS03d7nO{@ivI15RTBnC&+MFSBBc7;u8LLhSey@zg^AB!!)sq{ zzN`r-w7Q%vhGmR*e)9OpOOvgLb>h!i)NQePlbfuz=03eC3P+yS!<^>HLi5Y~LhC)x zrN()K7S{bbs-UkQk9h|T+1B7(?V?8i?!#!`ER9u{?XUVDCzoI5QXYds;88YDz5eC0 zAweP_+e>4%(IWyVOk|A(E+DPr?=qSP9PnnDmV^&A)t~yEneeLcuRQ;HrTbK0{<%R_ zr6vAP|KbYZ^vjO({`SgO`m=OAJVqUXw^UGJoKe^w5)t8A_dD(co7I_#0)G$0!MIa{wl$cl_gYV1lb{0@V3Ucl2xcDv$oP2Rxsq;+OC=l7fQUu^ny=qOSM?b02(CHX}`f0>=8I;~l zmj&(x5KiFFXN9rnu2ldznfD1KjLmTWDDvU!Whp`v4@CsH+s8b}+xu|faXC+zB5#f< z-z8p0AbPHKfVF^|&)+t>`IHUF?j8aJ5m?&rzv0rEdp~99{pNL3T+@cmRn{7>xJyq* zRrc-u%m{b(;XxdEM*AgVYoFBkwuN*qVZ8da02KMqaO}{nv^Q)UnSedOJEgs9Za1BO zrQ({ev{t1xHmt?zDu>_YE=c3mK>tl(n>l4g{=M&_i4IeeLfsNIOD>~IM)h4uW>PoW zlFHk^yLBoS)-I$ubYIGZd$A_fI^(wTvPILlB3V>qj0D<-N5$(Y@XEHDWWH`yQQX08 za(o^sqd+S*Z*ROGdodG6+6}H&sS$m>nkwQnX|-~3cE6~6 zVIe(Of&jrMi`GATA9iPbX1{-L;5ix(+JDb*$;l%3fm7um^|)F0V7CCgC&8X9R<~4F zzNrx#f7?$SFgJsU0-&nvqESAQM?KQ}9Nz<8Enb+`P%<*nyg9Q#$_4IC6OBgI+Qpq? zTJy)jaZuxi(TnQ1MX(qnsqkFs>ntz#| z!Ft7i81$^=EsY>OeoF1aJploDJiDpX%6R$0iu<>9=WdQIPaiJ#*P8p>HUhg_1ztB& z04jxOT5;KHPn8|+Dr1gS)DfmEV4Gcvt)*d zwT}CSo@_1ygyV+qs^L3H^Ht)+W+#r92aMz<6AQ{bXRY%PC$5S?tX3d%^>-ZSHZo@5 zDVNU{N9R8A_$x(k&qDU_Fyr|9)5I)MK&y|AjlUA!@YD&>@3VXLN*|9+;J9WI;iaDb zm0Rk>D!=_`%|V}ob6(kCXjk*`E0dL~28*@@tChsD-6*2l>X*2oS%2l|wd(d1M4mq< z(f#PB;bFZAbQVMY?lzS1CoV%(1K7*Kxx*$JwNK*deGl9uVlq^PWC>~yEl4MHj!V2B zTgSD`tU~j&?Os|h+twq8O6{elaFGN+JZKT@p-U5OFx3S~^d*sEOXG%lL-T$!Wy+Uw zcvw(Q#dh)1l%(^VQKi7rx4`y8LlYl{23VvguDUhRRB>L$h%gny%!dShgj>F2hSfaj zOH*{)vdKD)$^sSchL|}1w(XmkO%E5v=xOzIURZ${;yw;O+FPlebSrF?>fSvb@t>}V zmY?6tm_%JD^CdbM5hc3y2GM4VCBsUXbQj1vO;UnH4f_=;bW-sEZ8*0)e<>r%Hs!D0BR7~*0C3sD$OZ&lwF@CFYKP!^(2L*D; z^RA;AzwHE@A{vWVmg4++w~wmrFhA0J64vIG*j^I3MR~+Dg04;8;dalwZGM8_=Bc>+ z*|m9do?PrH>%_WACRH30GXA zg`~Ss)_@!McC}MhqQ`WH9YjFQ6=Qw=?1Ew!Xu$^}lL}M^Q=pU7@x@7Dx^>~=xQtJ##UACNXut^iGvh0sPqKz;244h}4hN_U#eNOnMP|gL5 z$FYlQzT1HvA9KA8{s%i{A%L>L^G-nsyd`d!H+@%z;5KrOkex~^5-qn}ALp;l8aVLh zmL>y?V^%-~jlGlgeQax4P8uCIf|nc3Umyz+Bq`ebvM_;tC(=*Hsyq;ZzqC9a1H-Ox zD*HGc=5X2Vey|bXg+s!cw;ksB?Sj=PzN0PeedbPwJ=pDT!QbciJ@)lI@e_mli;e%wlLGYbMx)JWv!a#Qar=llh(LRAiC8(Z zdv_Ak1Ad&c>7aQmmb{(gyTVj(XhWTCGzlwJoY0m68KwFLD@d%{VGNfJj?s<`NF%%X zTWCB*$WVFFB|^%+CCGsQeRC_5?N2raEgRujt5YI)=d+7Fh853Jly;5QL{|v^KJV4m zF{t?a9zLGRG6x1XSAPBu zUO~x@mZ71VOif2CN`E`Ko~EhCP192GmQ>Gp5+i( zI4di36y0dmNA*#^L(;y-?q!D^294>ZZhkuCWP4G&Rh@WGz!d%G&SH6TtmhNE;-=e9 z)jE|UqG$_+=bG?M*2)aghQ*1;zCF_2#KK`o$%W}Rjc-Fz(SPWdG9ISNjaLF)4O+2s zES(cPK3>U`@_oo}Q85`h>R7B6H~D4t)7+-S0E?B~l0+Opy0+lci_Q)W#KuB02945oDjz+JKIfrZLk-186T#N z9VRa(0^r>=$F{9FrUd&I{_~uP#+f#h&~JOF0!T}OP*%s}iC>q!g-`$Z@WZ`cX)oz? zx{*QKfH^!kxsf-|xy>~d=| zO)7NuECUPb^b-qWU$xu?OI04Eb%Nsz_>vOyJ=)R(X>uS#Fqy$j|CP&YdN#6>CFB&E$U|Lc z#1#Rria;x%dYYu#QIGb5VI2Lf|_k>k5WM67^Uk)J!3ynwu5fHk>VMf8ow<{8kX;j za$Hg~LryMqE(L84m*iU>ePe)UPACT3PsgKt?Q!e?jAnd!q-Ia8h`I3~Kab0EVW4i%b>hY4VQk?`o* zqy5R_wsByR=I1qlr3H3;b7?pvksFTvRGvh`$N)QCi~?^w+6F}XG2%jI2829T?Yul( z;@qzzw`o`NI8B+H9xmqYQoFAHm&ED*k`QAWatoa-?GQ@@_^R(KM7L<=OK7TR0)Zq; zUB6FOI-#(*06N8vG+L2{gAM2JpYD-?`dXxhw&sfJb>R~lT1u*3z)(PjPbP~0^l9Oc zTO;kso$g^NcA;`M zZM1SPe?0--SE;h?hLd;mBb_w>MmJRXjC_mtdaS&gp)?<_Z@Nt$2c%MDUN<%ZO&h;S z#SI|z7cl@3kbbRvv}2QIHSV{+f>clkARBUA9Bmkb=JG%I(UYRk&jG;tqqARhl4YHz zzPUF$+C9BAvMuQQh zX_#mnpL_WFmng+90`w4|WUYWQ(tn&7uKjh(_mb@>2wp0Sl@bxwrX7B;0g(I}5CPl| zFYA%5Z~5{tnv)E`f3L?Xhvi$9SJI#U_3}o7xK9V~q-eV&1-n0i14MGZs(j@Ein|z| zhc$qe5l;~J3IJG-uyZH0|3q<`H%>@u=eIY?)LW9@q7_Z44p_~}bm{cFy;O*)!_?NeENW3_?F zsL;M(B{}7d4j%H(Y_t~}ygWv(|8Q^9Bhl;?+6Q=;lU?-m<%z}JV0ZTFiBzaMl<}&c zeKkDBl&olsPq42&0Id6!ezvZsS}?8K3bE2lxS%{%ytR5h!DLKs(wD)D8Gt^2DeI=I zl0Aub0=Yeoe1!-X#k!vCrNo^FyBmWhF4X{{{<#8dOJ(7@^)@z~q)l7ZNEviLPJ}zq z-u|Rvwn^=ks{~vzW|s^nT?TnjTX>Cok^%ik;Vw=uedh96mXvcIlRr*v)Lt!arX#<* zB<5OFw`F`H`^9r7vTxi0Itd3l(!umx*)( zUzB#H0?b0_I1XUK_$6fXWw|QAS=>5tQ0Gqibuorjo*>Hp)(UY5ng~!+CzbvtrPW#n zFUEA;T0mJRn|dQtr6*Z#TdqxkTs1pJMGu~OJO=o;D;;3w&=YUx?0Dt9t{@_46R#n2^cLBD zSscav8BW9&0!kwXGU%>RwydoI8T%cqKlCEp*{(Wa*9Fz2q@#otz}Z&fNJ&m(4M&?r zCnh)-f%iCVS;MCuBF75^hQec4wq%64?MH*LUq8OglCFP6`T~K*c4mXN(#Vb}a0N^3 zO|)4{Y_B;i6+9<_XMxg6tKU$IENQc}DG$1G%MgO$IiC*bF_G&{lc?Q=Dqxqr)Jy}K zdIRHtozzQD0llxsMq4K#;-rssnO$B2ZbS>B9Ru9Okt`Nv7Jzsx>A!w}9ZT2&cD4>@ z+#%3!*gtq>84EwfR4G*dJ1hAh#0S`3xFHMwI|#a^a(MJ*ep|Zok_n)lN7%N)m@PH` zdgW8FIE5VXSr8{&5KndwI6Eq(qtRABZ`tNvisC-;Vhsc0G%6--io;%^haE?q5X|x3 zlK3&Z&*ODVnXGneb0cN1F6I04wys;OthQ3!5LBmhoZIu~7V(NLyqb!FE1y~d^yoE} z1rNW60Zv2_f}uK})GbmL!PQvT$)CKlJi0}6S$%GS+{_>VQ)g4q%l1FggNP@6hcde! zQ^0Y?;?H^E9Y@P^xN1uqso8i5;Z}DCZiS1?4E5YhJje9nrqJd$@P=DUA``Qz>pd?Z zEbSjpde2^V>64Tm$9Z|3fK@_x*4Lg1> z_&&k^Xg1B``#qZ7O}8%>vEew}Q?bU=ZG%2jZ)y06jLn@it7$}`vb5ne@WM$z2MoW= zANeywgTci+@}xfoU8z1-wqimD1CoRQi(aVpegcv@pZ-Ui({jV{k%KMU@xg-1_(_lK zq^qQoD6v}LvCw-ocfKu(V`)-gvj$+i*yH1Pecha~uTlXwY#l%57-hXeBUME~GCiu& z&sj`6ZmLnUT{8(4pjCxPr0Zs}Q5V`v?o@sM89$f%SYCY_Fvfd!QC4)DPz-|KS@-SS zO=5O3b0=6}t+i1^O6`qK3*jK$&b5Hvt06raZR_S|ovlKrp~B87g$T>;1kv$MfwX0=c3U=fc8QWYpY1KQ!%Zz>iOoZoCyjVee;B%? z_E&niFB|oV8B*XVAzJ=J&|3sa0k?upo`gb>``fFl#3s_TS-9G4s%d4fX)obvTf@ z!TSbL(~$5Hi5;-K)!cqhrx}vHDo{{8l+iTE`UcqgT>s3~Sv3FS0Lx4CZHSUqCgG|Q zpy_IISW@+3Dp#<*5NPTMnm==0I~~IA{%9=ajMv+Fly+?4*tCb#s_7t!HLg?48!!?5 z(N0VfLROzG;od2`@cgVE@{>5kNn%5ebe3m3OIioI&0F`Bm?_$%#F{%@viRi?PoEM5 z2Iyk@A9M*9@gm=_J^QX(EJe{y82Kdf4htk$q7Gkp-(i9LQ_10yB!wN*Y7V4@^(#QO zsyWKC)Bg5?Hy0Wk4U=KD7cT37U?lgsKO@@MZT<-8B*piIfb7qJpx@6vR)%&fR;8sVb{W8fUMGBn=2054&Z5C+(JY zi^U-dtpRc5{Q!p_hY)_dsMi*!e2Xu=G-Le2nHfexFS8>%ls&d=n_BvGNeqrsgrCcL z?>o}bA~bIxya!Z40H=w#p3l5CNWgobf?99(2aia74ai&pGDGcA zRXjKZttEvn4=VEC`<=ai!>($r>C&P5mRT0kDWFdZAOM9yfh1Nc!ZB(dX8E1j*q(bb zTKgdMDv$5wmp1PdU+TuA-Ucm8(fBVaRnMkfx#cBa!q0haw%SwPE@`KGYNU-xjzl9x zS!NSk^%#hjlwC;ih5vvT0M z$?k?mVf<((?R==|XFf8k?<*jwu2Q+$^wqiMcb0Osl>PUca$aQ9NgDcZ?t|@E4$q%Y zH5_;(wt{8b$i;_1GqT8EB|pGhQ!jME?c?@hUXd<_#fYQg_>5T>NFPGJG z5nV`VEu&yGG=UB1Ef+frGgI*Uv!zdJEn+?jiy z)(UoGs{MJ>r@52MbArXrQ9fH6ov!7qI(Jo@DbBzbhAOPa8YevG`%&RE{m3@!!4o;P8APY2SVSzn@aqV+f36s zy4`2F>#6A|b@E4#(mC?}KrpxI^NS+_Cq1$|l z>)2@|*OpbTX)gyD7@7?vTVj(}E^Si@F!B-v-_FUm`|(+|ES*kcOrsG9c&^)}aeB3V zLO?(X1nvDpgbFZtGfHiXK+` zG;w$;q7z^Ck;yfSv57gf(Jx_>&s9S0p6rhM#=M=~g=WV>*1PTB7|jKwv+aU=z!Y0( zR@9Z=yFC%H=qwex>zpSe8}hy1N;$QKRf2=130UO1Z4J-{-#xUg7(aWy7ZM2=-D=pU z#(mn)FM~lZdj}szv7_DjBV$MkR=b))-?VR^Rigv@CJq}yY_VD9CVynWlG9puH(8t)z`pcV=Q1p+%w zGd!#Zw4bo9&7C5>HZG@f)!gkScdLfoi+ZEnwTYt>`Cb+)TjQC2*hbGrg?BCyXMt+@ z?kZ@J@|kyHY)+|BJ>(Lzfb5?hwmh#*r=sY0_rP0Tjidm~z`Q*^?JpTxC3umR6_ITKnC9}(eg5x)1t!R`u_3N+I3)II=?*l<+ zcT824g7=v^{>~oan!n(9#*m{UHqIBOZR6;@>JOR;ny21;bWmJy^pOky<}{UF{auu! z_H7c=?862)c7%UET+8V63ThlinU0-qtJ+Xo(27DQDR zl)!L*CtR}4o65p;G1pHI`tEu3NR+HK1o+q zLZUwM03r8zfR@8_{S4BCDOCGx|Bz(a#l)ra<;%y`LEk!U)$L5UCEVP<`G_Ts+zS*m z2(3s-vC(6(xOZS<{HRdg*yyg1S~?GBz!{V^h#;%uD35K0fK=wKF$#WeV-Z7dA4LSN zCFRpoF<(oDmS*vm%xrQezaESdpFV{I5%7E0o7`wlw<{MuAt}!E_FX1d>LSc?xepscdwElV`6ZNBASkyJT-%{rNN4 z;t}q83v(!!Ijxa4ZKkcC&5nBvM}>QU_J`jY_Gx+-^L}5p;;0QCX9;SMjZD5L%8Zm^ zHS)`cshNHv*z#Z)EEGR42OfQ%UeMAt@pho?*DwKB^-a~atft;oGdZuaRa{7%sw+lL z+tz(>owUxMyja_V4(w?`_$Hjg)4WH07FQ$9DsmT(rYzcz>`hj55WSncd01pa~r0 zvYL_%|J)e?qz2zwh89s46d1@PhLJA1?q>&cA2^OS&pB0{p8_*H6Y(AFaW45dXPdq; z&hl&XtoX#cSz+5iFpZkU;+pd9RM7E9Nxb@9*|^<=%Xl`Z_Sk*3(7iy=E9#FQIzO`q zM9|)oBX%ZTNiT1NxyI2JcDS`h=Ea&0fGK#;;Wx708$_`}loSg=q8Um;H@-9-zkK=T z1y)jD*0ZaR*49!MsbA}tz7?WpP%;&=ZLmVUy?6U562Iqq^Vk~$>Iq2vd%bx?4Lv?h89Wut*kJiX^Eic zm=0;%Z~M2DdQcp(1`tSfO?Xb}zKE;LRsAqTx2yq&p??mu-KdVXuo1-W^CwIaD(va3 zyzY1bE3CW87#n|Y8{8%sD?0CZn_!ikqt&jhU#TsW#<~l1_DfTDU{=^guwiz%<&zvO z^ZYrMXS6!@b2n~icCxf+7YHZ*maJZ+BaqFmuAmlL}6Hs{;dXL`7BGoGTv$Fvm zYoL<5^2ag~vn&B@Ms1y!G8whyuf#fFptN_&oLrklb){i@N>gQ1N_SOb8Te@tzC1O0 z4^9r&1(sgw5*KWqa=&HE8V|kh#19Okr&vNzS+r$pc8Ma~==*pxPaF8f3nM0Yhx1fF zs^2Gz28i4j0#M!k)7+PuzbQMCJFZJkMZfi@Jtfk5`W|o=BcSwbX=DqlRB=4$2?IV1 zfsBfq%4L22?%Jc$uMlf=QZy^y%;`ndcqxFL&9P=hd=ZQ4cBhrM)l?{zoE}R7@RnfJ zvM%k+HKqPosp9aOXO7RBwa7tl8ehjNSqh=HBJ-T$)h_-(0Yg8x=q2hnXj9M>nyl<>;d@yApw~Fin-v-0}Rp{;!gS)KtE$o zX~B(+3{)Hhi$)&(9!78$zQ1g)2l=P%q_5khgUEZHzxRgBNFJm?3Kzpehm`z1cHFc^8-tK z`6FObEIIUT>!WT`Pe0b+WKis9?hK@`@L2(H!pX|a(o^Nmb_y_zW^|jtE#o36s}(fp z51>CJWiHNPCH5I@*F#vs$#~lBHyiAFlwfOMG;KcLx5e&c>MY}o;D7v6?6cng2^Z#k z-P|3kCXW!EJqQNBy4yLmOvE%=bHO1$Yd;T*%&T$T`SZc&pp9)juRnW#L}q3__F4Wv zmL-|g$xV*=_l`YStU!bM6Bcr3gK5{7Mu1t`jFs_yvWL>T@FI6q}uHg53$Bp>sXAgA~dbs^SY)Kq3_`K$?8gftzKN?9gx}frG$oSS;a<*?!XLdkXLI~^W}bkXY3C@ z3dSdPt1~+_;jDuga~mNp!&(oc<~Z>L<~+7IP++17oA~UrU9L>Y(`t{}Tv{HkuiN)> zWrz>h8UrDEY9KXyIm+?T!>02OfCEm0=g$3V6E3Y6u5emVnZ2YQ>E6R+_Vd_(b_AOJ zhiCR)0~NsZp&T9&P6nRcuSz=}Pr7UMt^VPO{&RT#8`zIDXu=TZyN;3!6Ixg`00@_z z86tCkfd`mDYVZJdL5>>DWEy`|8CF2h7 zIepI3L8cWh<u`zshm@4?~wTkAY3SziIeetSHpXcMwd{v>JB7 z1AuZ^&6KldpX~f2h@)MBhudD0N3&d=?ovbf{xd?^Rkp_Eu9w8iKP=$}p4Y>d3eEJf zh`yZcy;r()-r(99??#&_uEB*Bqp|~^Vw-D4$eFcia>%#uD$)*m?zSE=@3+UZXI@-M z;3!4rGe0G8I0yWc|4U0x|5HnO=tMJ;ibW|c?F<`G%^$7HCc|R;R%<}LYb#n}^;Puv zaoOQ-(60_(_z8}Qv!$2uyL(QB4i>t{2Yy7@teE2TS(Qh8zL7p+4Pb>AFvF|QhRl$j z$L#p>an4t19A1Cm@o&B}Tm2czuNm|HvEDAQ#8cJZwzMvjLl4V(bNFV6JM_W zo#m&%Uvuc*%}0Ez!5`lHmK%>E`jk_}{j6$%nnV#wWL zZtuKne3NC5*1y#wXooYCc_2I9Lp+#g@4I@!P{h0jW8+8cFqlV*mFSE$+VEw# zH}Fni6n40Y#GQL6MJtO6dZU|+hSFWj-|m0NwK)>x5q)Q{lHu;}_IdU0SnY*#r$SpD$! z?&5m;_)JeyW{aLRk6r5h_9X+fIfxKp(^ziZ>mX-?XH@27ut9LPC2j^H@n8?e_;ypB zax_|Sb5>?++(F=&h2e49XWU9cEf?F4^J>iDBUAP1f%`e@p#)svGvq38I(_imb7q&~ zXM49?N6<=%=PtZsu@kmpQLul~2b?1|dgICwLawWHnM5oV9L{Q!MgmJR#@;O#n7vms zm724(Y{U#zyn)L2n-1D=05eKCJf_1>Yzs}&nK%8JDDwvi-R@=?(tJisnNXx)-y z3$+pBI*uJT&EY-m)>LeDahFC+)h=ui_m4ZT5jTdx`WEdrQhe(*+CLY6 zgw%ZVPM$$>%b2fo2>!>rGm{82R{ZdWRwfM#5Fy?)aH)8>^^GfMJAj0H8F30jW{Df% zE@7B2jo|^l!$U?Y>h1adE{Zh>o(>Cq(J2*sL%1@J-G*V%sGWq)(Lwln%YgibZ)!&A zuggB23)99odiy~H`RPwoVlE!X(20m6-|=gxguy}M>T~9$K{ah1h90aH0AW8 zw#>)!vIN;Un&6#o1Vzt$Qi_Mg>P-H?NNZSlEch4d0~jKS0@T;%ertcyvpV zxQzKck8D{~1M5E}U}7izLtj5TOB8yFx*Hb%@&yHdQ>l~zkFjFC=;-$wu$e{T^4+`R z5=Qp#IMAm}xInP)LsIQOMHx_N(SWDK@^RCT)v_R;x%&MqIH;6W{t8cBvuz2Y?Uacq zm*>lfF}o-yPW|Ui8#xQ93tTpBtMv!&lj?&Z*t)#MmLF3`hlj7T&$U@Ng8#J%n97XT zs@?UtdhOn_bUcKEx~rSi0he_D;Fc@{7ead(xq7sZY#0W&OQu$78&OkTcd!|xvs1hX zYTpO?aN;qz`2$KFx($*XCQE^pC|)n~D^~WY>wjk?^8|*;=_g5Quvi zZM6I)Ik-gD#i0tQ%{Pex_#RL*EHcVCu-|B!&7biRoj~?_IB!`tqQbR`TPO z?h7A&-ZuGUD|@k&+rUCrT$6VhQG!s1DC6(8Uj%iWb?oL3ytQwMo=(D-U6#IQ*QEn| zYno8baEjq{azN;2$q?;>Ix~6sCuMMH=k*aL9++Wmfu2J*^_p$oHV3wy(@((-Rb@{? zL>fHTXe&mAhHTXWUOTy8fk{2_^y|TB$fXN`RcWl=&w%64&)e}OH|xt_FwE8V;GIpD zj1IG_M!HO6QX`)Qo6DBV?1=UUGi3$B29q(s(52_wfmSw=kJzem^(^i}FOcMM$>k=_qwZDGyM0I$3Tz|*e_L1sr(^xQ1 zcjw*WqxRf0_99{mr76|wvlos!b&K)p5y|Is;}eU`&aMgs2z}JXOEif({Z=dRmiNEi z2V;vg`Oi!@YzOK#WLFZ}T1y!W5h+GPuie7u5mHY(_;_#3#V=gl_NH@_fVD?hzYBxM zXImA5Y#lV5uQVe#9Ajpy$&x+!X)EDr;Z+wqH?+Mp=DZtAcTj7ks*UgPzo1#dL3%&UK-xOPI)gmozm$K`({^%wck z(43Ty3m=l?t|#;T^?UTN2t|ZZJKEf*py6fPoA|%~bJ!@vx~$FQ(frr{3FS!!drK$nR6_)ZvpA>trPf25=SEgR@hqth6NjU}3Vc>|2HWey5{Gzk z)l31TSzDP-@}G`7XPUOeQWbZ%x{r>w%NKq=AA`Pqar{RV81?Ob&|;7495Unc4jJ=p zA?2<<{HK4{3(AqUkyTOouJ@fmZhX#Iq~cj5|G+bC5;{3or0Uz5dM+=P<*|}4#|=8& z3FL0Yl;xiT;MLk5*(0Xuik?zerf}Ze};CfHiW26k#`;-G;I( zq;E>$OMJ;etbA-T_}$T-67hpq-DA7a+lZer4uSD%zAPm9dS&2ww^l@99)fJh_$3Uw zUb^_QGfaSAvgQM6#5ue7vh9qmOxU(`vCAanxC6Xk13wL4u2sRMu z9i%FXjiMsG_m)tlS3y7pL`9?r0RibH5K1UQ=q(7+5{eWFEfnd5z?tz|?^@s9`&{Sz z*=Jwt{M73q&ok#7bIdX3xbJ&N1DJ2g+K%SV-<6+pJuH{Eu?)60A z?4ITBvE3Y%OOEzKur+{$_qaUnhpyq`eo5Jl(b4{U!}yE61X9F50@D=LZ*`#cVIzRD z@^C|=fN_Rhw8-)(_!Rr-A3gxZcmj*W3T&mtRZ)?A@Z)>9O94g3xF*5b_ z&A;C!V5+vNgAg(selAg!o_tn@xKPS+ zcfO_v5#&}^f-1hUNsqwCu0(&WSK02A9=+Bcje|~BIeQ4|_dZ#9%iB_sOCz#qK6faI z-mYF-I1!e`bW*FpqAt(teS--0w0!SyhSW{@llXtZrh@uHxX%TMw4hodi}5}bF>0kC z7P}cB-L`PBRqIJqUZ|p8EemI8@m_+KDhCDfXed*gV``Lyp(aXWP$d4NMXO9jvCZAt zQ7Lz%ff;o~b;xgTI7ps&544JEmkZWXhP1W)M5?Lm*>jX_aE2X`%(ta!KUSxOG`dcM zn4c&!l!zN}C1n;T_L68)gr~xKj8meU4DvBIvLSYUf`|0-dP+1GyFd??1xwQcwV;WH zeB(ZWC;KrhW)7J)Dw}2~>@j9#t!b;jWO(o~;lviU#eijYbrz5Ab=XJ#3Mn~P9Jk@LXlvph8ZV? z{R%?xO8TuA8%<|1Z|;54PR``T${l{E&_QG*@wEFlVnbBs-`n_!EF64M+S9O-;uC!L zqn>)Cd0j%rv{hTuOffFV`nqmXrkuKG*6r2#kas~`iN2Sxn(2a2AsWq#9elH~#9hNn zW^Mk+I@vd)9&5*)MZe&b6|6jJZV$PS9+!WJ`h%!34Ep`*f}Nltx> z`?rE!&-j9vI<|4H=DAMH0uoh0ycx`}+wj4!zXNm1z_h+O=Zg-#%7m+eic8~zz#qQ7 ztZgFY#=LwRV_T8>l(K|pn{4r&*1^1_xKhDw)HGi7H#e*4^J2e+`}aA`&4R@~1;SOYic_9pmPa(+9-_HCJ22i~Ah{p+)_#a3gOn~$U!zwS!GO>Z(IU|qFHloZn zpm^IDyEbjyIZ-?%recQKe>%>Pg17i`S7xjC&FcmpGy8lXHP#&C)o#m)p73}6!(nqB zMZx{@jWQ+grS+QMpQmD6>vu87;_9--EZ3yaT#&8{8%ajSG@rR{j1-8KamkJGCpjOw zZB*>M-~7_2MJZIX(Fiw{>3xoP`LMDl4&vkM9Wv$KaUGqr9VR1jn)bcr^yWV77qUie zZ>^E;YOnR$w(rtdd1G(5YF7n-rl7-vz6Z2A3`CM=GBi&=lGso4E|udQKZ^k&cu)pp zVVqv>Q=?nUeO271db$EX_WR!P6kw8{h++#8%tf}+LP$whw65z!kk19we!4N&mG*{P z){*O|H;5pd*!9n`*XS+7Dw6F>c0WLK($`P4l>GOndEGHF!nCZ&Nh>p`v2db7(at7O}3f5gL+~51B)~ z4YlLL_OqN7NixUuLX!N{Ki{ia+@Fg_bO0WIwW*lXrx24#;R@lJ_g>|ycFpgEqK8K? zaUXdYh4Bg{uCb!}xth0WBBW7#D^87J3L4W^TnO#F{hl{&^R+pWZ%ad_bChoQStn>)MsS> zOI->R`OB4vaBA8OsEz4daG>l#qZNIxYLd|W~F4-(}!xx>aEAA}ixIGeqpL z=r5-*JldD@13;vCW)1w*l_lH*rCz@FUU>4ki{>VWW50txbtxLRr=;X>NS|$~ym(FQ znH`LM(K1LZt@4BLJFHwZ?I>$^ucV5HGd$+z%PTH$xNV^@JVvyl_erVCWMD>0uP0=~ zY73J1tAMl9E}8N!BZ|z!qQc_yLl&DmZOv~zC^6%Fr-T>Mgif_@zkG8sh==CI>(H)o z1?nNW?@@32JlVdPrz$;l(E=WV_O^1);bmRRH6<6+V zREJ-nme(?ZgpA?({2j70@atoWsWo@|56A6UeC1!aVqe?9XsAgCUwOC0r=wiFg1!4| z+M*p4WyH>(PgxNw05AK-xbnl(DzDUs#+Ku@K|GRZw$~dK0S+)!&j~`S*Lp0m(97gy z>xR+pV2mxaSF(fA8D7(;+gB4ieKW3a(Wsr(FfxT!JO~6!<;?(Yr=RL;GN~$~tYXX~ zE}X~)U65ybm|qr`{>w8E98JmQhZaWo1D0(0%il!Q!IenG#Q+U1cyOyj&uT?O<%b5G zCqfw^x^(|f=w6?-f_i%apms6BWg7QB|L!>B2|*BnXRzr?0mBqhhoe^uXxkO1)UR!= z?m7MwUyFE;C$n^WtG@-vK5Fk%Y=G`{UIij~JyQ;4?Z>wj^Kf1~S3FQt#n(!JtvI_jqP`G;C~F@-O!jO4wUq>k8;(luyfcu;x5b4%&z@>O#S@4B zsKfW+?5suCn^(o7W|IBaQUMfY>x?R;9+D}h_fvdMnM~?3J~WtkWKAnb=f?z}wpcx-f6d`NtJYc}N(^2&GR3XeDlJmIGeY>wiK?T)@*fC{T ze3zom;cZU#8x_`e&nj%C|7|gAKV`N}R}XyQl}zgF%@zRmKr3b^XjW~z1cvR-0$9}U zXGUBoJo^-NQlYlzTd$V%jtAVDTg(p1z$D=|e0r|?VeY6ox?`uBi0>rY9AcD7D`?|P zLNUQjgW!c#$*r5hAQq5@CyxQzX)gbpAGgDrHRSaPMfle*DS}eDqsN1-q^_TvafWN1 zUT!Omw%XF5t5HJG)_3iSw@<=YVTzZLS5)t`cPmx&={<}0%Ux4~R2M1#X0myApkf?@ zu+in)QvBMDDpP(Zz>+>h=vkUl|Eds$4pBQ+F-3&wUfa7k6+fO63_$Tj3=PJ_#a!HcDq} z*nN7APnCv@^x<9*7!-D2Bnq(df|4}Fwzyn$9AI-5Z-F_9(=tT^LGne0#a_Tn6ZIw3z;U`xhsv%H+y!I~MF6|)L4#{K`@3-)=rYeFT!e!I> zX;X#z)C!83L$Z&rFr-p~pO!MN>z$`tW-Pj^Iv`H3hRv5estXY zp|#to7|HKy2UDP27?=2Hx$xQGLW(@tyX_l0cec3b{lQKgzjw)V5E=w=<0%j80J?LQ zVW?hDhYH}DnpWH0 zxUdd>$K%2+eCMO$+186o#_oA7Vli49BiN=+@nSQV~Rs|RkIKVildrd=fq55RQv5qo{S70 zGe}!_@fW!jmIA)5-j{FJIj8ATiD7$cu{;D0otEaXb-`(Kr^+SNn@`0(hzNFvGyPl- zpp!-6_-cx#Zkc7CzlS#SLNxyOIa+?}Br?hK-nz8e)=iIRBQy{036=j*h)!%hKYL3* zq9`{0R-?U1O7(3=y{ix)qBwa&e+#nkuIN(J<+WS(;uI7QGeC*pG_O$%%zUf&tYTc( zEu$EO1Wok)T>=Qd_<(Mxd)QTQrz{ixGP;#&H>XY2iiLoAr2D|mAVy}(mSP~E4ySa{ z*pCa1c)3)~2E861NbL&|PqB$@<7re#vl%~LNMh=6P`LEIcA-jm<~-mm3wdA2mS2ER z+C@H$`4MG(!Is{AoPRn|Y~>Qnp%u7v>sk4c&B?MteiMNi!o82-qx|Zg%pb_%Hb+a= z4;5^%EnhsWq&gIhWvA^g-16423QEq6Pw@Nfr}bm2dK%{KM?D@x(Rr%wjcD5Nhh#rK z3xlrPH^S8>DN2t^=u*G-%5hqzOff4K#WT-8 z)7?oL15wOrh|wcqRg+sDQj?P`SO(v=e0NGRjd9tv|d9RpU|JC zOsrwxSC+A8eWV1jumurWDB@!E6G}h&OuNaNF3AL~AW}oQ!r+Xc% z^=F^BkyIFDM=x|~D^-jls>r6LB02N*Sv{Pw{uURH;Aq+M5p%EPO3HYIHf?7WT=@l)bAuB6b_x0aRBIC7?%#eUrJa9TW!{ZL>>~RUjq`Tz zMOo}ujvk;y#hc&2RbuE?W(S`ImIm}QiN84VMTuUv4=#%H{@lTLilW_Pxu__X$8Exn zD0jwyGf~bFZm4o*K#`uhQEUu^T)~zPXNEkOl1EkDnAIAhVZ^4^^u{rN1{459gbJK=52H)vvzRz0iJ zL1)>YW3~ZybC)^E`Cs2(+)jG79FJ1A2{gz{5FfMWTo>HEo(rSknHFB2gUE;Yxy}Tp zXUaXa3>tSz*G!t&_q=EG)B*SERnWaj-sCSDw_))L5>jjoQ`N}<7pXOZS${sjat+vo zicQ&MeY=d?nvV;|<>yg$ZOJJ_PrujR8GX{@igr;q;j>09rg ztt^fg5{g5O-=1Yl7r11c+ULiO7>7JQKWPL`-trvbz1PTf zb-1#X6#A?=_EYPkP*p!)^p6?bTOswW^gIUf6uDFZb)7pG+8reK_+}M1)7Nvjb>!sF zfUJ`@kDpV}zgOQtlZoodzd#m1@jdPDRWXotdQSEqH{n1I%JBc#gC=5I69vnB z)Fj=g8&($LSz|&EpUDdX_=uA!NiOt;F3^*6mJL4gC6Q}K;r80Ho(md#fX3D~33%L0 z?*Z3)l#ct>PtMPPn_DZ|bR@BM%aAmAKJ5|gwj4%_$GwVVk zMRx*%)xz$$JFG!uF?d7t&I_}O2aB)=@Q}uo> zfJ5pd2x|UFprtkmXxJP+=d*qQHLh-!FhYX9+0reawN*g>RFd`kW3vae*Zz*pX67aJ zB{ySeRf1?ViQ_Dqa51YK@NVx?F(~hUS;1D9{*^pb@Gb(>4fL!C7*s)$a!?)CcP@4k z<6MddycNE^4xW%!(^n3a091E=qhDeDFR_s%P6k+n14`g8NuPP+u~;YVcQZhpo&b7| zi)(c&B7IiZx=BoJi48#Vt_6(y>Eil7-6h7=E^C0K=W_yBPo#PLv6#seUV#9l@JX-r zsmVz|Aq79!eGdF(!Ap&vN>BpV^R>#H)&l>YS_9t}i#&5V4Upis`a(_!=03!oQ6!1z zFFk59WIlBSIHYr6C{KR|0*oDr0=y^totMeeQ<(MxsgVd*s(snbjF%{!DjSn)RQ zod^xTj-Ngxfaj%2hZQ<%JkEVV3gm)WrWm`1YJowtHEx6 zfOf(Xz#f*?q%vNKBdzS+K(ISW8UV-i=0SvAFJSc+6X3HZ)|*b&ay61?M+~g#st*=Y zk$~{h1}2mW$nU+rG7q?Pp7H$J-Ydn`Fp>@;637|+Ytvo4^NGDRvPT5jo9Mr@JlxZl zEXfZh;D#7Lk21-_?a|(V^}Xc@e+OWq)65!Yo|=&qZ-Y%*fp#ozt{G5kxq%n>c@aGR z>C6Q4>LNhv_)u|*S%d~G=jVYP_*%Uk=&kbjU>x%fo{0qSmDc}x1U~sGnlv6Ow{VID z2p%{hl0zanrqU3mJ5rZOiWRbUy=>LJ58BB*a5q;zrm7%JY=B$+hpD zXTJS}QH-%Ql$?_C)0bYQrfAOSV0ll>r&|C

jKrBZ@x`x5^gY3?Ka9f9@O7m$UyJ5AWXB8htP0MC66><_3a*eK6$X(VHZm`zYb&!r)_3X`l^}*>pCPjaQN4@(yH>_rPrFc{+NjdO zQZyz-(*7NBOPWzPCBkm3%yKu(h`93`oJo7YRIa|V|E2G{R*UP+x);4lbtfgq3UIQT zZ)xg!4eRQUtE^0c-HRCQJ^;J57f_l9U6PAolVD9IT@(O9nn2Ys|M77zz%+9~vMrfX z*wuXKxRe8H4!~l??>;Fr^PQm@0y@Iji+Q7efal&%B{O_{+0fE3({Ls^pNss=4EsJa zpJs}UeeQs-MW2@Ylpk(JsQ`^V z9{(cllG3USkc*H7QB1XIDWhqn!&3#wof*=BE@RvCS(@ZPx`WT9+<~{i?^L!_-jRM^ zEXAdUlGr|hM><|RnWZ5k4gZPni2i-q+io&06AoB#Y_L}5N7b4&`0pad{yd&BJhJ2@ zvC!$3)S~D$gI~*ojjstLOm#5yEOgTYV)Vp{FP7XKQ_axVAB11bWE?;HN<;2-`-=!I zB^eKCcucTpX2H)F7)yZ?jwjGm;D;uBfev#P`^7U{Q&a(p%HD=21=QGtmI4e=S?QTK z<$h83<%!aK-S;P`r!Sf>GpeUpY3cMWoif!_>R>yj^rCb~Vher#_se-lGQ8vSaRor^ zS^!${V(A$nVkoeS`T|TEaSKP2>g-tbg3ZxMfzJGCX>D%(oJ~yGK-Iar{5LKxS*dA@ zcKi_bVH8Yd3$opo+017sj)$8298#StD(0I zVow@@LdB{-u|fgJe_(UH4)6Jb7*v2jHXFJ@dvvZg0!LB;&jmH--BY(C?o!!m%H|nL z$hb~Hst6RECDz%h=%^G8HSuk2^BAO z_*K(uCWVjDtBp*BwrJ9@t6J&QRj27VmYKGNWPKScrQZ4Nl3pWO=-#@ zMT1W1hc7h3f@%TCZj2|Zl{6ju2MD>7`-4NO_L%u`$G2Kf++Qc%Y+koLnneoiOY~ot zJjo1GGIM;ZL3dq8wFxMOKij3a%T2P%7thwfIXVr=LkzxAw-yl8FM;$cZfR^CZK3lZ z*&BuNwPD%q4+IUI{fp)eZ!z^V7l_vUI)yBPNu>9_s1VZ0jiIB&@L{7=zENxF94@h* zCsr^bI8nTW#Vb=0G^{B=e?7$;k4Xla?)#}WN)h(ybv}3ubLEl}tH86?-AnwWoI-wg z%o>kwVuH1u^dusNgT3(T6`OM%*cEoR!!SAE+RxC%67)(x9~Vv4xLd<5nonWl8RirG zHKjj2+$!){{@F#+ggWN?khxLRbocs@H0xmwChzj4$)oG}DI+Qhn^&X3tMFQmdax z+w4+Pco7?c*=2!ES|6D|VrsAAiG=cMok5dTgslyJOICj&{z;@V5QHIpO(#Ux5Xsh7 zo|FCioJ?MQp(UaNKi|2>8~X+WPrmxbRH^S>aPz<_I}q)t?n(K5AFBvxJIyk6`*QM% zE4h!2yjj_giR&TN8D2mYc z&ZG1#aFLpyt18PsnszbH(A2OOo3m~Bi2lPiAm^l2KKC_VO^%To%$nh2IIRgdZ@dqM zh^dMzWLdv~0=ooBg^Mm4h$qPak&5D`hpP^!dHNdTIZY+qfKXHNF_Dl;67t&R;vZsj zpL}aFmmocywqP_WmE7ftZUT|*ZkDQSP_DzRhgD1NvXd`##A3M_`AM(SRzfdF=xA5^ ze;t;@#~s}D_6M?Ks|G%?bHkM}9H}&9)E9w)*f`Kuw%>8qsBBkvj;U;abd>;#^UmIg z1Q6H%oMP%+m!}F1kJzB){U{|EElz`9HyS+_*d zkV2i)gj8N0$F*M|G3@wR zsE@9MVn-lT_2bi&WOw4g$abYVitSzu+;iejfAJA?7r0+D>83n5iIduT;{rteBa=08)4@HCHrvK_jIV@K$}ZXqGSH(9?xz&b1NHner9Rmy-|Pbr_z^Y9cs;xE)$M4WS@>X^*%@V3QSjV>trg!rM)}?k# z5#u%P*^Z!ngX6%Lm!#9@V6W)}M9gI6YT)&>gJ~Ug<8NK+vQkSI`Go4&9~*#;ePr#~)5(8!T@qN9!f$MIRwf%zS(n&Z8Ld{DBT1ft<>NgIKAirq zhF%P}!id(vNUf(m+;NC6BtGTjGgBGO$2F`AA(Q;nro+oSop}!PR?pYz7Q)%un16uGAW3V0#w+~Cxn!Y4RWPfJLt@b&@EE5#B=zaZd*`_&oG4;?Ym}@Q zAz|A+MsZ{r#ce102%Jr##6B`IY7j(_8LDoHf?Q#4Ax5&l?BzbVu&ScoILCbRE2RU` z1OYMhajTZ}zY$15_JAmdzwLfE`;+z+!wmcD%Po@kZ11}}ooJzqiG^OvFQ=GSJ_RnlHEo>ZZ=Jt9go(r~h)rGqdTw=j*`!sIChoA0#el_vF2Z%d=` z6CV-%r6sH|`wTTKVTHbO`>^nNq1B#ooyL;l$=-k{)0&cNpkEN?(A*e#T!x};a4hq) znWJ!Asc0arO1!7y;|={&ula%|iSaMkoQB&8n!u!1R|=n1RVVV+iWLS@kPyA@ zl?*q+SS8e>P$LSL%`s9rax`bKVi8IqM#`$k~(xBIx zCR3Y&q@kuBOIv+C_WA>)-H?$sXBgIWK~-vLWi8_?gLiR|oi1b9h!WvJ_pR7uZY9UB zh6iIow;7eI3G&|9!qevGw>!<;@i{U!JB?X+0%rBTWh%tLzW0LRM9_(#7qM4&WHprS zBdhb(;XJZvYOSr1# zrbsvz6B3Wwx+q$txn!%a+TgWt7bzkyhg_b57n--Hg;TOF6P0B$iz7W z-+^23_^WWx#~(#^Pc0u*XPn(_+BmMweUf=+7dH|4D5mDXU#H;tL!=o>VFFj|F>{bC zE0ba;oRV9g#}TC~+oLCXZdZrUX@oG%2qze-`PS{-nyx2A$>L?YP!`XNwOkyr%u`kA z1lPh7`bm*BD^xu;B;qae>Mg?fZ%Cf_?lD0=DQA?D*TmzU?AhZ-WIw)^B11kZk4Yc& zpWXFJxM|uQX&cgDE}|$nAfQXkZSjWPziw;Sr>1auHFrm*Tj1YQaHqo7v7B6aSat5akgryb8q3tTzM z=suUrUbeZ|(MH-4S8S?`1nUmoGQ@D;4^>cIUIRmuRIUN`61M)0sq#!Ki?<=Uv>y_= z)^OD~-9Ftjl1P(H@C8uaq*kqZBwol+iH$Ao{jX-UBIbc=QxI#0QBO=r*F zUjzrI?t3gr3nZ;WYXLmRx3tmmMU9fr6$x$b@3^iccCJRKDU~( zO}#Trlm71}#d!jnjbC{dSJ)&{?J-l^LxsyN*6|%YW_nAna5TVMBQWOFK!X-ZDqwmr z>3y!BqcJmMSTG;sk*^ zDo;STmS1{9Ijr%QG87jKB|p$|O--i_?q1`BOv;EbAvY8OQS+BLR>e<#%!TOb3`ymI z?O-a4nuM;MRC`)i?Ie;xaV=ZiTe-P`Kpmk$ndEC}AN{AF-)=6M4@E3h@#~D_=T> zbynS?ie34_vVjevw@d6qYln`L&6~C6-VqoYy1Ab2ok12)Zv}QQ@l(eQQ8}+2FvFxx z?x;8?h2HsUzAV2p9ez_@v5^1h{oAOqspqV-JVA%el^aEtyhbA%j3<82m){U`Hi|9_ zc9o(S(CNtQ%A&C|yXNr1h6sD&g^v?!qd#IfPZPcw2%LYiGu>R3-!TOR8w;BkRvPJZ z<1P|<1jEhqmX!CpD8$C=lg1kT>N7Kph<^6FT^b28{B!#m_YD4Hb;JMNVeCko_YH{u#YGCNfqUR${lkfr76jj38W8IE>2dwouJwr|A^j2tV4qT@+Yh(yP}F zy!K}d=jy78%G+F0g|QhDn(v$;qAAZ^*B3@*$HN zX6MIcKJK~OG~5-1`x{6Hs$IsIihYZ|P^D?s0&f|4dY@lJLI?dVY@W_K3S%2v?KNjd z{DXpXIv3q)AUY!Y7|?~XN;c-%Q@H*7Y`kY}IciP3OR3~s%bubb8iA8m6ill_KV?0$ z%zD=62TVkDdpb%`_LA|?%7M=DlB7q&q04UE(~FTY0BKS<7{tLBg+6N=D7YIs*Sv*(Z^^TX{S>`FT1GUtEw45x6%) z9eIjcmR)_%f8?2KBT2)?T$)B;%)TeptL)e2s`xxLI1 zjOZwKbvcLV!bghx(V=e=ugusLy5;QU8D|pKQ3)$u(^!M0l@8kS6=@k`rELz6Om?x4 z8QG>$L*bz-B5XfH(DP6!oyb%ois;oGVITX(d#+G7<@Qy-#kZWhS%cO;b3@m5agR`P zA3Gv1!pON^r&&Mxx3zrEk@e+$Hq1Vvn>xb)hUklA<$MN4@=Ok`}$%sH3d;ztc%IGPG!kNc!PaM^hlf{2!b)^DI9EDKvo zbHY(w9Qc!GmW)hH_Tk;z&uvvImm3^%85>+plW=?6%bq`+9~^7E_>sPwpjdf#=jY=% ztoP0`U-e4uPVJ)ni_24u3q>RNnn~wT!k^K??RpnZ*_b=)hDvjF0x|6uoewhU1o1M< zw?}1C@(j_XqvE(moyTF9zW#bn>DV-*H<%MVt%v{_Qe#N zk25Y+Q#w43HP~jyj1s$g9M=Vf{xj{cWI!|J0QY-bSMzUaNM zVl{R$h>8Sy-m}mCiM{_DWjf5yeM_E{hbeSEdH(hBNh#sv1b5$Ynq-q!u|&6H@1By(c-PoMux*d)BN_M$g5 z+Hxdz(*McSGBwf{i6F~!&;Q^1lii4o`oKx}!c>%OQl9zA4d`TZ!M#G5MD;}Ncg3=q zh)S;#JK>;Kmu#c`fr|nwh}C^5gFMWi1$)Q1lN`+Ef%y>T2y={8p+;{GQX_{(?l=6; z9}V6RI5ty-cfKUQxm)Q)U5N2nvoqIex{Vqdsdah*@$?asm3HO^DawAIPPo_22=|lC zfI`_|0;}ZmX9W%1TEpPT!OkopgcfE^j7ARr*DJL9%SHa?%A^TAPx%)c z%>6H^*u1LUU z{pF6#$oKr5Qe`}?Vc4WHQ)?89T5Ym+Nwviedz$1z40W<>1aywypR0|;-LI~9$uL$^ zF+1K!{St%{Tu&AOk|lAD{(*uDY4;j>ICe8L^i|-Kk@F8FSuq*Xm1_8#4SDl8gKB(n z#`HUT75b0PNGUv>bvP7dtnPK~9rStPY)^ljKaM!*uQ`N*~`R;lmtb>8wvZOu| zN67H)=L){d?L$_jTl+{_(x(mc317k?(aj`mBxH;9=I)m+4cSdME&XCb97qvZ(M_ad z<5;y0Y1xv;ncJKwZKahUf|gSHylaQMpqjr6hVf_-#TDrniypVTxAeX}9~TOAYcLJL z(^(IhB~De}DGOGfk%el+qUfttMwONqO`@QS@)OP;x;1VypZ0u+W5${@SMCj(MYU%T z)+Kv$M)4;n89X_}Q~z})+Wy-BUfJ&;^a@OdT}HEpAKHy2bu|yXp1uAE12cijdaVWB z#!91HBxE%khfYJP=s5h~lM7#m`YU_#GA73-$}Mu}5{yKie|tZ^+rN&NAzXwF&lS1( zKWKY|un+vcV+|oxp*nIRaD^qDj)^<4VZjnxC4|X=S}&rysSGQ?>zsOZc|?{N6)Tg9 z2;q8VKm1|modh@1%t2iv>9ezMqsPV>q)E94vnOgF+v_DeY4w&>)DpX9RM8#$`CPwk zPlBw^6yXcLYGmK3*fpqo$?@-iSO0CmrhgBP|8c-ns!*N0U^NAymMX#3{DzgbAf3;p zY`$t`v8S@bifGD28&9eX5xeYLEdl1^wmWwPzc0IUoXbQI&!Vf*<}Mu4^8Lp}X0t5C z+6WcV;O(W#jn!cCq}s(y3l%6P9g1z&Y`}^jH_{as#zAg%#BUxf>VOsofpD5cfO}c` zsYd@)eI?>szt!4w?*@OIC|;$tQ6`Ax-1>0lQ`o6;zoP%H78;_Mu9Q~_P^U!cLd6Uvg!O`%BrkR6}g{QF@ z!|^6X2JSi)&m9#4ugEL9P97kC`Mq<*wCsu%P9!bC5*f+}lls0TM#OeG@@CgTE%dHVuGV(&lHbw$^9$XN zeZ8%HMQi4}Kka5_^J91UI=EdE8H$s~MqAlw!k|o_<9UVQ2-r0Y^KP`(q=?oM(M$4)Lrs|T5G}J47JTBR>3N!s)9=_^tH-fr zbg_Nj1a=kZyL9&Xe0`O&9uBK4sFqWJURf*+`abuh+;3?R|3ammMk;#LTvC`IdFZ+K z=a}&;gJrQ#FXmw_xm<{|_hm4bFZ)k)?_i(7u7Yj4^8aA9BCIgxwQachGM(OH@w4K{ zNiC)Jb3G#U%kCn+{icn4XX5kHz20#zWL{=6(zAHJ;mwNnTKB5Oiz^?u>V$RFOks$0 z=&qideKnafU>@^~d+L(3XLJdpzRpzZQL>GY>K~%#G3pK9mHf>uqn@p5`+7HQ-h0bZ zH8izkrK%LwQZ7RObF1b{NZ0V_I1Q`xu3=A|+f%~!w6nqE((#iM6_rN>Dn=3cjylSa z>!A*R4_>7>9*Gtc5ykalJ*fY%OJ4MW)SmmFTbiQ_({~8%4<$90By53MToG-oZ%RL_ zwj%xbo`-08`-#(O24ayHW`&iGl0-HaIF)v-3u5`hVaC;xrnwn#99$2Tq ziqFzqsF0$-l!G*#WQ3G5k zkX2Abxslyq1Lf1#Wb`v+PsBh8jFFtHh5U;J095gwl6ys=+6*vAs+4YLs6)d*B{}rW zFGhMYs&G*IIRE`J7uh*djqpmSnj9GuEhyLACHp_}gO)0R z>7<@M)w;)orM!&Zciaj`c4TrEq>+V|@wAXIW9D-z$D0huz?lfvl*?+7zQ(^*lt2@u zp#R}P;)cl+fRzH}mG`B8CnH(zRUDgm3j|ZyMXD}V0*J06fcGGXsjiLU+3K+sC@}y9 zwOBxBxRTl)bRdKWKEH6SN*0d#r+K(Q+E7x}fMh70v&0ZbE4 zIB7Z|sH}e=D?9n;D@TisM^B?k$lJ8w!zEIMSQ2y%NyyhN{OmYUQ=TmEnVSp%mib0i z1+@TD9NwV>WPo#hH)nG}CnLM1!JJ%BK3o6*&EzGc$pZW2F)&aj0^ixPUy(3V$WWbpD{bnW{86zO&J^X2IVSvwWU1nB! z1b|vflkX6P&+|XBgFf~He9Ke-wS|IegSJsfs2dCjW|x`8DXf;EAm3Ocp=6z||Mi|> zY^p|c)#F4FJ5bsb$eA*u`@Iir#R*jBY*<9i(*DTpTse8>8w?P&fc6B%+F=1WDgKWl zfSMwqApkqjTcBIGiVoRF@q~38fLZ>-Niucbi&1V7xcMmy0llIm{CT}~xH(2|E^)^Q zV2L+BbvWMW5Czu*e%_aW_8T_$@|EMQZ)QLt)PifLG13}rs(D$`Hc{oHpE-xVdjFk< zW8t%$M@GgKO-H{)5jaEZ^!Q>!_yQ|x3t-;4A*FtqM49Qg%z~>j9E~3!0A6GV%FMg= z0DEwxPTGx~t(O(6H;N2}mq$x8B*mDyiUI z$)J0@xI;WBE+=EEoKr#1v5}+Q>LzJW!lH-i13=Wt7!5;d-SSOj5$=Z&H`#mhe=H2g z0-Fw4SJG(=^uuNoI6}zj{I%e|w4h1PnBDhK@?6Mxg||?p9mNumat|W48tE(V5E`WrnVYg4S+#4#;6_gCvg#`3K@2eDQu2jbQx5um1M7n-s4Rj??hZ0Hk z{Qm|{?R^^n=-elW{^LJ8t(-@S#6j7brezBld*AYKVM8$)*#W?&kQuH5_JwT1P2CFW zEfQj;J3%lN^df4VrX_o?4OZ*@Z&JaWWatcwpi9sneNcZsPpUspfG0t7I$I3@FzW+V z{P|K)cLsMYWH$12liv)=v&L&Z^P0|)y+{9Da|Q3!jZ`u+c2FI+`eqgJ_FpGR;UN0o z&Ja@ZaXR1^``fHL|5+UU|K#-ao32_p7GC?6^elpgSweGJT8ierre=VK!Afk7cFHks z@1f%mVSwQ0J6K%w)7ym+*jF|Op?uIzqT&+6R$?KRhamEv##v!Wz{XJ22-m-ZPh)cVCG!GR9Y zTP6`m01{b^cbGEUe6cas;bIOk;`U6V>%g42({WHM%{u1^OAO!Cqo0a`&E+;y;BT?Q z_{}9-qzXe~h_x@ep5>k#Bz~}Tzh=tbg+cf%T?q__`B3y@+Bm6>D~Cowd02=7#)Bh z)jr`mRrvgy4bTf+vBA15dR3WD|pB z1#bpKzW&x3&wf{}YLJ20K2DJFavqVfM?fx8nnidiX_41k)v7^KPnDf2f|-*OyV!gF zFu|BRcjWeN>C&iA*;S7#63BH*UKnC?M@z~IGkQJ=K~<=sp3Wy95XC<}?1ZBoItJ;Q z8%MOwov<9EP=`dc+>DLn7wq+1_w>`=2j7gmdXXw2TE{qTyg-l^DT!tKR}xFQg?4n( z{w=d$`1?*?3;t@gNExHCDL{~>03TH?l0e|O()^!~(5JRX+~RX4xYS(GN$ z6FkyY8v95%$mM_iZafr0ryGv(>T8hIqCt3pGP9f?4}i&8VVV8ovG;e`EwFLQSIz^t zk23tXK(mZU4FbBES}7wtQfHfO=Uw|-ys!8BS|a9IX;n~c)$$lLSU^#fTj8pM0`99f z0zc^KKla;tg&Ug4+&_7YcJeBK4&8#rCvFtUu=eU6+TKw!V}h9**DL?_db*T@rv0Ca znwt3h`HjGyVHD_t!OBvkh1L2EiWm@&fJ7ZE)?2n7HT*SWFK-t!>W%8ETPv{OE#;(c$$Z5PN+s~~+O?0;HsG3W3?sX?gWNQFdd z_+tkaI=T-}9A2t?E`a+No>RtwE?NaZ4x_w}VaaXrQ;G8OvhSk7 z%-(P^LYMEt#;4Bet=-GTBF0J`hV-E3@%?7SBiEzXgQZGjJ9f=6 zDiWGzr5wa(F+K*&dA)*G4{wq6BwwBVyumD61_C*88zASyvSX^kpDyaloGj$QGGdM~R_B6B6wLWQV9 z&?H=@J!7Zp{k_6P9~8dE$*J@Q9rW&zd)*0``U*yM~tCP zPGTbKgm)S(Fjc`b${{zk*b^nDPV2sAW$qmwF`Y8%GiFZbWB*x&xHIb?6Vwyz0qqeWmXb#C=i=!gb4icR8C zMEvDPd~dWetcuG* zo2H)d5aZW$)JLrErY10`tq!EigA1ot_pwXAkGc?#A=uu1gdGM^KYhzET#Xv2NGw9q z2$3r)4$|63?#XIDyDv+o>{)yzoox6I5fEy#ZdbUXy3Hz*(W#*Ev^`jggibA+>N)N~ z)fG#WX!awyKU60kh`wZc20JRO5li3&<{jqZN{M)MOmT_=6TN4ebV(>)U2^U2*Awp! zPvK;6mD>gCAogN$;P$}LEOzyac-u-*nPYARawcTPq-pK`ZY3Y~u8bY&Eqg4TI2CBs6T9qWNAREbujqxuVP=DJT`(_5MD~4t+)+=wX z5vT7}R1;==HBn(Hm!_R!;7zHGelv`v-KvPll%Vgo^ix>SATbeWM-FSKVYpn9A4iI> z|8cZ=qA)dS+%Y>P)GBK&HwIRsaA=@=MD76ZFn7t|TAJraRv_yD;jz~3ODUt;~**e$gu`zq?bG0O^0?-OG&iJqKNRdp)2~BOmey`JO1oX}% zS+C)6rWPmef)NhS=9w8A_D!T9L-7V)50DIB`GM7*oEJ&v5BeFtOe|cF-msREv}oo* z_#l5M9$O$92!{=CRiYB851hnY@ z2^A7*K))-N6XBn^yOLIZCAeIbNt%ZKXy`Z}ix|D!a(X4OtTUFAEc3Nn#tXkmvKX`s z(?3~;g%08*_%*FUX15s9VQ+RZHo<%3R8Z`216bbt*ze4~*e1HT)TwiZ4OD`^gb7gm z9l&&F zhdjIUyRrF@sE2zYif-$V!qtZ^tGfcmvPYL_!br=0t7xJ;DR4AEb1L2r;=?!^xk6(|GuXXGv7?qr^=Jky^ ze!(uJIHLDTN4^@+?R+TG`n-$_h5B5@!{Lk4g7N1F>g@Dc?c_nI6bVXhI-4QWb}boa zmxmI;y}7nfV%J>J%=f*|WT&eIO0`vM9H_*dv>X+7dRT1}1=~?Pc~@{pQ76bE>?p!! z;D|p7q4|i|=KHezLU}1j%&O6AZAyYSEpJ;3DdAI9+S?Kj>&bl_jMh)@fz4EIR!?+N zNhkThk4|Ob*e>HHEmgR)J6mIzefxNP`f)nfn;^Ll9dGG{@j~ahxS{jXRc<`ZAw2Be zT6C2cUg0R(%TprK$z^Q{f;E}94Ui08XD+agUXnPUROnIZYw?_bXY(6vm0y}r3Y8o|%w_u&nfUT+No-C@!J zsY@KO7q-VPKybiy8|;hQ5OMrjGJf_gAatM7m|0*-$pBu}4d*Asq&BD$AU!7`6Q}&C zfE*=Yw~8DAqV9BHq0iS7j3pXsf&Q-qq-)&`Z}q5^rKDM?73sh#=yZ<4B52*9>5Mvr zz#<>8F~EvJ0PZYEMpNZr z_2$`q_S0u`K<_^cc?87VVqbwy((P--vs8*2eZMYN19Cwk>u#t4ceWNz3w^RtH2mHi zxYEptmRTKu?zoKrN)rTOPRQ~AI0?&O1~igZH@UY<&A~OYYQOj#@R+6pGaz_3D-Bqn zx;j8Zbptx!OdY^K?Q>u&tZ{?uZon=SF9hHMBiz|yRv#NT4USRJ44&$HYN(2=>l0&@ zH7qM%Zj4yOD3mYC;z@QSPY5A>o(Ondn&jjZ54-k$eT|MGy7<(B2DlqYF!rY|FkXIb zYXJ7hS8qA^3exR=2JR8ilQc1bFZm%_0krZ=|K9EF#VIv=5g>BDF_qz@NEOG*0aT~~ zx0w7$d&6oxY&ADuu&akn@%N`#tIp>b#avQM1%}&$jhnwx2#x^Q&caZ6S~fKSZo21m zD9fNH?Q~;jhNc}b}XD zG|x{)P!`A#V0d67;2J;VnWvxB+Nldd^L;UDFxPW=VbUGd)}4ff>fu2p>H51;g15Yu zb#Qizs7y3m3!SGlu6APb()Bg6TmWCRXpCB&7KD#HyD!J)PsK8P^%uI0JJZzMAB9MN zd{*xo*Gyb#0JH0o=dS^fLt1wYF_sAWpk8fQdLt1nqf4Kg9c}3Z2ZZp-zvLVEziJpR z?bYjGQNrUr<|jHQK#OvVsG5BIG&o+OL#kyDI8z`?N;OO%33*}ATWRopeM=nhJp57| z4d(iEY#dlByi9dr`?mjeg^0}s@86V4PCzRSD_ZAc5oNa%*)w4foRYE=4;<}i`!W|E zdK;`w6%10_Rmi4T2spQX8f0IxEp;B;FsFFJZvO2oAe1MV`DSv^1hy%-cu3usaUxn1 zcJKf+%Yh=ewkHYrN%}wm06Lsu%?EbI0OF@9ApQdgxMAl4OK2NvV>W<0wqyajxdW7S z)NBi{n*3|rfPl_>{n;HR)`vccYX9H-Bx~b!{?l2t3<(K)B3eNZzPP5M)V`QtR-ejD zEw*B(h&ycBU?BF$DNhdyFeg1Rxt9^eamf^cB{jOS5MxUlb@I^(!3=A`{X1+O?f}d`@VT@c1(qGZPsk2 z{$?`3iF61I#cRJ|C|J-k&JPI4j#)lz7pM9Wzl`9X8z_eV&&($;h7P=f2RUR$uRnfS~%`1+3`K&Ecv zCn{{Ms&3a_sAIuNG_X?=a8g<|$XtAld!e^8(WqS2uIgSgx;t3JWk02&9o`E@`=unS zNrvc;-RXt#^IJtRo@viwtp(b~buf7e9j!t_>(kgZNS0r$Bubwot>3 zaM@q4O2dK__bS?<*vneIT$Jy5EqjaBtdDEm>x=BSntD`M*mGC4K0(f+J~19?P-&_; z5Ac(BwqX@Q=^NxGC*wQ$nFODUq=)F@SnxGUnaoO7_FwHIh{(D4x}weZRA zP7Jsrov`YJAS<2T>e|`j-0vJQ_UmbttFAAF-%x>gN79EWDaacMSsVo4r3X#?j6)}e z&JCa>zZ?wU%q20;<);`N-7pbsn#DA#h*IC``RWA)iU!)t$h^I8auG^>g!E%WH7L1T z(i(ap-n}F6qtfi|6T3rpD9Y`)8~T>9tx5DF{=#9^QQwEQ51r$E> zX$RcufW~(mefuw~t^F$^EMqxxa;0hz-HCJQqc=n+dkO-^$o>ie?z5MbX$$iII@N*? zjtburRBTw9CAhQsjKWl5Mk&pr{aojCAgGhvx*3r(iZDXxzql(QCK^ar(|8xyDZPN?gug$+*Y6> zPsq04X*JS#wjAe*cyDa1Jak(7_P45FeY3G_vlwaotQ`K?!U1AWl82`Iv#nz`*A@W;#M5a?M?w4;}e|VP~?ht%fW>{HQ`23J^$m3m- z{sHKzDgir?ZJB@QP)5_)L+emgMxTP2u2|yC)`6{jlf@-1depBo$AyIBtTO7L+1wa^ zZI_@bx@2*&tS4(nf8I*R@LgQXyYJN2EG{jO<9xj^D2o_k;mXSD#mUV8^q=iU|5wU? z|MO1NAYMD2~FQa;133a{i+D^BpB>qBFN8SurrDv$%4Ud>Vb?3ybJ>- zbns>F4%m*uU~i$SDh$>Jl~7@@pU?lFeB-erdYls%cbQm2pUGyj%C_&Q>(bzQp^uvn amZGosHYwG~%UeOOFPK>VP>{<<.LabelMatchers>>}) by (pod_name) - resources: - namespaced: false +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +helm install prometheus-adapter -n crane-system prometheus-community/prometheus-adapter +``` +再将 ApiService 改回 Crane 的 Metric-Adapter + +```bash +kubectl apply -f https://raw.githubusercontent.com/gocrane/crane/main/deploy/metric-adapter/apiservice.yaml ``` -配置添加后重启prometheus-adapter -查询外部指标状态 +### 配置 Metric-Adapter 开启 RemoteAdapter 功能 -external-apiservice +在安装 PrometheusAdapter 时没有将 ApiService 指向 PrometheusAdapter,因此为了让 PrometheusAdapter 也可以提供自定义 Metric,通过 Crane Metric Adapter 的 `RemoteAdapter` 功能将请求转发给 PrometheusAdapter。 -```bash -# kubectl get apiservice v1beta1.external.metrics.k8s.io -NAME SERVICE AVAILABLE AGE -v1beta1.external.metrics.k8s.io monitoring/prometheus-adapter True 36d -# kubectl get --raw /apis/external.metrics.k8s.io/v1beta1|tee |python -m json.tool -{ - "apiVersion": "v1", - "groupVersion": "external.metrics.k8s.io/v1beta1", - "kind":"APIResourceList", - "resources": [ - { - "kind": "ExternalMetricValueList", - "name": "mock_traffic", - "namespaced": true, - "singularName":"", - "verbs": [ - "get" - ] - } - ] -} +修改 Metric-Adapter 的配置,将 PrometheusAdapter 的 Service 配置成 Crane Metric Adapter 的 RemoteAdapter +```bash +# 查看当前集群 ApiService +kubectl edit deploy metric-adapter -n crane-system ``` -### Metric-Adapter - -镜像及版本:docker.io/gocrane/metric-adapter:v0.5.0-tke.1-7-g10ddeb6 -注:时序预测模型功能需要通过metric-adapter获取,因此需要对prometheus-adapter与metric-adapter进行指标集成 +根据 PrometheusAdapter 的配置做以下修改: -#### 配置remote-adapter ```yaml apiVersion: apps/v1 kind: Deployment @@ -122,96 +84,195 @@ spec: spec: containers: - args: -#添加外部Adapter + #添加外部 Adapter 配置 - --remote-adapter=true - - --remote-adapter-service-namespace=monitoring + - --remote-adapter-service-namespace=crane-system - --remote-adapter-service-name=prometheus-adapter - --remote-adapter-service-port=443 ``` -#### 修改apiservice +#### RemoteAdapter 能力 + +![](../images/remote-adapter.png) + +Kubernetes 限制一个 ApiService 只能配置一个后端服务,因此,为了在一个集群内使用 Crane 提供的 Metric 和 PrometheusAdapter 提供的 Metric,Crane 支持了 RemoteAdapter 解决此问题 + +- Crane Metric-Adapter 支持配置一个 Kubernetes Service 作为一个远程 Adapter +- Crane Metric-Adapter 处理请求时会先检查是否是 Crane 提供的 Local Metric,如果不是,则转发给远程 Adapter -指定外部指标源为metric-adapter,prometheus-adapter指标通过metric-adapter代理 +## 运行例子 + +### 准备应用 + +将以下应用部署到集群中,应用暴露了 Metric 展示每秒收到的 http 请求数量。 + +

sample-app.deploy.yaml + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sample-app + labels: + app: sample-app +spec: + replicas: 1 + selector: + matchLabels: + app: sample-app + template: + metadata: + labels: + app: sample-app + spec: + containers: + - image: luxas/autoscale-demo:v0.1.2 + name: metrics-provider + ports: + - name: http + containerPort: 8080 +``` + +sample-app.service.yaml + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: sample-app + name: sample-app +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: sample-app + type: ClusterIP +``` ```bash -# kubectl edit apiservice v1beta1.external.metrics.k8s.io -# kubectl get apiservice v1beta1.external.metrics.k8s.io -o yaml +kubectl create -f sample-app.deploy.yaml +kubectl create -f sample-app.service.yaml +``` + +当应用部署完成后,您可以通过命令检查 `http_requests_total` Metric: + +```bash +curl http://$(kubectl get service sample-app -o jsonpath='{ .spec.clusterIP }')/metrics +``` + +### 配置采集规则 + +配置 Prometheus 的 ScrapeConfig,收集应用的 Metric: http_requests_total -#外部指标正常 -# kubectl get --raw /apis/external.metrics.k8s.io/v1beta1|tee |python -m json.tool +```bash +kubectl edit configmap -n crane-system prometheus-server +``` + +添加以下配置 + +```yaml + - job_name: sample-app + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: default;sample-app-(.+) + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod +``` + +此时,您可以在 Prometheus 查询 psql:sum(rate(http_requests_total[5m])) by (pod) + +### 验证 PrometheusAdapter + +PrometheusAdapter 默认的 Rule 配置支持将 http_requests_total 转换成 Pods 类型的 Custom Metric,通过命令验证: + +```bash +kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 | jq . +``` + +结果应包括 `pods/http_requests`: + +```bash { - "apiVersion":"v1", - "groupVersion":"external.metrics.k8s.io/v1beta1", - "kind":"APIResourceList", - "resources": [ - { - "kind":"ExternalMetricValueList", - "name":"mock_traffic", - "namespaced":true, - "singularName":"", - "verbs": [ - "get" - ] - } -] + "name": "pods/http_requests", + "singularName": "", + "namespaced": true, + "kind": "MetricValueList", + "verbs": [ + "get" + ] } ``` -## 配置弹性 +这表明现在可以通过 Pod Metric 配置 HPA。 + +### 配置弹性 + +现在我们可以创建 Effective HPA。此时 Effective HPA 可以通过 Pod Metric `http_requests` 进行弹性: + +#### 如何定义一个自定义指标开启预测功能 -### EHPA +在 Effective HPA 的 Annotation 按以下规则添加配置: + +```yaml +annotations: + # metric-query.autoscaling.crane.io 是固定的前缀,后面是 Metric 名字,需跟 spec.metrics 中的 Metric.name 相同,支持 Pods 类型和 External 类型 + metric-query.autoscaling.crane.io/http_requests: "sum(rate(http_requests_total[5m])) by (pod)" +``` -#### 设置EHPA +sample-app-hpa.yaml ```yaml apiVersion: autoscaling.crane.io/v1alpha1 kind: EffectiveHorizontalPodAutoscaler metadata: - name: metric-source-service + name: php-apache annotations: - metric-name.autoscaling.crane.io/mock_traffic: | -#添加注解,当前版本需要配置查询语句 -#metric-name.autoscaling.crane.io/${需要获取时序模型的指标名}: - mock_traffic{job="metrics-service1-lyg-test", pod_namespace="cloud", pod_project="metric-external-service"} + # metric-query.autoscaling.crane.io 是固定的前缀,后面是 Metric 名字,需跟 spec.metrics 中的 Metric.name 相同,支持 Pods 类型和 External 类型 + metric-query.autoscaling.crane.io/http_requests: "sum(rate(http_requests_total[5m])) by (pod)" spec: - behavior: - scaleDown: - stabilizationWindowSeconds: 6 - policies: - - type: Percent - value: 100 - periodSeconds: 15 - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 15 - - type: Pods - value: 2 - periodSeconds: 15 - selectPolicy: Max + # ScaleTargetRef is the reference to the workload that should be scaled. scaleTargetRef: -#指定需要实现扩缩容的deployment apiVersion: apps/v1 kind: Deployment - name: metric-source-service - minReplicas: 2 - maxReplicas: 20 -#Auto为应用,Previoew为DryRun模式 - scaleStrategy: Auto + name: sample-app + minReplicas: 1 # MinReplicas is the lower limit replicas to the scale target which the autoscaler can scale down to. + maxReplicas: 10 # MaxReplicas is the upper limit replicas to the scale target which the autoscaler can scale up to. + scaleStrategy: Auto # ScaleStrategy indicate the strategy to scaling target, value can be "Auto" and "Manual". + # Metrics contains the specifications for which to use to calculate the desired replica count. metrics: -#控制扩缩容的外部指标 - - type: External - external: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + - type: Pods + pods: metric: - name: mock_traffic + name: http_requests target: - averageValue: 2000 type: AverageValue -#设置时序预测模型 + averageValue: 500m + # Prediction defines configurations for predict resources. + # If unspecified, defaults don't enable prediction. prediction: - predictionWindowSeconds: 3600 + predictionWindowSeconds: 3600 # PredictionWindowSeconds is the time window to predict metrics in the future. predictionAlgorithm: algorithmType: dsp dsp: @@ -219,141 +280,123 @@ spec: historyLength: "7d" ``` -应用并查看EHPA状态 - ```bash -# kubectl apply -f ehpa.yaml -#查询ehpa状态 -# kubectl get ehpa -NAME STRATEGY MINPODS MAXPODS SPECIFICPODS REPLICAS AGE -metric-source-service Auto 2 20 10 1m +kubectl create -f sample-app-hpa.yaml ``` -#### 时序预测模型 -注:craned定义TimeSeriesPrediction资源作为时序预测模型,命名定义ehpa-${ehpa-name} -增加时序预测指标,命名定义crane-${extrenal-metric-name} +查看 TimeSeriesPrediction 状态,如果应用运行时间较短,可能会无法预测: -```bash -# kubectl get tsp -NAME TARGETREFNAME TARGETREFKIND PREDICTIONWINDOWSECONDS AGE -ehpa-metric-source-service metric-source-service Deployment 3600 1m -# kubectl get tsp ehpa-metric-source-service -o yaml +```yaml apiVersion: prediction.crane.io/v1alpha1 kind: TimeSeriesPrediction metadata: - name: ehpa-metric-source-service + creationTimestamp: "2022-07-11T16:10:09Z" + generation: 1 + labels: + app.kubernetes.io/managed-by: effective-hpa-controller + app.kubernetes.io/name: ehpa-php-apache + app.kubernetes.io/part-of: php-apache + autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 + name: ehpa-php-apache namespace: default spec: predictionMetrics: - - algorithm: - algorithmType: dsp - dsp: - estimators: {} - historyLength: 7d - sampleInterval: 60s - expressionQuery: - expression: | - mock_traffic{job="metrics-service1-lyg-test", pod_namespace="cloud", pod_project="metric-external-service"} - resourceIdentifier: crane-mock_traffic - type: ExpressionQuery + - algorithm: + algorithmType: dsp + dsp: + estimators: {} + historyLength: 7d + sampleInterval: 60s + resourceIdentifier: crane_pod_cpu_usage + resourceQuery: cpu + type: ResourceQuery + - algorithm: + algorithmType: dsp + dsp: + estimators: {} + historyLength: 7d + sampleInterval: 60s + expressionQuery: + expression: sum(rate(http_requests_total[5m])) by (pod) + resourceIdentifier: crane_custom.pods_http_requests + type: ExpressionQuery + predictionWindowSeconds: 3600 targetRef: apiVersion: apps/v1 kind: Deployment - name: metric-source-service + name: sample-app namespace: default status: + conditions: + - lastTransitionTime: "2022-07-12T06:54:42Z" + message: not all metric predicted + reason: PredictPartial + status: "False" + type: Ready predictionMetrics: - - prediction: - - labels: - samples: - - timestamp: 1656402060 - value: "15767.77871" - ... - ... - - timestamp: 1656409200 - value: "22202.37755" - resourceIdentifier: crane-mock_traffic -``` - -#### HPA - -HPA通过相关指标实现扩缩 - -```bash -# kubectl get hpa -NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE -metric-source-service Deployment/metric-source-service 1480200m/2k (avg), 2020300m/2k (avg) 2 20 10 1m -# kubectl describe hpa ehpa-metric-source-service -Name: ehpa-metric-source-service -Namespace: default -Labels: app.kubernetes.io/managed-by=effective-hpa-controller - app.kubernetes.io/name=ehpa-metric-source-service - app.kubernetes.io/part-of=metric-source-service - autoscaling.crane.io/effective-hpa-uid=b2cb76db-61c9-4d00-b333-af67d36bbd65 -Annotations: -CreationTimestamp: Tue, 28 Jun 2022 13:38:06 +0800 -Reference: Deployment/metric-source-service -Metrics: ( current / target ) - "mock_traffic" (target average value): 1470600m / 2k - "crane-mock_traffic" (target average value): 2032500m / 2k -Min replicas: 2 -Max replicas: 20 -Behavior: - Scale Up: - Stabilization Window: 0 seconds - Select Policy: Max - Policies: - - Type: Percent Value: 100 Period: 15 seconds - - Type: Pods Value: 2 Period: 15 seconds - Scale Down: - Stabilization Window: 6 seconds - Select Policy: Max - Policies: - - Type: Percent Value: 100 Period: 15 seconds -Deployment pods: 10 current / 10 desired + - ready: false + resourceIdentifier: crane_pod_cpu_usage + - prediction: + - labels: + - name: pod + value: sample-app-7cfb596f98-8h5vv + samples: + - timestamp: 1657608900 + value: "0.01683" + - timestamp: 1657608960 + value: "0.01683" + ...... + ready: true + resourceIdentifier: crane_custom.pods_http_requests ``` -### craned指标采集 +查看 Effective HPA 创建的 HPA 对象,可以观测到已经创建出基于自定义指标预测的 Metric: `crane_custom.pods_http_requests` -#### 副本指标 - -```bash -# kubectl -n crane-system get deployment craned -o yaml -apiVersion: apps/v1 -kind: Deployment +```yaml +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler metadata: - name: craned - namespace: crane-system + creationTimestamp: "2022-07-11T16:10:10Z" + labels: + app.kubernetes.io/managed-by: effective-hpa-controller + app.kubernetes.io/name: ehpa-php-apache + app.kubernetes.io/part-of: php-apache + autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 + name: ehpa-php-apache + namespace: default spec: - template: - metadata: - annotations: - prometheus.aispeech.com/metric_path: /metrics - prometheus.aispeech.com/scrape_port: "8080" - prometheus.aispeech.com/scrape_scheme: http - prometheus.aispeech.com/should_be_scraped: "true" - -# kubectl -n crane-system get pods craned-854bcdb88b-d5fgx -o wide -NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -craned-854bcdb88b-d5fgx 2/2 Running 0 96m 10.244.0.177 d2-node-012 -#指标查询 -# curl -sL 10.244.0.177:8080/metrics | grep ehpa -crane_autoscaling_effective_hpa_replicas{name="metric-source-service",namespace="default"} 10 -crane_autoscaling_hpa_replicas{name="ehpa-metric-source-service",namespace="default"} 10 -crane_autoscaling_hpa_scale_count{name="ehpa-metric-source-service",namespace="default",type="hpa"} 3 -``` - -#### TSP指标 - -```bash -# curl -sL 10.244.0.177:8080/metrics | grep ^crane_prediction_time_series_prediction -crane_prediction_time_series_prediction_external_by_window{algorithm="dsp",resourceIdentifier="crane-mock_traffic",targetKind="Deployment",targetName="metric-source-service",targetNamespace="default",type="ExpressionQuery"} 23011 1657270905000 -crane_prediction_time_series_prediction_resource{algorithm="dsp",resourceIdentifier="crane_pod_cpu_usage",targetKind="Deployment",targetName="metric-source-service",namespace="default"} 10 + maxReplicas: 10 + metrics: + - pods: + metric: + name: http_requests + target: + averageValue: 500m + type: AverageValue + type: Pods + - pods: + metric: + name: crane_custom.pods_http_requests + selector: + matchLabels: + autoscaling.crane.io/effective-hpa-uid: 1322c5ac-a1c6-4c71-98d6-e85d07b22da0 + target: + averageValue: 500m + type: AverageValue + type: Pods + - resource: + name: cpu + target: + averageUtilization: 50 + type: Utilization + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: sample-app ``` -## 总结: +## 总结 -基于历史指标预测功能实现原理: -● EHPA开启预测,建立相应指标的时序预测模型【TimeSeriesPrediction】 -● 创建HPA,在原有指标基础上,增加时序预测模型指标 -● HPA基于metric-adapter服务获取时序预测模型指标,实现服务提前扩容 +由于生产环境的复杂性,基于多指标的弹性(CPU/Memory/自定义指标)往往是生产应用的常见选择,因此 Effective HPA 通过预测算法覆盖了多指标的弹性,达到了帮助更多业务在生产环境落地水平弹性的成效。 diff --git a/mkdocs.yml b/mkdocs.yml index d80e64ab7..314504085 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,6 +56,7 @@ plugins: Roadmap: 路线图 Overview: 概述 Load-aware Scheduling: 负载感知调度 + Custom Metric Prediction With Prometheus: 基于自定义指标的弹性预测 zh_TW: Getting Started: 入門 Introduction: 介紹 @@ -111,6 +112,8 @@ nav: - Crane-scheduler: - Overview: tutorials/scheduling-pods-based-on-actual-node-load.md - Load-aware Scheduling: tutorials/dynamic-scheduler-plugin.md + - Best Practices: + - Custom Metric Prediction With Prometheus: tutorials/effective-hpa-with-prometheus-adapter.zh.md - Proposals: - Advanced CpuSet Manager: proposals/20220228-advanced-cpuset-manger.md - Pod Sorting And Precise Execution For Crane Agent: proposals/Pod-Sorting-And-Precise-Execution-For-Crane-Agent.md diff --git a/pkg/metricprovider/custom_metric_provider.go b/pkg/metricprovider/custom_metric_provider.go index 896683a22..54c827e67 100644 --- a/pkg/metricprovider/custom_metric_provider.go +++ b/pkg/metricprovider/custom_metric_provider.go @@ -22,7 +22,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" - autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" predictionapi "github.com/gocrane/api/prediction/v1alpha1" "github.com/gocrane/crane/pkg/known" @@ -182,33 +181,24 @@ func ListAllLocalMetrics(client client.Client) []provider.CustomMetricInfo { Metric: known.MetricNamePodCpuUsage, }) - var ehpaList autoscalingapi.EffectiveHorizontalPodAutoscalerList - err := client.List(context.TODO(), &ehpaList) + var hpaList autoscalingv2.HorizontalPodAutoscalerList + err := client.List(context.TODO(), &hpaList) if err != nil { - klog.Errorf("Failed to list ehpa: %v", err) + klog.Errorf("Failed to list hpa: %v", err) return metricInfos } - for _, ehpa := range ehpaList.Items { - for _, metric := range ehpa.Spec.Metrics { + for _, hpa := range hpaList.Items { + if !strings.HasPrefix(hpa.Name, "ehpa-") { + // filter hpa that not created by ehpa + continue + } + for _, metric := range hpa.Spec.Metrics { if metric.Type == autoscalingv2.PodsMetricSourceType && metric.Pods != nil && metric.Pods.Metric.Selector != nil && metric.Pods.Metric.Selector.MatchLabels != nil { if _, exist := metric.Pods.Metric.Selector.MatchLabels[known.EffectiveHorizontalPodAutoscalerUidLabel]; exist { - metricName := utils.GetGeneralPredictionMetricName(autoscalingv2.PodsMetricSourceType, false, metric.Pods.Metric.Name) - metricInfos = append(metricInfos, provider.CustomMetricInfo{Metric: metricName}) - } - } - } - - for _, metric := range ehpa.Spec.Metrics { - if metric.Type == autoscalingv2.ObjectMetricSourceType && - metric.Object != nil && - metric.Object.Metric.Selector != nil && - metric.Object.Metric.Selector.MatchLabels != nil { - if _, exist := metric.Object.Metric.Selector.MatchLabels[known.EffectiveHorizontalPodAutoscalerUidLabel]; exist { - metricName := utils.GetGeneralPredictionMetricName(autoscalingv2.ObjectMetricSourceType, false, metric.Object.Metric.Name) - metricInfos = append(metricInfos, provider.CustomMetricInfo{Metric: metricName}) + metricInfos = append(metricInfos, provider.CustomMetricInfo{Metric: metric.Pods.Metric.Name, Namespaced: true, GroupResource: schema.GroupResource{Group: "", Resource: "pods"}}) } } } diff --git a/pkg/metricprovider/external_metric_provider.go b/pkg/metricprovider/external_metric_provider.go index 751e75070..62a9c7335 100644 --- a/pkg/metricprovider/external_metric_provider.go +++ b/pkg/metricprovider/external_metric_provider.go @@ -229,14 +229,26 @@ func ListAllLocalExternalMetrics(client client.Client) []provider.ExternalMetric metricName := utils.GetGeneralPredictionMetricName(autoscalingv2.PodsMetricSourceType, true, ehpa.Name) metricInfos = append(metricInfos, provider.ExternalMetricInfo{Metric: metricName}) } - for _, metric := range ehpa.Spec.Metrics { + } + + var hpaList autoscalingv2.HorizontalPodAutoscalerList + err = client.List(context.TODO(), &hpaList) + if err != nil { + klog.Errorf("Failed to list hpa: %v", err) + return metricInfos + } + for _, hpa := range hpaList.Items { + if !strings.HasPrefix(hpa.Name, "ehpa-") { + // filter hpa that not created by ehpa + continue + } + for _, metric := range hpa.Spec.Metrics { if metric.Type == autoscalingv2.ExternalMetricSourceType && metric.External != nil && metric.External.Metric.Selector != nil && metric.External.Metric.Selector.MatchLabels != nil { if _, exist := metric.External.Metric.Selector.MatchLabels[known.EffectiveHorizontalPodAutoscalerUidLabel]; exist { - metricName := utils.GetGeneralPredictionMetricName(autoscalingv2.ExternalMetricSourceType, false, ehpa.Name) - metricInfos = append(metricInfos, provider.ExternalMetricInfo{Metric: metricName}) + metricInfos = append(metricInfos, provider.ExternalMetricInfo{Metric: metric.External.Metric.Name}) } } }