From b134e93ac0c5fd58751c75a412b7419a54effe5f Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 6 Jul 2022 10:15:17 +0000 Subject: [PATCH 1/6] Add memory usage statistics for slabs and allocation classes --- cachelib/allocator/PromotionStrategy.h | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 cachelib/allocator/PromotionStrategy.h diff --git a/cachelib/allocator/PromotionStrategy.h b/cachelib/allocator/PromotionStrategy.h new file mode 100644 index 0000000000..1c4b6d9c8a --- /dev/null +++ b/cachelib/allocator/PromotionStrategy.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" + +namespace facebook { +namespace cachelib { + + +// Base class for background eviction strategy. +class PromotionStrategy : public BackgroundEvictorStrategy { + +public: + PromotionStrategy(uint64_t promotionAcWatermark, uint64_t maxPromotionBatch, uint64_t minPromotionBatch): + promotionAcWatermark(promotionAcWatermark), maxPromotionBatch(maxPromotionBatch), minPromotionBatch(minPromotionBatch) + { + + } + ~PromotionStrategy() {} + + std::vector calculateBatchSizes(const CacheBase& cache, + std::vector> acVec) { + std::vector batches{}; + for (auto [tid, pid, cid] : acVec) { + XDCHECK(tid > 0); + auto stats = cache.getAllocationClassStats(tid - 1, pid, cid); + if (stats.approxFreePercent < promotionAcWatermark) + batches.push_back(0); + else { + auto maxPossibleItemsToPromote = static_cast((promotionAcWatermark - stats.approxFreePercent) * + stats.memorySize / stats.allocSize); + batches.push_back(maxPossibleItemsToPromote); + } + } + + auto maxBatch = *std::max_element(batches.begin(), batches.end()); + if (maxBatch == 0) + return batches; + + std::transform(batches.begin(), batches.end(), batches.begin(), [&](auto numItems){ + auto cappedBatchSize = maxPromotionBatch * numItems / maxBatch; + if (cappedBatchSize < minPromotionBatch) + return 0UL; + else + return cappedBatchSize; + }); + + return batches; + } +private: + double promotionAcWatermark{4.0}; + uint64_t maxPromotionBatch{40}; + uint64_t minPromotionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook From b4878e3aa57bad95ddd6ec1dca560e6f6a396af9 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Sat, 11 Jun 2022 04:17:55 -0400 Subject: [PATCH 2/6] Implement background promotion and eviction and add additional parameters to control allocation and eviction of items. Co-authored-by: Daniel Byrne --- MultiTierDataMovement.md | 107 ++++++ cachelib-background-evictor.png | Bin 0 -> 58912 bytes cachelib/allocator/BackgroundEvictor-inl.h | 115 ++++++ cachelib/allocator/BackgroundEvictor.h | 99 ++++++ .../allocator/BackgroundEvictorStrategy.h | 35 ++ cachelib/allocator/BackgroundPromoter-inl.h | 112 ++++++ cachelib/allocator/BackgroundPromoter.h | 98 +++++ cachelib/allocator/CMakeLists.txt | 1 + cachelib/allocator/Cache.h | 6 + cachelib/allocator/CacheAllocator-inl.h | 335 ++++++++++++++++-- cachelib/allocator/CacheAllocator.h | 228 +++++++++++- cachelib/allocator/CacheAllocatorConfig.h | 75 ++++ cachelib/allocator/CacheStats.h | 42 +++ cachelib/allocator/FreeThresholdStrategy.cpp | 44 +++ cachelib/allocator/FreeThresholdStrategy.h | 44 +++ cachelib/allocator/MM2Q-inl.h | 42 ++- cachelib/allocator/MM2Q.h | 10 + cachelib/allocator/MMLru-inl.h | 15 + cachelib/allocator/MMLru.h | 5 + cachelib/allocator/MMTinyLFU-inl.h | 7 + cachelib/allocator/MMTinyLFU.h | 3 + cachelib/allocator/MemoryTierCacheConfig.h | 4 + cachelib/allocator/memory/MemoryAllocator.h | 3 +- .../allocator/memory/MemoryAllocatorStats.h | 4 + cachelib/allocator/memory/MemoryPool.h | 3 +- cachelib/allocator/nvmcache/CacheApiWrapper.h | 2 +- .../tests/AllocatorMemoryTiersTest.cpp | 2 + .../tests/AllocatorMemoryTiersTest.h | 91 +++++ cachelib/allocator/tests/CacheBaseTest.cpp | 2 + cachelib/cachebench/cache/Cache-inl.h | 48 +++ cachelib/cachebench/cache/CacheStats.h | 60 +++- .../config-4GB-DRAM-4GB-PMEM.json | 8 +- cachelib/cachebench/util/CacheConfig.cpp | 46 ++- cachelib/cachebench/util/CacheConfig.h | 33 ++ 34 files changed, 1690 insertions(+), 39 deletions(-) create mode 100644 MultiTierDataMovement.md create mode 100644 cachelib-background-evictor.png create mode 100644 cachelib/allocator/BackgroundEvictor-inl.h create mode 100644 cachelib/allocator/BackgroundEvictor.h create mode 100644 cachelib/allocator/BackgroundEvictorStrategy.h create mode 100644 cachelib/allocator/BackgroundPromoter-inl.h create mode 100644 cachelib/allocator/BackgroundPromoter.h create mode 100644 cachelib/allocator/FreeThresholdStrategy.cpp create mode 100644 cachelib/allocator/FreeThresholdStrategy.h diff --git a/MultiTierDataMovement.md b/MultiTierDataMovement.md new file mode 100644 index 0000000000..8f518c66fa --- /dev/null +++ b/MultiTierDataMovement.md @@ -0,0 +1,107 @@ +# Background Data Movement + +In order to reduce the number of online evictions and support asynchronous +promotion - we have added two periodic workers to handle eviction and promotion. + +The diagram below shows a simplified version of how the background evictor +thread (green) is integrated to the CacheLib architecture. + +

+ BackgroundEvictor +

+ +## Synchronous Eviction and Promotion + +- `disableEvictionToMemory`: Disables eviction to memory (item is always evicted to NVMe or removed +on eviction) + +## Background Evictors + +The background evictors scan each class to see if there are objects to move the next (higher) +tier using a given strategy. Here we document the parameters for the different +strategies and general parameters. + +- `backgroundEvictorIntervalMilSec`: The interval that this thread runs for - by default +the background evictor threads will wake up every 10 ms to scan the AllocationClasses. Also, +the background evictor thead will be woken up everytime there is a failed allocation (from +a request handling thread) and the current percentage of free memory for the +AllocationClass is lower than `lowEvictionAcWatermark`. This may render the interval parameter +not as important when there are many allocations occuring from request handling threads. + +- `evictorThreads`: The number of background evictors to run - each thread is a assigned +a set of AllocationClasses to scan and evict objects from. Currently, each thread gets +an equal number of classes to scan - but as object size distribution may be unequal - future +versions will attempt to balance the classes among threads. The range is 1 to number of AllocationClasses. +The default is 1. + +- `evictionHotnessThreshold`: The number of objects to remove in a given eviction call. The +default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not +remove objects at a reasonable rate, too high and we hold the locks for copying data +between tiers for too long. + + +### FreeThresholdStrategy (default) + +- `lowEvictionAcWatermark`: Triggers background eviction thread to run +when this percentage of the AllocationClass is free. +The default is `2.0`, to avoid wasting capacity we don't set this above `10.0`. + +- `highEvictionAcWatermark`: Stop the evictions from an AllocationClass when this +percentage of the AllocationClass is free. The default is `5.0`, to avoid wasting capacity we +don't set this above `10`. + + +## Background Promoters + +The background promotes scan each class to see if there are objects to move to a lower +tier using a given strategy. Here we document the parameters for the different +strategies and general parameters. + +- `backgroundPromoterIntervalMilSec`: The interval that this thread runs for - by default +the background promoter threads will wake up every 10 ms to scan the AllocationClasses for +objects to promote. + +- `promoterThreads`: The number of background promoters to run - each thread is a assigned +a set of AllocationClasses to scan and promote objects from. Currently, each thread gets +an equal number of classes to scan - but as object size distribution may be unequal - future +versions will attempt to balance the classes among threads. The range is `1` to number of AllocationClasses. The default is `1`. + +- `evictionHotnessThreshold`: The number of objects to remove in a given eviction call. The +default is 50. Lower range is 10 and the upper range is 1000. Too low and we might not +remove objects at a reasonable rate, too high and we hold the locks for copying data +between tiers for too long. + +- `numDuplicateElements`: This allows us to promote items that have existing handles (read-only) since +we won't need to modify the data when a user is done with the data. Therefore, for a short time +the data could reside in both tiers until it is evicted from its current tier. The default is to +not allow this (0). Setting the value to 100 will enable duplicate elements in tiers. + +### Background Promotion Strategy (only one currently) + +- `promotionAcWatermark`: Promote items if there is at least this +percent of free AllocationClasses. Promotion thread will attempt to move `evictionHotnessThreshold` number of objects +to that tier. The objects are chosen from the head of the LRU. The default is `4.0`. +This value should correlate with `lowEvictionAcWatermark`, `highEvictionAcWatermark`, `minAcAllocationWatermark`, `maxAcAllocationWatermark`. +- `promotionHotnessThreshold`: The number of objects to promote in batch during BG promotion. Analogous to +`evictionHotnessThreshold`. It's value should be lower to decrease contention on hot items. + +## Allocation policies + +- `maxAcAllocationWatermark`: Item is always allocated in topmost tier if at least this +percentage of the AllocationClass is free. +- `minAcAllocationWatermark`: Item is always allocated in bottom tier if only this percent +of the AllocationClass is free. If percentage of free AllocationClasses is between `maxAcAllocationWatermark` +and `minAcAllocationWatermark`: then extra checks (described below) are performed to decide where to put the element. + +By default, allocation will always be performed from the upper tier. + +### Extra policies (used only when percentage of free AllocationClasses is between `maxAcAllocationWatermark` +and `minAcAllocationWatermark`) +- `sizeThresholdPolicy`: If item is smaller than this value, always allocate it in upper tier. +- `defaultTierChancePercentage`: Change (0-100%) of allocating item in top tier + +## MMContainer options + +- `lruInsertionPointSpec`: Can be set per tier when LRU2Q is used. Determines where new items are +inserted. 0 = insert to hot queue, 1 = insert to warm queue, 2 = insert to cold queue +- `markUsefulChance`: Per-tier, determines chance of moving item to the head of LRU on access diff --git a/cachelib-background-evictor.png b/cachelib-background-evictor.png new file mode 100644 index 0000000000000000000000000000000000000000..08869029c20132d7fca7534a98fba6b69673b943 GIT binary patch literal 58912 zcmeFZWl)@56E=t?KoWw5;O-tgIE3ID+zBBJ?(PsQA-EIV-3RyJ?t@zh?(VSnkUa1E z)c0%m*X~xmR8a%>oH^2`PxsY*^_g!9a^fh@@SefIz@SJJ2?Y!|QXvxLz1*%-%K{5+>w_k)nOJQK4OOd$n)=IYQ>Q9OA07SHLZDFsuU$@) zTJz1`3>%v#78cf1W+WjPI5ck<1QHlnB%w4A!mSx70PD|xf8SUp@%COn_*W|!I9)U$ zAvf6PO8@>aN!k_uKm9-(dt)P@jp(-ieD&Y%Vc?LA@1H)7^iSVUU=hIBWpDidPT=t| z+>aJuF8>|SSr`U(?+JqpDRhc|9|Kde`fm&1>HntmKPmnHLYIU{kf0}k-3^XdZ#I8s zxLiZ-aJQz0z2Aj}7C3?Vm_kBaXS_X^q%b~s=r{}9)nbk~S1)`L5-UAG3@;FX(d(+}*o{u4m&K5jT-M0Sr-VAQDNM$0q2_D`3~|kxuvKYVPr@EEx}2x=>C0 zxbMn(%9e6s=+!uXl5Ao?cMSyz?dJ!0iTHm^7}T0ABf%n^qGQ|=Lk%V~Iu4iRe8)FT zczFIhx%eMHTxU&k9S#;33Vm*_u3QYmSROa`E&Ac9u_Nmmv{M)aykMb1)#5L{zVL8x zQqt0eJM4UXd`wJCbaVwg@NUg5EncYt9>TpYwjI+`hr4=kqQx;>Z?To&b=dy-ArNWhEyyu9fq zHz#%V_;;yime?a0kYnP0pLZ;InDHNj zr}aO3>`0tE%A*4+k4u!#mw-GlsXz8D!6UvLh-VQ|iXgulNf$ZirUe=FMpABVZPnG) z^$~O1ehz#|_-l2Q5tgyLvT>+hMp6=&4HW#UdUjxFsG3VLZmZQe^I$5#E3bTGscVEo z>>jV3c|@YAUEM2YbkA^gc}h&QoVn%YW`E-57Z5;av9xHp11Zjlk42p` zAJk+pw#pqEd)A8-%n=*L@v95BO;T*VL9A;|OsH008OJa%IO}_epIyk9_G+t|xLfr1 z`n`~@6t&>vc5yaIVZfssS0Ee;am{FTw*m$bBrMd1V+JYO8qE}+>r(@P)&~;Uq@_Yr z_)2QOW6Hf1*4^07Da`g&jYx=XJ;DuJAnIaMsEbgnay)K9FMfbQ5A3&$>!!*;&#uiHt!6;KW zxK=nFs z>ZWi)rYj7;uYbnwsApegsi>%clC>*q;O2?HjcvrJSpj(oxl>9zs!r`-Pl5-yAx~fIJ+Z-5MAO^Q*Wr&U zr;LpDb5!|&U^M{PWkjD6orY}&OHs>LuV44g6iKdaY|Q0?N@ynv+LYwDMPWxujs}AQ zA+y5YKz6AO!ND(b4ZoTe>$Q0H_V&iIt0~!s`L7_;ru8o6YU3U+Tg=&mWBsskgT!lT)M%R4h^6rzK?U zuj0@)22DPsje&3i7M=EHDTl%H0~HZIs<@`PFAJ4pXL&SpU8L3NmELLd+SW!Y z#@mVJds>ED7Qj^bEEHQ+p6$Fx0@fHVahvvDe5%-M^u zQ4wIwWbe~*U0Wi0ca3HvGtuR4VlRfkau;r^#YWNZuT2t-L|tjbe@+c7mFfF}t3p+g zk&qOgAM2|*$*t_pRL2Khi%=Pz5CpPsK)phm3)Fuf-1P4P>Xaia)HiLCCdX?cL8;|~ zd%oH{2tEa?Y)`z5dxYEy8wl#})i)$e&>l_s9|+D3?O>y!fBs)Mc{>TvmZ69-lK*;l zAv-Fe7xs3|4Ff1}Xk|#o{hai&KiPaB$X%%4`4T78XlwGDd&ZamyPb=0JJ1xq4VJ}& zn(<{V1c7Go_~uY~GY6HJ9RnO2mr?3n_3CF_UFe+LLtsb=njOV)Q2#rWw>&VVo6|jg z>;D7_FmTOc04O(e@;myMJG*2D5SaWD(No0#0`+AxAPo3_sY8;Hsiz&zP&rO;`#dqh zW}>mNJd5_DPO8M)zP`D;U{_X?txIKJ-Ob^1?}8pg1308jkh@(19s-GVw9rzsN0@GW zK7ZFG>5vPG^<6tz``|~swACC)XI~6`;%qoBGvLh>0k=M?xX*s#2S^v{(*YxvhY57V z(R!8 zKz}rZ5LP~HeS13`^|9kK*))|x)wZ@aSXkKX?Ci-iEFvNzY;0^SER|%Rn5BHB+!TJd zu8s~GaTLpidI!L17#bSJ!Xp2f6gMn_{h)?EFuqWlBnHy6XLu~1`~q~fwRbl*a{B=r z5E~uMpjBNYITR=ioC`E#b6^!+&G4td;Flym|9JI*hV&;jY@g)VZ{b3UtU@AX4z}?E zZU@)a)`)HgpS6aI)aT{(So9Y#g;i+fR}#bli2$Jir!P;mj;*MKU$a5>z|c1{O2k#0#FIJ-TJFnuhg@p zXuP_DaY3OSr@%5P()h*I<_e<#+XrkIH4v0ilW4yK7X0=dNnTaNtrq!-MYqGw5{@Tn z*F9>mHw}4XT)SX#oYaXbZryW&Ejf8yL;)^{wlNM?Xg^-ds}8=;J>fw?&$z#}|NQyt zy2z=)JGN|psioxwMd;n#one|WDBEd&-oz0DAHT3;r9BV}gOP>b^>ou@JjZ2IP+;+B zfkaSn$rnZRanF~1VBluq34)-|M26(e?B-3?3bE<#RSk=MNJL8)dSZlYVEH{==3IYQ zTWLtpC-F~%PsB>}F9UHu_y+%aAuhKf7kE?0kQ=ZobLF<&3Og>zAby-K61eMtRa^AL z>k&q5_u~A45fonl6oCju5z5M4@hk>eWs@LB$Lc0@dd8;35*=Qh{ps@dJ_^`J_0vWF zh|gPUt^WcR35)}$lfnUK5VB?vQfnh-9T{;|5^+?Gj<#?8CXSFbl8F;zh(pRQ1YVE( zE5du;MPJO2NMEsrxfhegNBz;A$jHdY1T~fmJk-?ULm)Q`of54Yx$h5SV`Hy&^#gRL z;orQPHxDwoDjS4kWYTvVSY@*oulbHG*&0ae(!cD;PI~c2rO{#eMmT=@p#%DZfqY3% zQ#X%fz83FxD$#wj$QAsBQEc5!Y)3j|jKE;&Moe@qvibO(_*Sf2@Gdc3)2J_D9ETe% zj=|s$4q!KowXXUG29mh#a;kgEOr?3G&#JAK7W-_Zrm<*ZqQWz~EII@_K+SIo{XD<;VQoUPMYZY~yr7n77!Z5=L0jbROaBQ@4>oCgf@hACj_3x7PS#~XUzn)R5lY1IOvRcGlJZh+N!f~F2d2UlMICpu8G ztUzB@*QQaiSh5xwGU=3*=8q#feHK2%KB_SvG*8385M8O^kbK_#rm0ht!?}+(cay|& z>Jro`Vz!s|zOrsuM@%dr%2y*1vPJQ(is9ijtuK!IT^;yhcV>0cnB~t&M;8)0F@jp? zR)3++2(;gs-`-_rv5sg|fAA$NL}pvGFUj%XZd4vN6?Ro?Fr~2-F>(3IWigv2>LZn? zkK;O3J$NkBKgfWOYaEll=ErFPb^yzr#Fjy_{c0V5TjRx>fIzcnHMj>Wp9d%(T>(K9 z4BU16*Ei6#m_m#MNlo#Hfd3=cldJDPL|vFwBDXj2Hru2>(5cZM!4Ob!=DuO z1w)NS>jsj$yGib7vVn)C`imZNXdYwObIzvX#bf5NSpxp_fq2N{ z`uxq?|BMd+pXv{G(0(57fq| znUkaodalw0kOthlIzc3C9^YV;+#zB2hb^XEPv*&hl z;OTNUcY%t0cwx3}r+@qS#%tJjZ+}4UK7pbC5g^#UjJw^6;Xr(eWi^wGi2kV8#vcK6 z_4KKn6SPTaoRFl+m2YY3oLiX$O7X1grgLJZz@t_>O9DWf^3A|wbk6DHvG;`q3fQ33B-V5AV4mfcg?fv-dumnwvGUm5i*c;JYV>0#sOd?gMGr@G{TZH>!~Usjwh`&Y z+l!fX4pPwgXC#K3L(U~!dbnAq{U_u;LVL)E5vp#E-38p|V}t6I^a9J+mB--SK}rgy8P_Msi7v^HBjZz@#|<3JlZLyF5G9SKLclqlqy z9ajiUnUT9Pjx>UodQ=fMxDFw%O8h?ie@r5vB-?R-Y&@^VcpkU-(R>QA0T$H~#1*~u zkC6wKF~JNd)7^2l!+$NW4+seK76vS7#-ZBt*UIMU0lRp;CcOjIz`qX%gMl#;78G*+ z_@>8q6j}iPa@HRQc|-z#oXHPdU<|f~beVsd{Iq|I|8JMGN0Ee=K+xKNAaHj& z&b8>=?$q7g^JmLI2yp-5swSd^OiZ@i4w`B{wtXqqs;M@c+8i7FClLN|9!j>GniEkmZq+l}&!5|Fdo*MJtM| zKLFuzfz!y#KGowSD8N=PJ0oZ1=7tgRUsv=qxou^~sg~;AZ5NiNcwTQ^EI2J4Rl5~k z1L0l6Q5XKkKtiSYOoi9|N$TCrgw_Ne>fVCi8ckT}I@(IrgadXwdQ z=p^ad5`e}&gFuU34|i8rS7)PfO@qR| zPwy^Q!y4%IY;Cb$ziz(#^*yJJv1VRTQIYK8^!YR2)$gch z*>6{a*(_V`&MoGuHJ<9ZANjv3%Mi$sNvN>fkYii2$WakRK6~r9t4?`)cURhYEJf+K zH=BJ?Wj4j6QStldbLQQt(o`k`1B240^Vwg&K5=ZOOT;$**;5{1&$oLXquo|BI5yCL z0%0dHapih1b?WVg-~%^3y>xm1u)xi3nST54a#coAI%!2_y=HAGsNg-;V$=%yPe(dK z33-(Ztc{uyeF&(NtiKovS}zM_11aMD^d45KKcP$ska^kc^CN&Ry^10Y26=LKG_#v! zhrYz%SI0Ntr$X8Oxane%{>THOVdQZ-DlRN698`sbjQjy7Koy~{)dJ+8qpYmlgI_#$ zGAyWfVC%5MFCeg3!Nse>2M#3bkN=}LwHU4Nd;cBhtI?$LbQjCc!#YG9qYuc>LVbo5JR6-{5=cb!EQWR5Q72Y1<45u`(PYPy3w#Qx&T>VU}tK8 zV{BO-{+K!7M5wpl$}nB>yeZ!3i>%B|vJE;vSMFkBV1R~GitpIL*eOz_ zb?bOob%Wpnb!6_8D$(R5h-Qy*kO5YL!0uoomlg=0(_DdvlCo1I!cYe85dmrgc33AN z{Lv4`A}}>ye(Gd0d#D9?6ir!UqfKDz2kMpI&3!V)=Ki+Y&)nh;$jZjX28fHQQoE+~ zyo%LLle<1a$COelQcp+8oxA}QV=7$)kzViK{E5mA+WSuP_ynHcqPz|=DSR^`5h}_5 z?9@l}bb3r%s3J~83TbN2@?XrKjEJC`OnsMTs(b%(%V7rsghNX5{3;vhnjt@K42+{t z2%R!Q60y?tu~3Kww7bUocq$bTaYKm(bww@Nis__`>sGQPqrka=u2;3khOyhmSalh| z8q7mhzC34oyOCrERz3n&(YTrM{kKnSNFbTp&EKd=4hGgDR~QcvO6w7Z^I;oecA|sg zH^(%3X6C7PLfiqV&cBeW3Mbvx;RJB`&ZiB=W`aCbaZU%hmWue(2g&w~s0~iCgtpd! zh*B480W^zPPzKs7NGGVP-&?fYGChAVdIV&jWdW1}{smOByq?yejndd}UoU#9^W##ih zdQb1z-dc1R)#S^FIY`f`4IH0^`|VSXsg=FCYU`Q6nyaDRL!q0YrjHbW&#xpy`*2ku z->WzV$_b_0ov*8msb)|q)vB?CWG4U${dx6Du?fhOj}`;6tU3P-&FUDhASB(PF%lq- z|54^YzRlYRfEmN?H2Q4q@ z;9D+Rrv(07(7y<3Kl!K(u!b z_d4fao*V#hCtUw+0W@X?@KckwKR}(01S4PG;0pVAhH$iX?KPEyWLwaTdl61sM+I-Y zTsbTcLO6DKcCle~0^gWC17!abLATjpSxr7v<{PR5Z(#Mebi%0ih1Zj0Y;Q*dc{2L& z_7O`q(-#pPL-Iw1rh6y#RtMY3f%kai<3ox`M0td;rL@+Z%)$rgacAZL->^D|+hBtc5WPM5~!%EqDrJdr)D;MYmk` zLU8-fwXUQPo?h~Mh@_tPuFy5DTlH6<#!XAd5RLIyi`Y7=S{^O>`q#gBcK99XIbXDz zqVCF@U?ddhy?tR%PtQ31YU6`8GI3Q4y*nOJJv&e!6Kw=6tmjR|_vTBN1Ky% zjnHj;uj@mITvV}z;DoQxcPi^s$xk@`ooo0`M4lTr$!1P(i8!3Jr`I|>k0`UTnB5!K z;!%U>UNh-`zmll&LEp!44eLtOaD6Q%w@UEhcl5z)LxHpXu1p8hn6dHQG30#g zP_vzb4JU}l$$OE(-2sLkdYZj)79I{-nJrC@`7!zxAImK-*a2cV6fqJi);BzSG-T`w zvs~7Dm|skEi>%s=VIrwtfRByk-r2$K);w0RJ4rgr5c1M#^=?Q<~=IF%5Jha3wZ77M2^CC z@riL?D@5g#0jkD__9m6$np#g;n~0c@FT;u6MXMoOtN^fu-M;U_;(qYa6y>Vw3MZWD zN#=|ARtLQd`ex5Ja%~?ZP^Z^8{;?!J`ZeJSjjZL`x;SGmM0YQquqtC z5$4^)Co-@Kpolz=)JL}ArQXAUZJi%)HF8{_SiLH-fi&lD^%R_{lgl}LPhAfMK~{m| z23<)TQp~5+#nbLR=3dIFsu)Wk&Fh;=Za1A9i?~-*gBB_gZ3&qKWL5QXpI`$oY-^ix zneRHSEX1fZTb%oh?9M$-vS%qX_d+_}Bdo8-)!zkwe})<@UaMu~*((E6TvbVsd?XLdCqiW&MbLzBDOtw^p!IeCIbu4Kj4KWfthXhPLyj6E#_H<4c zXgXza+`}d8dLg46`sv*%?d3&&;`zp_!oE2#yo?;qT|T~AH*--@|7JRKx#3D4#qfYJ zD~}qbZH;gFf;nbMCdKV%qn^tONAoI(>~eR3-ql|t08oHAUCrgS_`~z}t>DG4fB-Eni8cRbvxyaO6r)`F=HpAzY*xjos(Pky(H z2hiF~Nf$QciY+=j=7xFBS_F9Ykdrux+(1^2$>>ja7Kj~bS>MM-iG!P0cID_fH+oVD ziSY8C61#gO9~V!BNng;)nVj`p<^|510bAopA^HO&;Yo0CGrS&OI^hQe8ARw(+ zfA^Pp+y)xy?u|pL2hEVYBX#Ov? z6Sin_#4yT+VFF;@JH8a9WkI~%1=Cd)eq#{vukP=%4PVMv3g`gI6@ose2}K-mbU7Jb zO!G}-iBYgAdCS>#a-A&_?a=ShLN{7H=7cRJh3%PBmAJVK0g46{ibM|a3?e6Dnt@V+ z$U5boBoMxn;H7HyAU=>Ik%ohf5zuXLf)pZ(eoXWFWL|cadR01!*Bu)CWim#uG23u( z0AfSh?$))naz5$K)e@G#d97ONcO|cj3Y%}Ov(3fCZ+)sKDd}Py-V%!!sB%A>v-IoU zRfo8#Gfm)LIo)UCYm( zxmE4K-<(R18Kp5Sk!x+bYOBaG0dywhfi|tPB?y7Tb6Zu+3(_{nA}ksR_Vp(onzx7z z#g8UI95|NcMzoXm)UG&iiX!hqmAY-Qa-7{^T8Jonn!lVN@vBj-A>EV%7DPFRUKNPY}7?acvp4nq)cV zywz1C0Q<&4frgDWQD@QdRo4cS4x772$xP*RTR5F095_s-kV(bW=S14ZRhfRCXgdom znk#6fo>e-6On0WAD~Oby5k?@+T7t6A7PML-x5jQ~wi`rmUKbCiTra=r-CPo=n_yQ$ z`)cUb>!-|dIevT>)?BBnhp=Ctw{zJhshO_-lqC@~+}L6=d;8)<@{H+ZZ^OCKtg^UQ&ZS%O@MocaV^6{RpLZX9sspl$we2g4@bA8f2yE>GeuBF>ktd z0@eH(`VTS*8V)^<7eW+!Z9mQt3WDM@8H)YJW{ac_#IlpQ2YU{Myod%~(lnA-HzMRD zEv;qGA!>8yH@zNaoC&|{P2yMmz9>KgDxS8w&$UJ>L}yFKYCR^C#@q>H?j%d)99Rj0 za5+-5?Tj5`tLx1clhq)fBxEj?g0+7iD~-SEA!WOfR|&e0%+=U8vI|7138@`qCw49W z;83lRGN=2)41`rGZP#Ep1R3jL%ikc0W zxh3J=_#Lfe9)S;dqFHPb)0`!yfr;+mcZFC7HeC{d!JMm4!eHB&%nP|f!midf$9w+};$IBO~QRi?PLE`obIRlHm%$oO#F@+kn@p ze(E!%ICvs=RbZGm6^!d^QCQ2V!0;RBh z54VM(xtvFhq=HlEu0`;rYHO;aQ*6u3Gbc{MH?eD*u*l5r6_Z4FxnY=h6g)*n^S)~5 zBbH8q&!v(b-Kw^KAhU7ulpibg>1Kt$zsXME%*4XVvRU$)QqK;e0?+?e%~5(+h>E3I zJeURMGhV4TkD;o077?%dOQ>Y^WH@?#TbXx)XA)ZAm-ocSNN~DHA!wV<^(HN^gmBG^ zpIHP>Q}41$C7Y?`)}{1riaIw4!io2+p!%S4<=iK;{<^t%!7atz$QMGdB};qVFEz!Z zFLk2G-{R~d@2&T&$w&+ZSSS;cvB$ND+F7`g#vj&}30MgRBH1 zZB8H7-?@FetnK;e0kb10X;e(eXZpdz^opvRiP$JL`qRFe!_lQ{hZ_*OV1H=k*Z zt*{vuNW%KmQ!G#V{uJrnaU$Z3^c2d)E)7XCBmm}E(~KGUR)RtB8BYc zaFW1QA>Wvyyv`=x;zS!hYpK*=|7q>Ogd;p*Y8ghbFU+@9fL+ zEzf$>Ef#hI(Pt5h!?^sh&UXkhby83DGqc?Ufpb*MK4sJ!#K;X_-?5#mNG&)%Tv!Q) zi1V`NbD*|;1?eR^^8UdDX94(t7Nj%j@YtXtjblZq%MD(jq7ZZhhncf zeY|mEv~>S!rcy@jxgK-R$Wy}Es@0eV#4U!WjD>1Bjs>jvQ=IBj(=|Z1mI|GbY0$CqUIvpDl=D-k>ypl7m*{G?r z*oLb?E2mBIH+v+X_bYNS^%o$N<7@zt{wM1{f$NjRR|g_!L$+Pw8ERRG%5_-d60G?( zXccCe`9b%pJ=bA9m5c@aJg~rC+QE3R6ThN(Q{PVgnxc@wVb}3OdZCYLVwCGSpcgRWzIYW`zqfsO6!s zgXC}<ta9KC3 zF~C?eBE`$YpQJV_InryL9TkHuEy3Uy8%Lsby;Ed)5_?@=_NZ$u^t)L+ zm27~hcn3CbnBfL|ZkmiPOi=mFZ8O!WLh<2$Nca+uQVqIgi*$eL~2N%m7Z6~uGbJ#MY2FYj* z2-!b67BKB@&zQ?@kZdAZO7qi?PXg+&#n09-eLcbcUF^ciT(#K4G%?84@dtVw+954m zi9EcSbBQR2mx6joJhFHvIZ&K?y>W2rXJa~3P?CpycSu>~p|~dKUdX|(SVS36;*eeJ z5F4b!UzkytF!?>u4}9OV`10Xm>V)6l%+1(`41wJX9-12ob}_4XrI@iKzcb< zNYGM#e;~fm+800o?*Y(6C}PWAWsiyA=`~MXCSKu1KoTvq8?wXkWBA+!4z>M5v2;@G%Pprt zMvJUPe8WD6McRLg07od0X_Pb3&k6;4`YgoD*(JZc0J3z40Yc;g*6cED$AgJSpS1je zNp%JKzv-P}Ad&l*+yi3F$2^x09RSetzW6>zV*dbVz?=-ixeimSLQoy7Ynf$z z1{h{M%w-cLOl3Z6(|7rjb3m!`>7KzM+FPmR1Us}5ag(>%IqvMew1Ln#3xeQ0>nzp{ z-_4e~{y_>!{G>K);7?xnG2@ix1LTqLk=PfY0q*1AKq(w5kaJtNAx`+$Z3ZEDpj3nF zFJbvN5oJsSEmc?*V0|o~{X6xiWB`XnGGhNP%4viT7$dRU9~9xlaA(x4=g~H7=3AD z1&~HAR4Z-1JD(#bBdd4VQT0l0IfH2Fd0wpntc~qYWNP)M=$M!$fIg};8Wyl#Ze4Ek z^}1+Ssugd@UhwYp zX1}4d8IfVC6-a~619HIz2ql1V0y}XT4JCcZ6yv|YTy@>d-)uY?asyZxS$H5r{aQi< z26k`>AioKGu1+@eyzb75HLJ$NJAslwHa0e+!5@==UdXCNHzF)UNQ3aHHyCk^K*qTC91+9q4+8$?;BqhJi54+-6&exVJ<;t-R4GqO61d`>7 z)tavW>dJVF>b+I-9`2aSY!*s(^+b>lUiB{&%GUzr^V9^;c~?#87RskRXQ~xS4?5md zhtg@C08yEqL93J#U-C#~6d*i(+P4SxymM89Elzuxpdm-%vD?|v5iduMDyrH~0jTN{ zMH+E!4p1EqT;cG#+tJh4*T>7cHxax&Y}YhfVgJb3>U`WwjU|;%oNw85rt0Be{bF7~ zeIBH#60JZneY!Qii zz6v!`fDgXJgMA`lEWzQjOj9DVv@L5@-dwozf&7s-oa`PK6RGHSh)<>?(v(l!4su6f zST(T+ES^})lNT^>i|Cj3=P9B)mG8dXihO}TB_jBp8vFZF9~{8^gru*6YE0?FsB5o! z+m~bHUK5LYznIKBmguW)!TTe5uj{B!`641Bh5gE&vbxW{zCc1M)A|tyt-=l3J%^G= z#1slAh@r7U@vDONo8h#zRqsT&&yK(!D$fx^6CkdJY^-w@5vUc05+Tw+K}Xk``~lkG z51~}K(G$q$L2b|FIsXIzaolvyH!UcVn=J<6Qt68_y5;4CVRHMt!eQjHdk4&VQ1t~k zs}fR1AM_#BLNmi*g&_G0gq#mnmq8;kB2MHQ{L@li)7C*$Q$TwGiLVzLH~g5d^L;Ih?( z*qD)cLO$W)?!c=GO4B%klh&FX^sCe-Utx>RJ$GA7mIJi#H~uex)D$@ZEKa4>OZ7cYwC zs^jSP&f9)*j^}6WWP9?DW4t6JH{>4&Gd49BnvJGEo<5Tmgs(o9D_$@%EhcE{wM;vA zxLXb9Shv?5(Q`vP1VnhXi2cpI`NL=xV%7}jt0f!FG4<=~yrX?}iF>e*4gyy{&*;oa^lwc^3u&+F+u`8G^|dh3y=b$>lyblFS~1n>C2 zlpbqwN>0n-XFr7*b@783I}oIe&5q>)4vR;sciwcI!QtkkO`=H^i0V{m9Mw=Nk=L#9 z@aJII%ErU+e;t>QmEHA54KyA1X;dkqZuLD%{TkJqMU^#0+V2?XgThP zPmt`o05P|!zTK_9IQMrg*OR3b6@#JR_6baRK7Gr%B6-%64$PW_a~~5m6bCw33sAGt zm0~5v(QNn`P$JsA5`b^nu*Z4T)8KSaa-sxA`O3imY6hPgYJsSH5Q*ItrPD^RoFe~l((Q~YYbksNb>f^eLBzy45(a~9{yxHDo zWHuQ>HbWiVKC`pSN6T1q#>a4}QoG9idIvSgcf_TJfg5 z&YOFMJaKK3xzXO{i&#}H{LnS{&5Vs0Sz{+ZLj}!s*?1v6g$0oc8W6jISHL+{3+?K5%o8 z`<8bjV9scYk1`iSj*I>wTl~aGc1^39Y+M%v+(Td+hcwK|6oRRc>7_a*r$r(PSpW*w zyA*xoM40&|kAczxWafguX~W|UxySu%ky0QPz!G_POzXKfg>VqPbf~`g=eq!||Dmc|8GQ_%8`n(s z0F}Jk;jNJM6HXWuL=d2*HB(>%P{31^ap)~*4_Ydan$b==`a^g4-x9h@fl zTOgKxkd^163qI)r6LQgzzVS|`G#f$%zz?*&&?*!czjSi7SV1^`Cfx$TYtv|$qw>?? zuWK($5uZA&vSyWmURMR!dQVOs$$#K(FAxd5uq0DD}1hClQ_>7zVvkM@{;?-PXRHF<*a{Vnz`ln?Y zGW4cwTVhwZM`UYt(-n`n^?rfx>Qn0Gx7l8{g$k(zRKAbB`nXb;Sr7Xt(v zACAUdDoM)Rl|q3sjTP@O|GdGjFatWhPmrcy1BY48sTON&uhAy$1WVJn^0nCT zJu|SzdS)Q4`EdBFAnxIf_uUPjeg+WR9Hzj8VIL%X$g~8at(^e!VlWyUA}W^GFLmH1tI`PHz~ierCmA zbeDg$(u?x6K7jDd%I(us$2|x8xo~A(c&+eW7sY$k4c z+u}Ro+6=_)Ix2}Z;xXzZhO-iT*@BOc#Y0pG&W#g(7@TMC(|i!@&DDQCfwyY1y-bC> z^$oS7iZnLe&Jb2QU2=i9#GBkw6Nc3#{x;8OvfSxwbpUZ4v3Z#@RQP-}!NbdoZ%~)# z_mH66rIo#L;i^d%#DSOcwAp-`|86ctdFQoz*{`5M?u(X+^&zifsUQ@WyG3EH)bTo5 zK3?;QQ!z=d4XoH+&4ZlC5))bAP9Q9jGezraCppoxoj4&8KkKCFt>E6K(q|~orj!?T zZtXQ(dXENZSt|CB3FVcZuVJm-dxkQ@w9m=AJ9b&DVTGca=#ZF_LV1zO zdQQB<7Uv1R6`fMF-38Rt8ldB5b;(=Gb{j`&FZ344-{$!=Th9iLW{t;s^(6TV>NFual&a4$ z^4om${V82a7@d2Np9k3!CK8;$hL&Tw7Sk=00H0m(k3j@m1y1^Yt2DW?l{z z$|HBn=0mm0mYJwB>xve32cyA7sIlpp`v zz}gs^IlcmOSP3|GAyOKOhRHPi+}){2ce;?qJ&QZ=WR|WvZ&izX6|>GOd;5K|R99eE zmt&6KQ1|LexRbsix!+2hZ=~`p{4|MiPwPx#c=5P~*a|Hv^r)#>^RDZj zPh6t#Ym96T->O6}dm0^$(>o1UX`02q2a0cU?atjab$I%~LubMO8rIvaa4u@ZH%%--N z5J#}ktd^(`k?@?{sCkAIq2)9LVa_fa-gip=@FD&6ODvH86YKN#`V(W3W zK2RbO63&~amiIf~FjZ|WY8jjXjt7siZqb;%;H9(javu_e=wI1uAmw9aNmg>v^r}+4Uh7&+xiF^7Lib5x);VB$J>w+iu zIRG*HTS0fruI6-{c0O<8X{+30Asdil%#lc2Ozz8*H$Mn})Vxw}ZZseE>x zzygu{z%0c~SCZy*T$VTT_FE@5Xn~Tt+^_2A8&@h+l?4`^p@`!s!At+t{aGjHsjn}i zPNna{@RuHJ+HY(tq-2;EuME~#bdoWyd)}^&gc^b73#{NUB5YQ^C7v^?mH9O;_eUA& z@E?OX+DFo&aJe6N=5?z7&8G<3C0Q00i&8{ek?grBW^hjCJu;MGIR_lDi#C#A^! z^2FDlAH*E_bV4^aK#Qcgwhv9zrs?`ioaERCNsHX+2d$3OrStr`@`>nUo8KL8jZxvY zlKJKxem$@D;!RtSwDoLh+AQ7PjJHduJ2K#2A0!=Ix|bE(f5{!ZNB&BZJD%$^)6LYb z8~X%(;L;SAVb+VR$TNU4EV?9S%lVA;d0TAWX}b_(@LR;5;wPRCe`>}V?z=&M`98%0 zSD}We21=B#NTHCm&>|F!k#R_~$w9X6cwsfc4<2%lz4LiCF{&2N;pOQm@oQ$tS+ynl-Rokj!hS7N)4`og!MKW#=`X;e459^c> zQ-telOE>E>I=YR8POmsA)r>UO9~z-!4!GZEyX;j%`API4vBNTg^<+HC;LlSsY!YfE z^A*Redng$)!_1PpO3QS|@8L8fAfD--u=hR~5XIqw2J3F85;#rFh>#4aeg3nS3Kn0L z-)_3iJGpk(ddx8l#E0J1)P=@JL&UrA*1ShO;W!P0EE~rQ*G`N#sWqP24@J^!qXo9~ zLf@Qq==MeRxMg4oiATlSRX+wyR0dJpi8>hHC}q6+l${nLy8A_=YNuukP}j9y#PinSx@7)VF`4`_x4*tptPJud9|-`70BAP; zfbgThdlBr_d3>r-ucFv>A#>>@WiKu|vS@#-w@My~Ro1{}W?=l$4_6@F(2lio2b7EN z?pjLExv$04;^28iiq%`}wsGl2o9-Vb?Ko5xEPGchz4fjsE{=HNR0JLUToh#6p;TcJ7cDd>m^w(z((*?J{(cQA9#%KSrLh_K z*1sNam!EeF(YJ!o-bp`a0*N1oDYDYjHFw_D2|dSE%i{f9a}_xe>!u}ekG#M64h%Q^g3|S3()#PbnuV z76)ZS4d{y_c6&6fT>TxkRwzSC6dBK=xHLV^OjePk%FS$aKx>8)ie?`cYIYVQh^H2R zZ0+-VEf4l-Zy$c;)!v2l?{%C65$Iu%iF}e}L$xcPEyoD*{DvYuF|ix8l=SW*OITAf zA1l^l_U$jOQ&z2goF(x@=ik^HeX@&9wo?{s-D}+Y-$}fh`&N`{bZ4Uth#-uwUlG0yGbrUyN^px>^i>X~y^&5ChnF_=;7db7aI+Qu9; z-9tMBT}#?4Ct$?oHCRmdE;IDhE45dHeMUVgFVQ-M$c5enp60+RXJtvnMZ;r^4CZr2 zo>LR&{BMq=VTvrZ_r%0x4^WR( zEhbNVh^f#QtEH*xQ#ys|vV*iq`oC_jQ^e`903)minzt{ zfCm=J$Ly}->PfTEg6Ug$VsNpXagf%CkIlCqmlpqohqPEl8g|85hkZsX$Q4%c?oXTc zJXI*>%A=DHca;e@U&+6XH8GJ&oqQ!)oA2ZDmU{@nnp$^Lp&Q6vY^dG@po|Ii3R|mB zH=xX0B_F7*4*f4`fhK={{*Lo*!cSfX`%E|4hqp?o6?sVwiGi5#)Tg_rIi#3qSX6Y? z&Xx!9j6_WrIa(myDsO3vPhFb#RbY7a=wjxUsvsi~0@HwqCiGY7h>O2^XUUBA?^|n{ z2o;P07RqHce)q|5LbN)kQ{Plu-ZNF8-fHyAm@O<_^PFqu8^0#iawwf@-Y>U{)q7>t zKgZ76g$KTQ?oz?(svO=#;ZwQ`Kgq7ok3FlRC4WmB*C_rr+6>L4=y5t^L2FYU*j&bb zCs}2?PN&}1K15d)YE|iuvvCS$g=k<+Lay3M((t4#+>Fz@cQJajD=NZ*LYG~8Cdz%`7Iu%*Kck= z<|>RVSxPp+-5fHzDU~R@nu*Gc=n%_kM<{_aenRuTyVuN8Bj5nKo0h|}+Q8{28R{0& zR}9Ki3+X?jsw1REq$)4LUr>Kq29*x>n{7vD{I-Qx_`{?^v(VDqm~R{aCaAq%4vE&D zt8)`15I^_kV*jYgsX5SW8iEATl!7zLkbb`WS@RRsQ<>qZeF=xTrNX9L;^l$!&T`?% zO!n|F%@l_}N(gBXI4&`3hqJa^IT<+Fi~hKaY)n}gDeFfTZ6^XiA(a-!0@|mLZi|ZK z73iHEoXDTYe#8SOOLL&K#N^raW^JT&dqrz)m;+7Gcj+nA+FTfgiSWtAOptQJNxus5 zRe^??M>nVGBrWDCjd&wx)Y$S_k51ou(RAVnj${U-lbMZrvb|+`r`5=`+4eBH8~H<( zgPoPwx?|IkI=lp}CTRYWR%JoLv`8ktK|)(rc^2n7+<`h^M>fO5emKl)2e?_kkeGe? zZ`%J9!EXp_YJtT2>tW-U56<99dA`tg_g6%8{3o-S;u$t?J(peJ#)T<529vSMm%G=m z_A!j{pNf$Na7_4xBpdt1RPSClY5$o#{g-|M<{F@9pNRyM@rM5W)BhA$VA%7+u&0|X zCimajl>hKE%-q17Tn|&R*#B-WKrJGZ$_axx04~Y(x^-^f98rpNvV|5}g7-=p@HnS< z-MbfA?0liHZ6eI4ib5!5K;q$+3$@C0xIqV}B{lyy9^xdE6GlTcOM%VF>2aIoR>hTr zi-<)5QuNR&>xkXZdMI#KU`h{%lbnVLGP7H2*jd?orE;9lX=$qc`&=Sp;uwCOPyCOI zkmH>A7vI92Q^ppl(AgwmDE2*#15Ks249)V5Twhv{ zvp|c{%}gcbMDx_|MqIV>d@2W6T(O*gjlw=Cseizt1V29Elb)kNF%#K+CndA^z6-`V zt@kqOwuz7_Ni32^`O!mdDwI8zYuJOk@Bex8* z8cn4$KZ+t>2cAX?@o9W&=cAY<`Yv)x2;Px!FH^K@&I1vOodE1&yitD3Q!EcDRQ#W> z5BN9u!995bg}SbeN869A6A^yTm^pJkjSulFJ2vO=D6%Gstu9^Tm<7Qev|DEe?mH6Y zk7Fmh&kHQwdpeKL@U2r$6;tXPpO>@#u3GOl zxX-$ITSowatEFqpU!rL2G*>=y{|_ZS!FLKu7G9#IQaJ5kcF4oM3H~?N=gl=|4cxh- z8e3A?nuEFn$&hoig+mACz3>FgJe@z*-3sAKD<56=9*!G@LMd+we?T0mm; zTogGgKL^0=POaDFcB>g`&R^F1h|Mj!EAjvrGTjLz^`|a~7^(yVxi$uq`z~5W5*4V! zv5mXa*}`}6Q1KRk5h|_K2AB{=2NzzpYs5RlJkZs>SKYQzgC{^E6!tqcET7NpffsgZ zyawY;w>dOctTXul_jFFu8|kpal9we--U-SpSC@EqTG z8`xahES#7m3R+(jTXmTmk6V7Vr^yK0CP3=>P#o-D>EBZpdrWV3y&MIML8+9YYn#(H z#k>|9SAxFytUo2Pf4^=b#yePY4;?p{^5Y66z zm%&9x{#sd06po$Rdef5O|6147#XTXI^hkOZ<6X}t_%X)zpEEnl_Gtrc(&WC)+V*I1 zd%An~5Y$F;hreW3r(HlMTJ*)rC!ENj2~TT=$4qZOuydzP`{J=ol@`y_LtW(Y{&-`i zj!E%^l3U1nuaB!3pLQ5;*1E#6MVtTl%PD~AoeObLlH}xm758+E5j_e(VUUXi_x1+% zQ<<2|Lo(?uYrwb_jY(c>2eddIWCoEwIy8AMv6%5s{3S&H(ypwa^UGa)xmkqE`yi2< znUK-w&)s>!VKwhoE&w|-eVg|uSt%BDtb%_SwW_=WO%Bd06;~eQh}o;p!_##Jr7r;C zfvRSSA31%MTs@(dNB7UCw@>Bw<7|lJs-2z2;1GdjNnrA#E*zS1yTGYmV2l6lt5>>z z3bkEa-j3{vj< zXypr6@gmG(<*y`Ph%k&ofs3DUIjZYCtF$9kBB9J%5SVyHq&tg2$t45G@Yp zrOf&K5&&5o6>sr)dG1}ksYFsSs*eUG>N_Ryqig125c+B?|zQCbC21p|mk-VkA8HeElNuh;UwWkiJG9is9uB$ZRnnk z%Xs)k9-kI5@o%K?(o`MAWp1t^R~^Sn#AZ8#i)+A3MN9W-o>e>OkE^c2?ICn5hD~nU z`WxJRb#qpX6Y1pY-d+1f9Gbi-1}>z@-&x2;bFV6s3m!v8vXgyvKrViwg`S7<9q6d| z-gb998c-ay1BLPK+C>MbFs7jS(I|YMy6LUhPR~XB+cRc#rz44mR@Npwwfu&Z>HGH- z1`%tI)HMJ(|C$1j^GO69&xC3``vl{V~R__=N$@%(MjJ2gpn_PLF{3t%msi=96 ze5n>0&dipj%Z>!#$;JbXRthKluI-Cvv)p4lg?kmN8c!v@D)m)Lds!J)3KLjOe~U;% z$IQg7MS~gv)^GYD6xHEX*ViY zRQkwj$~HsjPblpe?G_5Ks#?pN<$eKs0h1VsNQ6d!W<5pwzCCzf?JqxWV`QEl@vAOR z!cZ*K`KM_G&NMNnlsWo}NV0v>2r}(q_07{L>5)84l*;*>ay2aZvGGJ>mf@)@GOpf(^&%)ELNc)emFuo zk;ljJjD`}KfFTM;g!l&05^X;l_QT6?SfOERVi%9&65W>>y75*I&wDsgM|No)*+(z@ z=e;s~R;fgK<m&{^rwy47P}3o;=Ctjhe8pp!XmSA6IMm2==$e=un(aU7=- zQBzHL^5Dist*z}+RW>TPK^jFYXA`~TC+wuz`DHOQGpp=%=`~A@qmI_2x!n=XYK@`< z)x2*{3LD=l*2jcJ3UgFjTV=FM&)C1t8#L-Qn8_?*WzEIu&Lu`oo+%{*bDzjPUHb)T zy1UVhyCFGtZ{^_cPt|Brxgf~nRG@wO?d(AY23I}r9#{SN#<~Mx&!|4^jDWBF5UufQ zz==!u@z-8>PuA@5mC+#xVUJ6N$UJVt|8#>gw=ggqrnJi4=`jQrv7Uv~%x`mAn<-C7 z)5&yh=4eXdAU#+oPk(ooUS7KxHqklnh$N@-4pC)&IQr5nP{v{5mt2rm`Rr)ZD3IO} zY#@5n2oF_gzWy(}^XF{AJe2s(b5Mjg$M!tTGx(^6)&!oswevWCqM;yD3s4vUhX^)J zhepY8?wKU}s*&N+%q}Ko^PM)YxqdTuW@Ac2sDUh#k|{IXptvGs?*|W^Vzl$<YTO5$E772$*g=(?hPt?-X-ygr86vVg-Iq%UTPndiP5Q!FWEpdc)dsSHB zoRdo)5*9sBOW$t^FXH<4a0I%1Y8JJ$xZ9we`5V~(q<#2@y+wUrhd@s$Om@dMZ|=8(URA% z=-tCpBd1yk=MpS6j7Y2P0*p0HrF4kPxP_Y2Ir?TO^*;u~Z#b6Yd#75hFo2V&u*Z(~ z>mf?QwEe#lH^Jl}_zQKP;u=4e3qyDwuH!|pa|eBZDOzK0U(ozII`SU|jS^KYl@e%; zL%tbNuCd2wDD!lEsxy@~2VqDti);`omouaBroP|K= zIXSJcG7A!O%I^f?T(s%NX`!prq=}MW6L%K*dilg72u`CG%ki0XkGQ$nH7sb+2J$M! zM{gtUG@^CDdioTJG>=s5D2&hw_qIjQN?Wxy6-s0RE5B($?R*rbF;OKfRHTaCl%oan z`G)Wn^@Addiw5?tOl9_O=BFIH#FBfJgmLU-pJ?K%*LW49o>zJI+UZMRKH>X#UW%6v zN#7WyYw+oKx+%bOux{M)r=!A3AgtJlhiTNQJ-YREohX}Rby;cbq2kr3%4^7jH*lG2 z3eHd{Ett2E!7r;ql~G9Ylp`01HCv78tR1PsG(DQPH`@<<4buZ)AsSkyfk=(o@|xum zR{G5E3UBR?%b#|-F)^hRqgkH4|3KgW z_eL^)MI<>++e3?0`z-VDlS9;;uOz(OEwzwxJ>G33H`r;;;dfr;xn5rBh5gQR&dY#` z`UL^H^1RdZzx@k95@!Y)i8lJj1t%Fg?|ZY7t5$hjorIOPwR8Y}sOLG+uyxVuCSfW-}FuU=_O0(CTscm&nG4k2f{pj zd_EIUC>lGDb~{jd|OKMw|RU7O( zr`@7B-zJ@6=XT3GfdR#UrNkQW4g}Tg+RGkXb?m=#?_-l~PK_-NKEcSCU&}{>xH>^? zM?zpdj&Fr5kw@;;v>OfR|C)3B8&Uj1Urn*VTCFU}DOvy%49zkN@uPzyOoHnU&3vMt8i2Lm-jw)xypUfPPRwQ5s+(*_7f94zaRk> zdGDLCPv;TZA@>r{ZqTAPX@#qKJL|Xje+sO+m`-|B2HPfC&=2m5IZ~0rM<6D~@m(b) zq{xR>Z%)Q^E4)u#i1F0wEc)9b-f0aX-gxN`-yKEoQ8si9;d69{2O}V*A2+6bsMi@@ zf#^bz!7g@aCk@?Df2lW6q6US?_zc`AP~`>;<3HauP1l*+JZ3`T1IQ}AiS10YDE2F^ zhP84nd7j|5HTxlq^u8|WoJT7o9}ms>>n1+I7OmPuTR|g6 zo1?4=;0|5B1Df!1%;uZV-T3GB@Vw)lcsvEd0%Spgz;(%k4ywhPoV*-^`vw=U)2FbG z4wYWoa0p?M0-(d3jxd3QBmwDHZ0yx5>*o>sb7}FQ1ki+6($W&p8vep|@^*ic%7nCi zo*Xm3hAS04M1^68HGldj*?E2clfmKk#+NRpkaT?cjVk4wuaSx-2^{?6r8yK@2?^{JA3`)lafew!-jBxS=mPxD=Lnbg8Tw}FnaKlv zEBq&?u@JeVgV0X!Z)|X<{E@v^9ZWcNCJD_g%{?(N^pk!quCg|k?+0fE@YJ8Zbz_<~ zcNs$1Q0yluE1rLo^(ZpMOl)urxyP+)^E|e&bM(xfA6(T{n?&`LUfK;9rUeU!Jc@C0 zwbhrdIdm_}9(EY1p%0$j2?tcBrKd+gb%1P?+cb;a0}#m0*KKWoL}+OHdzz;+qPptm zOfP^qf1)3KiR-A_)n3b|m6hRZr#5!~IcA*VB{^Wimv&#$kVt#o$xwD3k?ZXw|JB2B z1@yVAYzr-N1v{JP&uTtr*TvDSrvCfA<1^p0`}&WI>d*F~Dx9+|cD8K@PR~rO99-T_ z&VlF|>`7R2{(hlPG?yclCKNW&n(K@ezE@G&X(R!6j?%*HD)`9LfU=K$0(e_sLu zXlSrLksJdgQw++_%tBVHTWzf!;Y=9& zm%7-U)EG&bVaNT{?r%ymd!Y!j4dpXXjZl8UJc_chke|LD3!<-1-5N29FCv*LT3t`d zEcT;vlA&%FR_V~hZi1JP&wh(}81@1!2i^<1DxX+_!WuTy(YN?fd+TX-imV_1xQ8=V zgUgviGZXV9X8sXx6V^#f%zWHh6^kyyzp}l3I{#grJ0-xo6-IuyQs2=r62ac5Ke0jW z)gYo~;(B!C<>0M$NoHVs#=6CW4benM-=qJ!_kb3PtZ>!!DBFv>@tGRwY`&pw1<90% zt;*~gAEf&X6NfVI_4Pmz4(xSi*u@p*Q0x3_AWHN=C*M6`2?s3hT~@ZTaAbi@t@D1~M38#7A?eZ2mnh_{rV0xrk?sJkixL+Qjo)So`Tnr$1A6 zk?I!;!E=3G5D2ZstiI^h_8D8@7~)-Cv=r}zLpM=vpQCY7xOQ@4n|Gk;H)%NSMiL+2K*%7JByNi(M zh3ci3v$JioMMf$SukJv zh2&q4q{s5seE8PyXEfe!%~fM0qWKX0So4P@`zXtK6u$i@f&H7uHoFPr7@!?Bvgnq& z(in)NPQ}f^;#V%Zs`y-!kQFwXrkrMlbYLbr3YG>NpuX{633?Le{CbW#9kp;DuM(*o zA2BT2$+t?`Pa|zqGhwtwo;O-rHw)e7Gxi~&*~&0Z3LrKMX2Pi5^(a=Pf9d2X4lNI6 zMlrEhkC0cYTHa^(4p1aX3zdw1w0?Y_x(M^m7UVDGb{W}4!e?SNWg@a@)U(Dn>^)y+ zFTgdyO}k&aTCh33dHG$$)`znMs};&Xad94HSUGI5FOsyfVBy2}XZ2wr)ec&szRvB( zZz@^T=4c6WbCL1}@}>DR%VCj*;i48C6?bsJL2o8cBx0@2#b%|k()Z6dg;CMZHhp^= zm17hfsNmp!5|R9b=NMkkcTNrt=`0p1zz^Y6Y?}n3Um3FjS^$3!qrj>bgza1>2MJ(p z{wn-8%h)WHJkMX_S7sD5azNVp7auR5aiPy?tG~6PuH$rN1WSqb&P2{MNxR1I zE%?Cr8NcNr%*t@v*`CK$KC*Nb-FgKoSA=JZxI!{YMbyWUyaZs)TM;t**W)>cj3M5? z`3mOeG=dw-u^)wQoc~yUm?)~gG)Gx~9Y1+Tnv9J5`I%&$bFs-eKK1}{kGGCPzoo~0 zkt6e$-&wBCIf^IPERMMe$V{h=vqQdg*p?S6X3(AxrHAEu|i-16w#bK0=Yph zQGR~kwzjs22uc03uh1?>vtiG3dt>PfRoa^JIY?ZRq5MFa>D}p${Lw7XYLu6>aFUJv z7iGR_s%9O&TK`3EQN-izQ0E;sqi&|@Eg`}Eq8eV8rFsrxZ{SN~MTXR`B96eYhdqtA zxCbvnpV7scmhe<3wu;k$QHj~G)c18;&W_CJOoB|DvifsMvE5&S)~mxVRaVdZJF$1; z!xV?Tf7U|Si1@c7^3B3V#R73wT&&6jBXOK7a6?CP6JC9m$K17=Js6W_5iJE;Eq=QG zAkY-g-*m<$hTQyzjN~)O>L$H$Y>C0I56q3t_r$y72F?CD?&h;Dm)Y|j?pmB3Wnv*L z?tD5dA?BD6m;X`TT6}UJ2CbXJN@BfVgM3Y;4! zpH%dtJQihINvgpUT7O;E7t&dT6i&rgr%SN+e-Ip3i~c#hXTvfTIMi`50pDDxzf#~I ze7M|}-BMBzrq<-s8Is;K*ESUSTeWyiA_e>=EzEtdVW3oMwqc&nc_b0o0c!8s z5R6F1^oqyA%^y?iIiqyFL_ zQcjL}b0a=R`yJ`)^9m&w#aVsT#T+)!zv(AyI+Yx2YLG1;uWl=Wo zFdHmDdF?nnDCo3>m1F)zPUei#t!j8twS+7>(Dm+Oj2W2A>)M2UTfBfVGBMfTN0~N3 zwk^On%%2|_`TJIs2;0OU zy{Fn8Hd{QIbXo?d6Rd7SjfsaRrHeEW&RPMa9!2~)E5Y}(;2OGU7JAlI!&4_J+5QR^ z$;HMtkJ=(HnJ*_3NA!^C?uuQdtEe^VpjZN=>6JLC<_6fFUzIOe zijgJ00!xd=&cZ?=#56g0E7Ob<1qhAHjzt+~(5+15Q9h}RDe)G6Ii}L*Qka}X8iy^v z2ThCpUJ`n&R~pX}nOXHt93E2`Umxz%q=`^a{~=;gg^I5X49tDS!TBPxC;u_em)>mR z<$>nmk z(sT|?gVb;+2li|W2t(=Lmj%Bo{_rW4R{fUCg^UKI*=-`ZD{M(eM@Lsz_9omj@<5Zt z)#RFEV7C;oC}FkMPz*2^he4#ODx9toB$CtTWs8-M{tiZENDH%0fjOYI+~bImES%WW zIO-C~*O^@Jh%k|KE%BXwqzEZmVf+;JdKl?bwyE+yW?@F6F1qqLi}L=DU91v+YdIE3 ztV~Wf43jlMS?n64W{q_@D~t-Q*8~@kfm)v;tbImOQlrnicUn)dRl_n8&;);0b&7@{Tu<#V(|BPp|3X#gwG zegPxxs-3OWOs3Nym(8kBsnh^QJuMb^O=_7$|7b^`dp|GS!ZGn5?g9@(pL_B*v-jJ%1x)+OhTcCZOBmYdGO1C7Jw%vmseQZfrG z67>DbDX~)sSGR!sl<71fO+HT?_?#8c2`~Ks_M-QB^E_jEeY%B%g$)l6=d|7QZGfXt zH3FxMgleVKCBbYXuoXZyW%z-cUfsC*Gj?xO=x9*M$uj!NEPuO8Q6~y5Ya!=&E)MfZ zY1^#)g_8=J=u;=9YD*TmS#Xb^=>&s34ksU^3@&!9Pzb_S1_H2!1laHc%+=ngQ%$Uc zOb|~}p8}u|iwZEzevS_LvzAHh*N7usJ|VlOFT3%9VkC=Qrig$`T@Zgx3EQ_8@|8>OtcA#nigr!`kA zACVb=BEY zlHC>g-34V6O^NiRA^FkKILQUM0TB)5K3~@qN{H`6@UK%3rNVB)S>RAJRJt2 zabZ*%j|h{H@1*{@Nutq&X2W4Njm?ra9X`NOc#@k4qFWKujw1+v7bV@NdF&;TY|=|? z{llyY3zQo3spY$fj>;Am8X(JC+3N8X+hb006LL8%ja#J*LN)j+P2FM|} z*|TD$wf(5?`;lsMX30lc6-2sBkO>z7wXzvG_M_k7$d!*}<~URuG^=DPx`2vH1g5v$N$*Wj z9w}~$V1xqXm^!hKFFJj}{>x7{)H9Ju(>(?U*O!OOx+V?x5_;%RsMc<+GJDg5U79n3 zzV+@kpJbtx-hyoc{-^y%yztS0=M6{GwNe30x>p){RgWyoERaX*9 zcE;{0J)f_{PAnKTl$}Rl6uM5d1P1F_f7z=&A|ZikpH)%7xroi0z6|-3-wucFH`wUK zZ++4s42^S4H5$6dTdE_r%Ul|Ezf*JH2Ee6y=@hr5A6kogjIsLYja7(B>g_A`2GW6c zgwvpj)q7t>!EY?8fb^&v+sekX-`zAYRBFZazqu0n3@oShG}Faf{0a5;PJ?-l;lnMh zKLte59~c(D_8K<~bEf}#v^3h17@hLtjPI|~>2R<21*g+C|2a6ND|zd^JDkW}=qem1 zMajaF*m0OQ5B4{ZED&phDP)wLggC@&Kcj&*7mQe}pHqLZGi)afD7XidJ2jRAdlHod za%lgSS)u-N_9ik!M8x_^bx^PtKdYSm=h}7UW|rDS{Wtvdv)zk4A_DNIyzcXo>x(0J zf3F_(!^x>1P)=*dy5o5?^C;LmNpJO6xE;e`W9mup@0NdN7{nS)ALY7Qy{?jk?ytYp zd_BmLS2fQBc#7$Wcni1sTT5Fsqj|DG{%Q{T_g_V9P1Q3Y4V{&c;w{reHSgMe)kEf`#jRCjc`$#H{@_{9g$G!Dw_+AT(*)+-N4xt#7&QIui-oidx0T$sW$Av|d1afLg zDrhx&`sPVHjhGZvf!I`f*=-yYU33S%Coo9k?6iZ{I+W!_(uvtl~4GpiaT{0Sf;U9)8Rbb+;IOPqhWFrEc*#{3+f#Qd7YX+h_ALUJCM<*N)3_v|1*Dv4sY= z9OYB%K!yv{0j}rQt0~U;Jh9ot&sZA=$Lc&>eO zb*A?lifr$(<@Tp-Ae}*rmDLTBf>$`j;QhV5cK0Q^TCv6{oF?XDJy6zT!L6=ZkW;Fe zbO|Yro3aKN6#?)3{m%F;ZED@4=3qTVhUeAAk=J!vps+_<_W10G#XpT(k~(cZ0*=BP z0F_~4jcqe*@)I-q_q_fZS?lvr(3rYJN3bfrJQ1fsASguPjx>v1uFX9m0)&0+oo-e5 z!DIcG&hqCc73Jrigsr}gI>mQ#xD4dA=58h#<@Upv)J^u z>PhC|_FqAEMv^&wm-#Mw7gbqrTA9%or+)oaf)eFe*@vQLBi9@c7K~;3k_P3)PEP{2 zg{lI0&mDxzs7VvTVPVr*IMd}yUP_iUMLrv5RAAxPwWu$!MkCzdDF5MS?uYf z_ha^Wvf>iwIXj;#GqWtz%a7*+l^>l#AfIV?u6v8&ve$hq_jAbDw5KhFt=_*M;|O@T znLFMv@!ETRgSs0DkcBeq!&zr^SUGg_!@a^_(#HrC-cd!ktPIFbkCw`mG<-W%ve8Q( zl-xhf7}NvEIcavhHY&4x3Vy$^uoDaS5RyH810K@dtKfey$?Ja*`kBuUN9bjm{T4ZB z3)O}d76F&){g+QJ(SU~bQz2)JKILl7mx&6VKzyNK zE$JdS-3ea-X`8Y`Tu6ppR6DGb&tBb&J7Y$&%zZ@B-^PxS({+3 zLchHM>MMk-VcFp%ZlSazr2!IhLSN->DFs&Cp}fX=RQe}7ry*qY1wbMai@J*b(>!q~pu|MAypkRj@30*! z(=mxlf#Q-0WBuQS-wN;I*)hLn;yZPo!VAy*zqKrMI36~)aFJ^W&BB`bHMQ}f@TDYq zU5tjb39?oY8K4~`Hiv5=W9|hB^F!DB-4(-S9WA)3?Yk#$TUlW(Z@m|-89Tr;3pm<2 z`&q0n7&=;4Eqj|v3l8hFH)>4JExERux>QuzIrH))+8Ui$Dz5l(zgKy=fD@%NKB7Ta zj-Mwq;0-u*q>YtKS=+-(5xtnqusJlo-vKg#n+_|qWdoC$7eJ`nb~xN+omHog!HmB- zye79K{nV5+EYevzq7@Xw0&gIL_7FLdau{q)RMin?iTSP=hWc@#6ePH?%|;|LXPz#? zljgbs*qDPQ1lE2Oac7^2rqN-5L`*%v$*x_T)j(|7;`ia3*LF%h3WiBni%gl_j3<+srfL~n zH+nb3A91PDCDlBK)LLqpY81F`85!QqcH$ntEJqigW>Welq&J>GkL2AuA&h@D7nrN( zQn8FNzxpzG9n1RI;FS9SY%Wa>m!f7Z9QRR5z`@RzY)Nq9SU&yfEKJAtkEV7x8j972)lt-t+6qWu}-(U$1Ipz(U?0S^c739??4Jb$Mc(w z+~KQsk4dSgxuir2P75E^bCvP#%@2~F?eRgP0m_Yz7#r>-HkJ2ipJR{!R3Hv-df5CXQGcYH%y zGbTzCmSSnn{+Fhkmn*&_WfsV{{1Adbq7CQTEE%V}aAl86mBRQefR$>(YxH&aMLZiZ z3^7>1*vO=cuen6}G)rsE*~z`F#XW^%vqas;RzH7}m95dILSGJ>C`=t@cej+76-P^| zGdjehOE!by8cZ|LS*P7ZLQuledkyXRJQ!;6;$R>QvGi0I|IX)1Y5ClOuW_N#^4!s1 zRwLNy^0s7(nIx^cwjE|ub^g~M-mP9WF|iwRoj%L-wA9gkN!*iK)0lvt6Yft0rF^4g z3IL1i2kZBb&OcWBN3RZP!J=*1sVrNQuAZL&S*lr9XWk&;&9zTEj#`MiFz_X-Cw+j{ z4H=5n^XADJ5t+VU3h@P5m&R>qw!h5_5Q_S%-RbTQqBS{rHQU_1yg)pOUPrq=HvPEc zUSa4?7)RxbHl^5;f3K3796YxA+c>=Y=qz~oA7@?%dU0bOty?aqA{%w8@&4xpKtbvW z@8PhwR`JB#yhKw_NbV-DA~PRZ$#*OV&$!BDgRO|m8S`lB>~#D*s6UaJik4pk8_iWV z0*A8T-Ab~jTbaG`un(8r0y?JZs(8~gI-%anwhb{!;xMjW;!BPcZKg~y9q&MWANk5V z-_Rcaojv&XZ1U2IPiDs}n{}D&&HIU(1`&g|M~vmO-kFfEuc1+QtE`k-!AF@zL-p(K zTveUiLhHB8f@PPxEUh7f*6&1F@Cpa1u#JunRqf&rIrw|r$53q$X|G*%%orlr7;oz> zxCb~95De`h>D2A(PAeh}3KT_fvS0~h&M-#z80cpU=fF%`vX?? z!lpLe6d$ijl<1KTyMf#W3qZVlx^X$qP5g@(#P+OO4#THYb4(|G z8X7HoS!=!BIx$EZj=f#Cmp>V>0>)5@1X(eH+H51_Y+{KUBl|jSOuX4JskKd|!1$}! z3KNQMLAv6Z7rvTQFD6|lE3;DyWy8afiswkyK}}>{CIeS^J9pzikR#^g!LdXaoK4O8 zw$j|?-R1f|aD9vG!Q~o}g!gFWVi`|f^+aZQ5G62-wm`3&B|p6g1 z;}WCqIy3ElP0qG1ckoSttwb;c)ouC@bv0lhQdainQaaG)OH*MpjdDC7nd0$r5P!P>;?IS4HQ+Kirjf2*@l z->I5Fkn9F`^yRJ)4Noe1icJa^H6-hwYPp{et%VR5CdX0)&4eWkO!G{c`SX~vjY;(z zpgfdBDchg2k&Ks?sM}Y9BM8NPDTnmInSUc zl(qqR{_^ur{3_IX`<3h(H_bJB7_Sp1QTgWyXpi!}GfBCts)E}J$v!1`-Wj)M_pPVQ zc3^cyAfA+%z-Ksdv(_+4_8lckmfl(9Z-Rclu+ni(Vs{>yaQMj_kig>50m^<{Jo0Xf z8S4fWz&d{v{MXqnouLG2`OsC=6nFXlkLz581tgxlS7@oAS(Z}FA!-mrN_$A)(_A4V z&q$3rE6`3+QUTn8ojr}%`e=6dA2MdM1U5s^Y_ceZyFM7EI3(Z4wG}Vst#10|)=yE7 z0-s>y^`uL3E6-nz52@yddcRc2@Dux|A%ZsH&o~sc_>?K0rGB1^>jP zcmY(roe;8dwF%?#LC(jKBgE8LFU1>_^U=<1Jr3jLT6U^10hmsao6yWe3=vG_zwH8|Ef(m(tsTVF z80F&NOnis_HLNE39{|X5*_(EE583(oq=BQ4e*J?OmHxQEf{!e?R{lqM^pZd+AlB&a zl=x|NX%{WuEi2_LLHJf`s(cGqFcg|1M&&%6%V`8L!$6dvYWL5W4IvNH50PtQjqEd3 z+4V^To1bIn&h|(grl4JHDX~;aK*tsn={5 z73mJFp@Pm1mp_NYqulnIYqD{0q1E;S^?p_{Ad`wksG}(ep+ZHQ@JMF}fyqPXvZDO` zjBD5=V^?b?6T@wHDpVpJUlC;Q{rpQa{MJCZ*S^+<4$ZE{NFNkvn*dRkr}%FWgopcb zNItZG_HFFWLI&AuAw-2On2^b6${w1{BU#ir$HGK^N8qlzcEM7ot}tlT)mjD*Kd5Ud zR=5wH;N*(gQ827C>6N;Z!qlipt?jU{j13l>iDId#NQZB0pcEdJh~{8Ut)KtLama-6 zIJ;_ygWTrVoDvOz$H&3V0S<0e4!X?fH}LS_-&di>$=O@X^(XibqEpMsSvR8v#gek> zhZdB&gTf`bg?zmcrE!ahj5duGb7<#`Om}zqD=p6#{SjudH`-d@mev#+V|`EOisR>$ zeSOW5E6S}al9)S2H?uF{-oSBBAhi0@a z(gY4!1wDoHiON$IiNkoQ@Yt+sMwDWJy_}=}-(iM~GH%%CE;T2Hw`ca3F2%%ZNx}gg z3LCm09FNk_A(CzdKlz0JV_-h|ij>fR#H*S}MY99+1$^{<{kQ$=`^Wpqu#tJfz&jF{ zewYBq#@QlwQ?RaseCd%6GOJqf}Oy&Krn4h4_ZdzweT85s?zz*)Wx3>gPUERk{XZPEqlamww z+mR&$V*bWGb8Fregi2NxVTeY12Zp3G;?IsJQ>UC85Mf(KkjzWYrctR=d3Ls3S3Jae ztT)d0=PEwXsp;z3eHGLZU?ceRHwYpQ&NAP;4k4h2R;8*eEY}@e;~QqubaDkcw|hVK#v@QgcK32%220S^b}}nA3{}u zmZT&V7~}YyW$Oo-nX;Dt4Z;dgdLT9zLQYP+sHlGNC^=Db`WZg_gDM0GAbbk|+M9!e z1K4EFp6kYUz6AQPfK7~xlSimTi)u+Y;`oV z@^E$iZGC+ltq6^Jk*VYniNiWD^RYxuB;woa+w(aJxr~&QlrS|l^=PhbXQGaDg2kBU z1Sj{DcSHCSKMU~u77JFtEEqp7acg(fGgOFl`xD#+r-I!TSFj+gXh2jLsldH=ndKW> zzAi0*+Hyc$j$#+lAwh5@AhQ|%4G^W)qg)^^< z`@WJ_co@tHVWy&B4(O!2AFE30kcHeRSDuN6$!9jzsx?s{as>12<7O#yS@*3}%X9>7 ztbooMvN&yo6s-zrn&*^sOn7dBu!>9)j+(XSykuBje>7xif2i;hu7*=%xD>2MwjAfH zEhzmj1Nc7FYBj!CI^zNU{#WtIPs4!#SqxZgC)7fN6^8u=_D-@G4WqtL6_(D%(eeg3 zA5N39vtxuus=5Lyx@RCaX1WyC!Ggl2}8GFB_t&L zzwaP1PAKGfpRKmELNay$;xC>(gsq*0=1QGrC?KCAL@^x+SqFQT6>KC%R9~4JKNK&q zr?<)%{%xUbTGM~X@>(vrD^Jjca(b{4+RU?#xqV&SjIP21#1{aF0~{fAoDt1&V1gIG zXS$Hk_!T}>S-P1x3U`@Xua~A$fSTSpJ)Ujy>9Ua=ejcMoyhWT^^+Z9G1Jpe;7OjmP z7iQ*+Y~lDCgZ|Te_Ba=#76_c1KL?7rWol*Zy!}eUH(Jf%Ie89WYw3y>AMw3$s{?2b zeFq(CzD!nNu6N4d1LSTJJbu>^Pn`r)#-A~3eVViWKd0)DL7i!V^)axPKb8+S7Typc zC&IUzT-+!ab1;Wh&(d#!4#EoK8py4!dRK3NzBAKq&>r1``JKKpv>IUf#{| z9NGW2nYU81*kZ4NVj|l7wClYZ-LYZ%wHi z5l>HU4}*Ls?qCX)4tJ-;>o62@$7|p~rxxRUGoNS18M-;RuoOtF@-Ex$qWl?nsot^% zOU#`wsypXgdIhZ-)hTz@kf<3ee^KJ4F;uHwj4ujc!LxV>guL z6Lufu`?XJ!qqj-sJa|D*WFvUtF0|FaWx7A?n70oXPNaVa&chgSLiC^R@Zu!L@yUaP z$o#I#M|>>k+7ew*pBohNk!q8W@c+Z!Uq)5kZSUi-AdS*Uw{&+%cMBpZA>9qqNTamU z9ZGk1OSg1)cO&^<+xtG>^E>aKmyctQ;SkxMz4nT^=Dg-L=k+wtClvWwxnnza8hUt` ziNSB^czI~KGC}YDwE4;A(Ra%WUw5AJ>>UrQ(cJu^3xLy4Bq_R+HLRcD*lWXqxyJu} zKlL%4Z`AA+b5q6UBF({j7o~@!A#*nUMQW*W6y7S|ge@&^XGe!hyqs4)Me8F++9L>} zAF)`@usuvQrk~}eVZUOwWDU_15AV*I>)_xB95B_lwCeAjR`)vVIr_QrU4jZGQfcUH ziEE$#2fml_ed`x@DT(G{Q5#kbwhxE!VNcrb{!E=|v#!QNvdXg(kIPzs9J1Kp?6xfE zc4ky`FKo(;var6MXs)0scTX6eSk-e~!Yj}Nd(->^XlA}7=Dz`wxJNVP>&EdqS5qHs zHBmoQ@d$l(m?5}`zug>&@9ahFEOU9$)7_2X)Go-)9iNM(GY(T5#S4oD&#t=-c33+| z@f{7pkiq~t*R8TQH{80EikYLC!XmLR1(kL^CX$%oY(Jr@v7(8XZY~o!4W zPS02Hh_?wZ(s?`z1w@nUJ=Q9X+aWWF;bL>NM!n|4O{o0>zDx5%u4HF#($H{Y z(MJGK-tWK;3wT+k;k42yp?&j|8Bbo1=owt*{Q5~;&Tyl2 zDf=pBLs%z)Uoy^tJ_FgWfzGPp^OlfZQAe5})PYPRqBU*(oCJuWWPV>&ph=xEb7RW$3PUv)oJa(GK` zkM7@1F-36Cv5Rs(=pK4%ba~}uapOG^-SDUs`xdY&QF`X?@n?S$G^)!+^|V8QUSfQy z*8Jo}^(jq7?~DKUWsL+Bp8wEI){|A=zn}1Z{%6?8Y_Wk;az1TX#@a5Xf!GS(ctjYOgX3z4m5Ln@E)sgw^tZ;tCIA}Y(@L2PiCee1%nR)JW_5|McJbT(;;%vZ2F5e4BoTShnUbT`LnuNQVz z)JmMp@El;=q?W&)j7-lWZQw<6Gj@xZ_G9o}zkSWNc{PVP?F<4{L?uX2XsJ%a>twK; z)=A((Wj%vcJHa0t9oHcgW@pV8)csM;D6;*dZ zuoR9mUeeHq{h^A!pZR0Pp1-#=E1%gOjs1XcC;sKjpj}tSGg7bf7XPsF3NQ{Mbkgd1pCgou%1^!y|Y$(-*WmJXD_P%;` zV7{?w=Eoy8jPmPr`7G>J5~X)Y?JdrB1lu^8nU|QI#mE5sLCA8oo-4jBi@K{)p%&CU zd0l_r`#4s5dRbZXBpbq4$%uDhm6bK50&-B7mQD`}L}?i&V@G`^4{pd^X1ExvPr1SA z0;OBX6fBODncSb*mbezRQ{h}tnloLAccRPpR;cM5s`y&u0hy>i(PG=be^ReL7$4Ch zVpwSvweUyf2vwnehs}O{e|FN`WgR-X1kC&4*@5>7zVY|n21}AoY>oFvxWH7eJtv}S zue4{y>gM{E4+HW3VQ`d#ND*QXq3bRN6Z*B`Rc^nh0gc)+yz}m5^2FIG8V=kVk;(2#2gcvPO3yrNxEWaYvMi= z-1(>H6J6s4o;80v^D7Hp9eazGFhwFVG_Goto#5iUdAO!EspYRDRaKdjTuPSGLWgU2g3x7KVRbzVz41P2rb~yi!2$M?8=GnI>#)nbmyGRi&=FWFae_iWdgu zP%?L2bGDy@?^RM0rZn_feK9)RJ?BMEky5ZLX!woc&GW{tRk0umDkntO@_<{f6yq}| zQs7dEyP%^%1SWeu%T1pG(0{A@G->QuhnLpQn=Z z{2M$HQHXDuhqGjnF?;p7rUMmI`A);J<1uD)LhydpRRs1^%95!wUa%s)UYqewcevz4 zs9J4wRvu1+D;z#w$4}i#Q96IgrB_n3u#i#-g#kywb67CqD8!B=AuZ;kZU#8&WOYBM zZ@dnBJR)&FKijDG;kk96(kOSK3igPwdi3a22fTbI%iL`q{-ih_$9ZO~!SI?GW?Q$$ z63{W3zIpd(hI#&tH6Tn?-|zppPc|+^vAjbg#Fy#i5U_cxZMesZ6r_`@7ZTScZyZNF zo6%P9F!iYjI50Uy<{`!y@|pzN=D8?%x5C{`nVDKG^#I$$hTwK@&64s^)SCtlHdL4|A#!nVX`gfJ15tfhBt@TpsOdW;4 zu7?|OYDX!kg$NuGs64_IaCaJu&6lFW?;|I) zVNxtXdox6JPCp)HM5JpY*hh`}Ar?*eU}Ga|J=a-)Si9ANv$BHSIiMV{ROJ2me;b%} z_i2)|4$`_4erFl(n#JBTUmlg5k={GlaAJ$ha12SX(u>5;PweBPhNH@aCt=Aw1DfhX zdqF?Z#tcj%yPw56)|@6e4V!RIYQYKnJEprK^5!Zm`Kr!n3EGNj#h>HdbJb!AT&+D_ zn+wXM@)gyjdoNZPqb~WdxkfVJxnYTXyGTdMSYv3BEQ^9C^Gj=d6M?{ zx-(pApAz)&LN%_KdG7|__p-mjuX-~Q>5vhk{Y3G8V_82~<=LpaYm|}-^vssM}5_Uv-;#iXUQVO_ot_G{F-Ex2{H}j)^ zj}D0OM0+*=&D&>-X9RG+a&Uo0M%P=2udCf5JiihiN@VY8d1q0805VF2y@7H1NYI=3 zP=x$eTvT&B4ucn;`0+VCrp~m&@GY%^!s-5&`_O$6F1KYcd36Ye3qTVp*vw4Kgx8Zmwj= zYV1r#t$_yv!dv8<4vgEqSQx_Sg?tDzPhGl2Pl8u*?^6p&DVjAPbEDBK*Xq{>E0JM% zz}%cwkbcNke>i&<$XM;1{7L@}gMsZPVxn7P@2*-3$0rHG$}h;!5|V9Kaaq{fKc56Q zA4=XCIWdbruWyb)GY04qna^+S|KOE>Vap5)GE$fZ3msd$;{P2!B03!uFS}0e&qr(PiFhZPIG~pa=EF^6M z3wbOobHj_jsR}4~gp1HmN}5xuu+%RZf~jITNwf68_y7Qk(V*$sQ%#i)>&&eP^|Nr^#3-d~3{Lw4e!j`BU;uMC}DfN@0iR6>8)R5j?OL#8^`%HRMtA2stshlS9>>>l&;>uLshJ%NZj8wFxuSo zl@I&#lA`!uyxI32eBo5^Nk0TRIWIGViT5T8r@x?6l=0Bh7Xl3jAVze3b(O~JxOM;M ze82#gm6a7p2Gu>@?)OBJ=)Jm?$OX$9BL%Y{`hJJT*(^o>L+=nAp2Dq8{S95I37vP5 za^LJkIGu65@)DaikzQ8QeBKUce`BI7Q?$o7y6-acM@l_gH0fMO4#(3XM33`jDm825 zW&CTmo4Hm5*QlPIHG6)9Bl7OIeithBd|#Es=%;;{PRs&?^**-A)i(mL(i6yNoqz_c zI1C!kJiLL()3WEeLdt0N!YrP1NlABl_)>a)^k)eC{f=T~FIVfa5K!nBa3}REW#@}h zv{h4kCTr&3|6rc@_F2^?M|p-{K6_);j!hC)+eB%v2rIUsUOt1iK&Hn8&!jFG_fFnk zh=3)=&Jx^b#X&sItJ5v|jKuJ9Gg?$VH3(KSW$iVx5utv7MW2){I%wqqUC9v5T#RcCutPr~W$ z0@`Cy-||{F>6xlt8%tzJC^k4;GCrW8G=TANwCm|U>BKfkXBROiu-cHDATQhC0c_=Cjr1S%l;7F5|J zp&g3|O9R0WV%HTY0Dvsooo)1i+B$;a15j+lf4lR;d2b?9SH)D}hr+{OZjfw*58#>d zhRpg}dFzCG-Mi3v)$(!Z1}^-p_^!HUOUK=rXDWBwc~c*pkMiODjy2sDtd36q9eLu$ zMoH2`$;pRO60;NU*}prU1`7In;18IU*vcGwwW66idkRTL%?hKrDswI}!hlb02|!My zuvk%1(ff8c7nBIaB`15#8pkiXZKrwwTMuAY#}NS>BBIo|xS*NR)<<`s^i%qwwgjXz zgr4r%%gGyxT{AdQDsW&5(4ZUB-q+6-?7GIbEaEz^FcHe6Tv)gDS+`O9QA1RB9t#C14HM6R;DSQZae#ej@uC>!Lo9)p``^ZGZo$MICvZ^xL z$=O+f@{gd>QrmVoco>*EP*z!2$I_MeOs48|w&v5}oHY3-Q1bu;m%4l80=%FSRMS*M zGw4UE8C+jOs$fSm+5-?V1s^U{5Lok%W>5*Z4!%hdXRd->2%;?6OdzFqpkAswl$P89 zWckwEw(u5yFDSTn=6qM+AH|2CMTS;|^^Ii=OK}PiVEB&$nFOTg-wzQ$rc>aHSbk`UM2$n{Y{*ue40Ox~V; zwbTScAzXG62e$(9Ab&(u7w@xodWmRgs7}TWN!xhHFiA=;vv^2HO}5@FC79O$ok2nM z#6ytApn->jW74jR;#83ix&E`APQvQ|)5>|dU{<#nWib~M?+5JldmR{Q;Kik|2056$ zn|!$IC8+j^;;ZJeU9!m3N_$1#k|h_<^!leC`8tMf ztM>ya*(*Z2e|Y$_gO7oc^_=BnX8^<5Pj=%3bSz@x;n|Cei=kxhc@C*CRdjj%@gAco z=}ZR-;2=2YEjvo8gM|k|g>Mtt0-w&C$T4f$cO}E|%a_Xydw__dL&Qbmb2W~is+A@~ zam;$&Im309cKfXVB;kRX1BFW@P~>B&&0&9XZeqQ%RButRpNb346J(Qq{;s9f9`;qw ze>lcpoUbF?p~ zq^1^Izm=9IZKn_Bf4pHK03sW2GZ5XU5Otj~iAyn_6=&vMczpFOnTgk+cAiR4$NkEb z5PW6WxRqpiabVc&Kn8ueKmD|w{@L|U%hTf>fK1+2pe&79P8M|K1AMv#WIYymKF(Xu zRr#7)1}+5MkgEwZUp>f%|Ks-w$4a*S1ID_GcX^pGz>a8f6%`H79fNwwKWJqAifP@t zrP!t4kcdn6nhjVEQldjRmLwMJaL4^##9sL(^mE<;U2;vobu$_Ae0Y?gm1o&W4R zcqsmUCbwwQ1?Vs*rpdd!i&Q!aHOl)uay|7+DFWnT>aaJ3ueVx5^xDuS9V*HXHp^#C z-(qJ~0m;!c0gn!MH+hPv1^Mc8k2DqQT?2dhx)_9K*iVN%Re&`=Hs5r6 zLVNwvgUXsc^b0y-ICEc+{JHex*&O3xlyEXqVk)W#b$RM!uytCj=%yHO@i!ZDV8eGB zk6YHO@LH9l@=_p$a|Hoy8@(c%)^(esKhNNl1Iv3(7Pkt#Z0TEV3dXiA1`sCiT_y!I zl}BRQW3czo_Qg^7sPV}2_p_E%6k0gRGf}-WBbXWXJ|>d(nofd533@?$t!C1GhLlRA zqA08fl{CI>m6tX-rtb-5XL6#AH2ste>--+H)1R4g!;22Gro^{(Sg+yc;IjMOX3IB!!L9-5%z34lm#Hns-NrPY}-T z(wTS6rs)rc`p108=;t%3D-5k$m7~h=<8-oT*?*g}G0tKNZ()pPkm~jvLp}n<19}J_ z0i1bCZ?VsFi!B1SL>{4(gH795P@`5CS@8tph4dOfe`@feN67@lo;Pf97xeZ@>yP~L zrMUceNXGucHcNBdJ6y=J#+0chBk$q0vrrz2DpZCK1*B5>-oLmC!&qojG9~}MKUoN0 ztso<#S!MQZ%uAQBaw%FlL`FP7gv}qVz*>?&nQKLBDwH@Q9L<6XIVso47u$2JEsWHJ zydzB}v8SmM)Va=m>C?{^2lVyWvx@$V{}?FW5dUXsyYCh=dV#>KiH`6MYm4bkxmqL+ z9%}g~39&Dl;!&4sJg}%#Yl*n1UKXIz?7e8dd4wrGe5Kfa!sih<=k8Sn9H$K>c{+MZ zh#B29juipLTM)aB0If;{#ouGaMY9uwZ$det8Na$<5*EB0ZsL$ho2Uo1HyF39nO_Q_ z^OY2-t94UC-htzLU7Ds<5C8N~r2)s|pd`LYoD-%9V!wVy@c4nlZ1heT-*=K9mr4w? ziYs>L9*tDDcL9vqW7*E_{|tJvp*DQFa@Ce+Cfa?*#VFh5{oSo#fNU_$m9A{xu$h*t2BJ|dQ! z%HNPU#0QWOLZ8Kb0Yp>C>}Wf|NWLhl^sX(Z(k{I9##zK=g@kYgRN!kCZaK#*BSU^q zi41*|m0<3a;gkUT3?pRBPdHl`f**7l$B6tFRQUTm8bCABqxqRl^1l(dtS^XOWbN^c z(f(bOfB(Wk0#*nI$$VD|eCmI;10<{>1^4D%p<>chNTVGpy2YiCU(v}902iGE{q}&8bx<|{cPvP84 zX4VeiEua;}AK)=_(>t%`B-?;=?eapsV`^%bqrQJQ;olijYDbH@bA@gEXix$Mf`#gf z$8MtS&_@YESeF?>-Mhy%PEMD?RK_7^7;O>yocCwu4)(f+r`G; zqod8)Oos_Aqr<|&fJXwb z4FU<`S0p5Q6DTE(M-2%HxV;>MH=r1LbSKMV^bBn$AOPmjo2iDiYhMdeu?U%}7$yir z(cl^ap{mxQA*@J46sKJd=Ts?de+-a`1-&5TiRu9UR?F>PA>)UdiF}pdFUX<{dH2gt zkKFnlFi0fcH)OEdWAVD~w80W{U;14^%&^NrO`VIYkDKm!)(5ZyU5~@+`!IVm-dEly z?eHLs!YvA3sJ5I23>ZRH+on^vK|wyg+dk@a&_n?LyNry?3$&c0-CaOxP{#mO$WS3= z|E#Sr89H!5$$T+b!C-0VVYW&XWCapcT@NAp+w(PtK+jq{8t@ssK~CImt=Z$+fd_;~@lauvIfAAZXEEREQ{K*!a+x+eCGLs$+!|m(`{5U$QI8$t4SY z7F_M+gZYzWSyB%kvKorYf8tYG%3O5@cEutnKnHyQ5?~Q32|=7Dj#)|5)(S%B_KxH_ z_e+bpB_L$0bUrvZ2*T^e6Kf*bIRX`*O|8yv6jCQNB>b*u>%cSr(q}VSfE_^mmS6rh zQs|MCG`ZFfaQXyKe_^&h-qMpTf|+W!Jsf?O|1TYoh77VZM4vG$o!?MUGb_FQ9ZqW3 zj@~;sEbuMuVX^b=AO0-4zW1gwh~x=agmT`h!wi4)iR z@$l?yg$o4e&6hyBfxTKLN%HvXTQ7H43hrso)<|?1`3qA6q)Q{fLsi;c5S zDYfQ-?ve8|?iv$k;BdJ@SQxBB z*FJC(NeMbqA2%BWRKc6IQ09~N>%;k4KxDIR`LhKA%<8a%zjcs+Mu`KAS9YjrqS@VP z{SMn0J)mx!`L_9!l+0szK!2Fyx**}D$&&b~*Zzr;it2l8;?2wN+$yBUZ1PD|-rka8 zDCrr37Q!7@|1U)pKeNOek|j8rbIj8A}y{7Nh<_Jq}AG6&l6?sc?K} zpkvMwjBzCoHNqj^zykLr?B7H54PrN5*NJO@IqZK1MV%}9&UEn_sIbq1hDIHpU&r%w z8f$h#tw%GUY_vqs8n<*p9oH4p56{=E0 zcFS%f?D6v`XU+LIfUIWh822$UH_bgHF(RO%NUK^Fq3Wfe1EeYrI5rTI{r)2sCFH{J zgh?Y_VqqBV?NIA21~9{M>yscRjEuh2v*$*q5pRxsm@B@JGHDBuk3J*JR{3-H5<-isGp+GvV6fQRA@% zeC4bx&ZuEQ1$WM~f#R3d0>3nX-dzK;QO3Z}z^VNZ4-b#ZAcJ5OA}gyXv48GQ-*+Ny zG8lnzND+UDyP!J@j4yk)!%}N-KD2lq9Zkoc#Hv;mG@dPojkyw zH*jLV^ZaOhd>nDIj4m_i<|RSMAYb69`+dKf48bNaeFm_vsuC}+x_PP=a>&>w5`5_w zI$8S$2ILAU-AJDQ>WcVgA{ASD^1wkcmGb3JKX+9KTb7Vb!7_b;yF|I#L6sE5{SoWI z!z?wXCx%}0Wg@ouEem!Z6ACn%lr#h)dv4pk-M=<-W0Ox-(#e6am!h}CmwG=O!*456 zdSZ%QS-NTkq9QsIy&BW1|BS;b$6I@h$|I-Z>hr|vF zjkID7?1BK1HD{ZpCSKb4UVo7YBQ@UoXU%dr|M4-NAz-BopO1}(WDI3qzSr+(Q2uap z*(^;VA*x2?q=CCh4!L7v5uqyd(em03ge&CBct|!s*wkmUy;SOkP=6s?c^I9>1OeU> ziRCn9_YGe=-MHCJ5D#E*ZWZI$whhEHFISmkhK{Dy@_v_hH9dPHqcK9y@b`_h=%g^3 zBPaC5V*+erBJU!8zhiQ@LUVoE%je?!_sn#OW+cEQ<+FsT9MkS3e%aDb6n!=l1UglA zAREtx()>+V{2zrI4Tg|hNgn!8DPFVwW4j+P_X&twMRjt1)LRQIW5E}NMv)!1PKVXg z6>!WQ^4OmNhK)`Y^$qdB#IaHsW%{k)pMYmlYV;q?Ssf}We4vw}qoezpWgBSX=GX#d zj3wM@w{oAhg`UVrzbBxX39yL=?HH99g-}joqer88#jqfL%5QAsF_? z&wwC4BJ{-Hf7oMiMFR_VY@QLq7t^XD<~-Ni0et;2NX0x3obOtXuptSaD&qV{(Bdn} zoN2&EDZB{G@M1k16ET`{1Tt84~cg3Rr;vO|rl1Ah*3Zdrw!_!$~J<4xA2Y z!#DFwG#lx|sp1cyS6<>^+C6-j4m}*F5`ufy6ox~ z@6(JlqFf2^ymssuP$;wC{c{ukyY?9&q)Oz_&7ilH+7mRbCI088CXhy%lzBvq`zXR7 z${9f-gtDpAQ)J9!E&$SjW4;@TgTxqGn>RKd? zfdDsvAjnQ{#6PK~E6B=b>D{j>xm%Q>#n-6DRv*|g+@`kEQ5pYZh5oH0fWVSpMR^E2 z-h-}rakYUQl_D`;8-45r0gexNu4#PEbnT-_W#6m)@1I6Bivt3yt*ZXq-BoTRxLtwy zhyA(uQ9@4*nmy22gc&=BX#S=bF*0XqWF(Rk2Z5HAabk=DA(_c;92cxe|NnGb0H=@> zoyH4V;R~#C*DthA%~L$FB#}6pZ6BX!S#h~<(@Ti@$}HAmzvlS7dz-+be=t+dg_8kH z^$uABelRN(S=1b~!-n(z)Jqxu1>m}K|9GQe=R4V+pdfw2$z)3OLZMYWvdjsOYSo*H zcBwm@U>CoYv!?6UuQ13qO<f>1sToM?G+S?iiwex4&GBVbRr3`qf8fn zkaA+Pv0CHye*9y)g+MOM+(1#tk}8^!Id)>`%2iHzdjJ4IXArJ@SA8xpXgTrF5!p8^ zqQv>b#|SfPF-rn@t^VoZT92W<2{`3zJW~Fh3}g)>Xa#n&uGN)}+(ZUu_G-g}vg zHwlP;@^qW|a#_M=Uih(E-FB6yCM6+xaWjhGCYPfb&N#s!#Bx;Oh0gxKlJ$FLM46`> z{M{_7#!uLsY&2`WFmJAI=g;qV8nKtJyNh}wh4(o=QNG2w0kJq5F>t8p))@@JdG!tb zBFri&$T0E5mYg%HaQ87XO5ps_?$QBYj(TW+qGZo?i3mzomMBEWr4NTHDzSzWq4nD{*11QvLWj~;-Vp{{KYD`)XG zl~45p;_Bo5`N%gCmXB=^C-7`TW>3=qgr6ivlfpNa^74bAB&7wMbfL$a=K>)3;0osc z{CM9(>|u4WGp5t{sY_xA-5B5b7P>pA1QDmCsk{3vxT22gw~03(;cv2S%j`i+M@JyC=4m4cJi~0dLa=XR zyrfp_wo_ezwOa?cLT;Fbfx+5yR3PJQIs!~{5{h$Bq?;93BvBL4QfgY;@ug+tcF@&k&6~6 zN7uSHYr5DYwdj_R2+ml{&Fl0Cj@1Ob^1QbzjJL6VCrW8@j^AR_JP!k>F4WkR7axq= zraMOqGo!%DvSDYJ8@1CtDX$vk zh7}j!idabAfaMYb;b@usuZ&iZqRd)FW^Q^$MrGauVI9q;KBOP4+4~WUu-s=#Mb}~ zy0}UU6B|^A2o4MmFRL$QwZC1O8KrX@!M@aOt>kUxKDId>2qh{x#O|K8AZB7-p2FHP zl||VllnoF5!1`30=q{_fGtHEzsk8h!u6sl;{k^F5Aw{8uDkS7$GgUxIUnyNu)paogZpu8@i^vR>{%GQ&O4gs z93zFUo`_N}QJt3EaiqeSThRMNk8L%g{O(0Gez>wiPhqdR>K!ND#}nSUjenRL1>wT1 z>B9E*_Mk^(ZOIipW6OHPn~%X@{T^=CBWaI;mJXP(n}RpadlvvbYq|I>7o|LUb$VLA zU((oeeA~W!oqFwL23s*9X%m`$TJH|e@43DQ=b%%ZcD>>OKr7~g zpAIx)TF?6F-~UGUct87PH6!w!|m9P_JlA$BdlGxx1$qz1JS#^lH*o|tutCz*NEP@mAXw$cJpQbeRehT z*}iK%)ULnW39aR7F7PN`-X=ysEt#s-8TGpi7W!j&JxW|%a!eptW_S|d29Nx_w=3FS zx%$cQJ~6w9O?!ZNmCh}VuUt1U476TU;@vH97utN4fpJbHM3`I{$ElPE|NdO2zt%Bv zRvRlC#k=H_;7rUII{4aL?3IJL1SgO2xha^0_ysTGgx)y71zU+0q$YRTd%VLD} zU~X=1r(N9M)zy`mAQ-BGw7{h8A-4h3CsmaZ;avvCD zbODT4uY%lO;&Zg&QqhW@etk~%@p&-Eum_#n@=7*zDLXSpLot9y08Rv&xB>%-9R4g| zMPzeXtUavrNd22FE=lU5Wg*`P&2|$H67jvrbn4`*(=Z1*qG?(_M+b-XNF#=|7cHS|hHt^NjxFqg+>gH6!61OBl1db=l=8Rd$Zf?l%p2 zLVCyd*E>whg5xE|UP8l5?lQlGY@O^KSs%aORlQ6(Gl{>gZlYN?Bz`i04^Z|V_p!4W zJbcW&EIl&6>HX}M-}$*D1q-qDBOJG)lO98)8ADHiDW7klT+p@L&pmgtJJr9WgM9S2<*`(6}IrE&1i-=5(D}KH+4}vUIny?5&PnofP)@VXqn#y)lLES}=(> z>5J$3JXN#5G85dU^xJdvcG&b6c>KR!`^fB5T@fk&7-Dgr%2R$$7sl!ahyO}6^)%xN z0RO;B`0-)hhOy|6gUoU3<AKc`I;%iTx{ZHtsrr>mZ1!+ zNU@??a&Ylbrz~fFp`(8T`i{E8$l|FQ^wr6zf{Q8bD|LA8->z7}PV~(3UABwm9i=0{2wzvvUCDhLNEuXCDw==Bm z1y72xy($A^^K_Z%Pdht$=LPkviOd6~w4{($ancyQXum`yRGqgLFT9}Fm)+f-Mj$7J zCRZ{>nAy!DMIml9dDn`I8Zn=$%_$fTB5Y!2_%jy}yBU45_mNNI$3uOdhE&lZ`AALz zm|iuY&7~EwK!OIt#43i6*WvY- zt&#K*=+A#Pac&L9OrkddXQ?i}ZP!;b#X1zQF1fziU2v!vo+IEIZ5HrH#R=SP6Qz5$ zBA>b#r*9b>$Gv<8B@z{HSbX#$L-|v)Nj76GrfY=c>bxU|>_IB5*PF=NYPqwNXJW9b zs?F{*>$QYKcyDjlZ6oY&`@+q5AEOxyxpVK!+?U@(-kY)#-CczB3w*ymzujo!w;XvK zjGmk5BquBr;GNr>?1(E`j+EA^-z8=A_>>nJz!^SmBeOZuXT7)1Za!6KvaNR6a%+y7 z6<5h=`JTX4WM7f-5R`b&pc<(EBIYnYpKJU;VH4boiP_hK?1IsE82{@2XVa-M;a9uY zJkh$|pB%NxJzK>~^WZq8v4!sRtsW6RDV*nNscaw5#D6#LX4s2;D_3oy&s+Zq=dQO^ zhIBlVETE_|e;r%LhRVT$UlXCSqodgFjf>5sU|)gm$kb@QkS1+tv5iXcAkV&CQS6na z{zjg4Lb1~j% z;b8xeLbszMyc7NbE0eR9LE3CVh)!cje!`rLFU~ms2r_~rAF7T)>3~^SapGdY&8yM; z?rVOtQPRPoFAh%BAK-?*I=*M7afMeVS)H^UwP%o6YYagNrp>AuvlVq%16X^7VX!&%TY1zIIpn%K;<`Ms4=gV z#fFe;3N`R-*W;e$a=IreL?u+xLI{%#aaG(J&pK|}USssUH0ZEvjqCf&djvJm7$$Uc zIBUPj{A1BC!_gi;TTP07&td;WvhYsc<4}i6Yrmw-LAE;eNLXu1fVr6L`QlTUiPzJQ zJ}G1?`##^-)ldCGgI6Ldwia3B6g0mMt6SOIp$zCk-L2=)B=D{#KHh!9xhFL;Cb9P> z)TlScw&&I=rYNSWXReC5-9E43@oEmpGGiIRg?D9Fik>RxHD7^1SPYPc6V;VpA zwRVXW64kR^=l~!Sii>k*Alk$AMV^e7)IR*gdod7an%_9_V+! zr8ABW_j7Yh8@We_`pbqG8M7?xBhh1RS6MJO3Hw3^KNAvj1xa2Wju60G+c7L~OhH_ESy2eK9#Fv}m=eE?Q{3_GWoNb=d z4713(Dx!R|I4*-Vi$QFjOoh_x?XKu0Vo&oog9O_bVM(q0r@@2S6=GaG=Z+> z5{+!ohxSe;@`^)yQza{Ui0h(cmrDgQjNYzpA-%WO<_TrV9C?4{yA6MDD7w7G5{`+t zv~HT@^U6MSHKEtdJymO@wxWEv3%|mllBq0u^<+q?$QV{*LQps_rB;ovrkGT1PBX$y z$FEA2O)9!ZKpcTGGV}TOQEQrodtY5F#)vkV4pwL~6#pS;jAdWzD(YoajYk8B~YdpJK^aFn67mpUh{d{5v z^|SP^xKqpherZgEG*44m&ifq-It(GN8^)ihz@zT3XwE5yfA(F}d5m2uYWEMNctvgb zM~g?64#8o(NWgaPm#dG?c6`w<^H-ddyk)Dp%cZfqzLy=lA#FNkju^^31Odjx0lp%i&UT?@O%fwWfo zZ1vcM#GCM7o+r=Mf#F|aUbLTe>=i>ds;q3#-SS|OoCl7aY@Q<#)XVPI=5(iEE4-+x zf4o;K9h_TBdC?P2q+6`xMCAE9r$Rlq7izm&yNsN5Q>i8&&r{~Z+zOq{NN-sP*Y;`|F=)JJKt#MzO8)pAJhX7l%qfVQfyM@d1Zyl4&;POQ8C#tqM(xkV$t4A!2~Iq-@3%+& z{U5fj4Hn$o?#`Tr#HbNOJ|++w?^V#CHoZ(_&%#b=&CZ$}yr5nCrEA17rVR2Qvcs~8fU_a9_L?BQ{%Z0>K<_+5v zf_C(FHpZgoY2uLCNZ7#ncuA<*VPl=kgPpK{U;97+{m-lkx5A(-8ESzhUQ>B`k$>w0+8XBpF6j~~^&b=?VK zGGXFpxt}d#v_4xnwED8~l^oab)A0ZY~Xku9|6tvcVZ!czAgxNy+lB}!#NNeIJ;XUL-EVG-CD zz?d_3M7HHmhuO|O%D3(r(mhU|Eg&E2Ibr@nH%U$XXc-gGC*t&Y#J&IYiJ7*ALhs|p z5f{(ygY1Fn4A@lH`7&=9I8>7HGIpv6-ISJKHV4bo;gV{~n~d%ndfbsXfW^~QnXovu3g_JUbyh=m@PLP-!V!u6 z*Ae9i-jCb!A&L-J|VYTN^ zG+8HF`9IpURTf}Dp`-csi;~{Fa5C1+TzjDk?KU+s)bD)Q+e1+z;Ax}Z-_+EVp)LOm zVR^8KA$?g%8g4pIjN<@(nu^yccXn@^7|o-~Z9mU+iI1Z0LlRrAr{Bf5+)w`G6YLZ} zjj`w)wrhOd0o^E7mZNH9?i}X>TMzH;o6b^riuhpnE7NRzh+zaTnjjw-CssQrS(yG* z;ZMWgT(*B%f1yt#Xk6(04Ffqle_s-qQ9evb;lp2q|6r14-=22pYzjr1{PP6JL)f7M zBQt3#$vC4BzWDt2^`or|ao?SUD!+w{@q`5m(F|3OFwlhPNz=ja^}mAxL&KIA zCf-4UE7^rS?msU&ZYac^ey<0t=Kqf5i!dzs3MC&`k`({@i{cp$m{IV6fZp0qEYNyZ zTfr=Hs*Vorb>JMZ1<#hH<~5a;n!miIA|W`S6@&mDfd(G&tT&0A#TT-kf|*+Ad3Mm9 zrbEWVqxbR=uGU1BB-|S@NVz45e|LQaMpWRB3SHhVM#zY>smn|p`{#=D?=zKd$2j~T zg9|_aLq)3})yjeV3x<@%Shp$P83_tMU-s`XS2Un8Mtjai-$14T0S}t$!78rswNtP5 zC&u?+{$3zR3&%r=za)b^KlYo?+o{=YYh&$i2CpEyD})R!ET!X=T z94FC`Lk9Ps7b;k#tCKEeGsxKf^ID=Hg*t4u%=+?wOZ?BrL(g^MNM(Yp^?!TLHPQ#B z>bm4BImUk<`QPpH{~zdoXB6$IIST9B-Ci*uf;HA(3n{ve*Yp0P@>XEeV@5>26pEW~6H)i8p+H@rkWnvMsRU5r5Qd?n1$$oBeHE()e7eF4z>fe{Efk{fy93yB z@aAOzy=HE?$x4VD&WB{DK>sm-W4kyxRjIgTA%RM;1WwB-P1Dra)YJ!{P;oR{nalCb z$k^Bm)cFDO4c!o_$i*VyMuOr*{~e7W9Ys*W3%GdPyu9@>rXrIQhUPP3ZhplC zt?Ju``Ies*l(8A8oh^af1)y$pahwQM155z$HvhlsuKXS9wGZ2}m1Ue5<4~BPBO1nt zvBW48jioqw>lE2XGReM_#ul=SknD^!jEY1EMRr-DsE86{O%f7^lk=I*`=0k)*ZUW| zbIlLmYv!7HzVmtJ`8@aYeD3>x^w(J_-$s0BTSJ1S>tLE@q5G*a#X^Tkni_?mKz*?R zif`(uPoG7WCU4pc%jm3pu2!X`P3MMSZ+>kSF;JmB-47$1Sd2)b3O`gX{d$c!D7Oe! zKo0I}pxmO=e7FiCDMbuMZ*#GEYpvY?TQe38T**wo-rNHlU@EomTn`I7tH0x z)&gxPz?!RguAh|z8~l`0>=MmZw}JlF<}&R6j>7)&fWcA+x_ZPrE~MKn@A7I+z&jMd zYb*+BcO*?mNon@U#zF+kvwCz2vlal1LnyB02D}RbN~Eq!?S>ZGym$W6J|gCijc@Hi z2P-o0ZV&9@HB^=#zP9t3auGkxII8-1CEKUhp_ZFZINcgloI(*RpWfaq_68#7O4*la z|HU!GCt5@WpJ-7e^04Z+;h>)-B-$pk@NU8`e`Pl_NJqDMiWi{U_ry1%+_DiU)DUc7 z@09E`?iL`oK$jyYqu+`fnmR)r(D*mL9SB>Tw`gbFFvvL-PB_`Pg8UKO7H#VMJ=+TB zeT~Zd^Ubp~N!NxMllJR;Q7tV-FXw?v_6|%#CV@jH_nlHZ%b<+bjs}0c7ZbP_H!wY;?A8gO~ ziT9)<%uP9@p|k}mn>~SZzy*!%^QEo7v^0=QEcYMtA(G|ooSdAFy#-6C+oS^?w;QRi z?LzhiAX#7O)9>TWs9v=GD$n2kMQ?arW)EbakTk1m%9$jtmJFn6kIc4eK2oG^E1~z% zxsH+gdOa#BLwyqXKr3Ye{0SF!h%V3$tiDky%5G|Jbo=Cbyh#n&5o-t%YnX?UIkqfj z*%>>`%q-tBPan8UHCGnFxk$0j^gsybMMo>*9dIOU8BKP^I9@-0n^`|o5Z7__cqmlt zW)~=?8I{VVWNUY^JI#jIym^=Cs5%*}Gkp{dd?I zl*b%q!td67Fb!LY^ULOTD5SWY2d1VOV*S_65>3zHo#Nt?g&CcvGpGG{=A%;Q2k+d} zp3D>T50Hx=7_jl~e7Gc1g(-M5A~fP6RxsoAQq0X5{x%n_2tw+3ywa_b94SHL1Yn80 z2SVt2OkkmQkC%1Qv)1iT5AZS_jVeM!x6~iGxfbOnso5qa?n1b4`(nLx=0^HU>t z)O9nbY)L5kE0@@Uqr~dx_ZNDln1{9X^s;SlAxxbl>G6G-h`2l+rS7mqEtH3p?ztBz zIt-jZ|D3cM^tplJ4=s&394;xE{(HF-pHpqYMUPpOTML}y)Jk_!l^;mn=VOgx#|k3-v+E_Ya*FZlgN%s_SZ> z-<2BHrZ9QFyRX<%x(#xf*cUZ@6!}tP<(>=XX9*^OqCARxcrISDFD8_{qJ_KleRSFJ zv#Vf*({rW>L0{8_!kf-Tr}@3nu&X2^>IBMVCU)UUQu_%}O`7j^xRCCT=^jKO~2 z&q*)0ygs7$vlcG=r>d=C)b8YwlNUoKTJnFyfyY#qyd13qQHW_2*wcu|lB6U(pY765 z$!Vp;(bDX%9t>o;3|Sos(@^_a=rR|u_QZn7-@mq$QdJabN4e*_=EIiVV~JX?_*d=> z6-4u8Y6f0n8uC9tB|fHJX!t|(#{6b8;_3auP*h$~kX2_W!3pVLEu+i2f7&{BZ)<;- z7`anfSC;9!oAj(1SBw*xA#OmH)Zf@95Mk)0sdEmr}2MEH;||T_q*BQn4UObIRt} zKJBit`1`IsQr_MCWb_krMjA)9X%B%+aYVF&oirVO`0jn{Dv&i{EjE+V>^&|LBZEk0#qm&B}KfZ35)&RumMB>E-=xy(sA+3jtnt2dJIv?#WI zqfSmH!|oa9;29nx*;Ij@EQ5onuvb_%hF6fMcKHLRPrL#tWGzU0+x+%S0sr8eJdOro z8H4J93Io(2JBJz)Q8Tjn%(x$r^e;$HoDu7DW%WH?J@x3HuZqHIB7(}m{=Ghc3pyLq z`IbFC@uSIqzaz>6Nb`Tc6MoOAN7)T_NB-vtt#j${ljr3ByXcS?t%Y_DeH-?e{AWXk sum%+7%B=nSqi_Ln5t#XZxXrLFKKJ26nCXO^NDg?I8e15Z9y}ZVC+fN$uK)l5 literal 0 HcmV?d00001 diff --git a/cachelib/allocator/BackgroundEvictor-inl.h b/cachelib/allocator/BackgroundEvictor-inl.h new file mode 100644 index 0000000000..e5a4b25eb9 --- /dev/null +++ b/cachelib/allocator/BackgroundEvictor-inl.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +namespace facebook { +namespace cachelib { + + +template +BackgroundEvictor::BackgroundEvictor(Cache& cache, + std::shared_ptr strategy) + : cache_(cache), + strategy_(strategy) +{ +} + +template +BackgroundEvictor::~BackgroundEvictor() { stop(std::chrono::seconds(0)); } + +template +void BackgroundEvictor::work() { + try { + checkAndRun(); + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundEvictor interrupted due to exception: {}", ex.what()); + } +} + +template +void BackgroundEvictor::setAssignedMemory(std::vector> &&assignedMemory) +{ + XLOG(INFO, "Class assigned to background worker:"); + for (auto [tid, pid, cid] : assignedMemory) { + XLOGF(INFO, "Tid: {}, Pid: {}, Cid: {}", tid, pid, cid); + } + + mutex.lock_combine([this, &assignedMemory]{ + this->assignedMemory_ = std::move(assignedMemory); + }); +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundEvictor::checkAndRun() { + auto assignedMemory = mutex.lock_combine([this]{ + return assignedMemory_; + }); + + unsigned int evictions = 0; + std::set classes{}; + + for (const auto [tid, pid, cid] : assignedMemory) { + classes.insert(cid); + const auto& mpStats = cache_.getPoolByTid(pid,tid).getStats(); + + auto batch = strategy_->calculateBatchSize(cache_,tid,pid,cid); + if (!batch) { + continue; + } + + stats.evictionSize.add(batch * mpStats.acStats.at(cid).allocSize); + + //try evicting BATCH items from the class in order to reach free target + auto evicted = + BackgroundEvictorAPIWrapper::traverseAndEvictItems(cache_, + tid,pid,cid,batch); + evictions += evicted; + + //const size_t cid_id = (size_t)mpStats.acStats.at(cid).allocSize; + auto it = evictions_per_class_.find(cid); + if (it != evictions_per_class_.end()) { + it->second += evicted; + } else if (evicted > 0) { + evictions_per_class_[cid] = evicted; + } + } + + stats.numTraversals.inc(); + stats.numEvictedItems.add(evictions); + stats.totalClasses.add(classes.size()); +} + +template +BackgroundEvictionStats BackgroundEvictor::getStats() const noexcept { + BackgroundEvictionStats evicStats; + evicStats.numEvictedItems = stats.numEvictedItems.get(); + evicStats.runCount = stats.numTraversals.get(); + evicStats.evictionSize = stats.evictionSize.get(); + evicStats.totalClasses = stats.totalClasses.get(); + + return evicStats; +} + +template +std::map BackgroundEvictor::getClassStats() const noexcept { + return evictions_per_class_; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundEvictor.h b/cachelib/allocator/BackgroundEvictor.h new file mode 100644 index 0000000000..3b2886a3ae --- /dev/null +++ b/cachelib/allocator/BackgroundEvictor.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/PeriodicWorker.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" +#include "cachelib/common/AtomicCounter.h" + + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the eviction. +template +struct BackgroundEvictorAPIWrapper { + + static size_t traverseAndEvictItems(C& cache, + unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + return cache.traverseAndEvictItems(tid,pid,cid,batch); + } +}; + +struct BackgroundEvictorStats { + // items evicted + AtomicCounter numEvictedItems{0}; + + // traversals + AtomicCounter numTraversals{0}; + + // total class size + AtomicCounter totalClasses{0}; + + // item eviction size + AtomicCounter evictionSize{0}; +}; + +// Periodic worker that evicts items from tiers in batches +// The primary aim is to reduce insertion times for new items in the +// cache +template +class BackgroundEvictor : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param target_free the target amount of memory to keep free in + // this tier + // @param tier id memory tier to perform eviction on + BackgroundEvictor(Cache& cache, + std::shared_ptr strategy); + + ~BackgroundEvictor() override; + + BackgroundEvictionStats getStats() const noexcept; + std::map getClassStats() const noexcept; + + void setAssignedMemory(std::vector> &&assignedMemory); + + private: + std::map evictions_per_class_; + + // cache allocator's interface for evicting + + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(); + + BackgroundEvictorStats stats; + + std::vector> assignedMemory_; + folly::DistributedMutex mutex; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundEvictor-inl.h" diff --git a/cachelib/allocator/BackgroundEvictorStrategy.h b/cachelib/allocator/BackgroundEvictorStrategy.h new file mode 100644 index 0000000000..17accd93a0 --- /dev/null +++ b/cachelib/allocator/BackgroundEvictorStrategy.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +// Base class for background eviction strategy. +class BackgroundEvictorStrategy { + +public: + virtual size_t calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid) = 0; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundPromoter-inl.h b/cachelib/allocator/BackgroundPromoter-inl.h new file mode 100644 index 0000000000..b280a59e01 --- /dev/null +++ b/cachelib/allocator/BackgroundPromoter-inl.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +namespace facebook { +namespace cachelib { + + +template +BackgroundPromoter::BackgroundPromoter(Cache& cache, + std::shared_ptr strategy) + : cache_(cache), + strategy_(strategy) +{ +} + +template +BackgroundPromoter::~BackgroundPromoter() { stop(std::chrono::seconds(0)); } + +template +void BackgroundPromoter::work() { + try { + checkAndRun(); + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundPromoter interrupted due to exception: {}", ex.what()); + } +} + +template +void BackgroundPromoter::setAssignedMemory(std::vector> &&assignedMemory) +{ + XLOG(INFO, "Class assigned to background worker:"); + for (auto [tid, pid, cid] : assignedMemory) { + XLOGF(INFO, "Tid: {}, Pid: {}, Cid: {}", tid, pid, cid); + } + + mutex.lock_combine([this, &assignedMemory]{ + this->assignedMemory_ = std::move(assignedMemory); + }); +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundPromoter::checkAndRun() { + auto assignedMemory = mutex.lock_combine([this]{ + return assignedMemory_; + }); + + unsigned int promotions = 0; + std::set classes{}; + + for (const auto [tid, pid, cid] : assignedMemory) { + classes.insert(cid); + const auto& mpStats = cache_.getPoolByTid(pid,tid).getStats(); + auto batch = strategy_->calculateBatchSize(cache_,tid,pid,cid); + if (!batch) { + continue; + } + + // stats.promotionsize.add(batch * mpStats.acStats.at(cid).allocSize); + + //try evicting BATCH items from the class in order to reach free target + auto promoted = + BackgroundPromoterAPIWrapper::traverseAndPromoteItems(cache_, + tid,pid,cid,batch); + promotions += promoted; + + //const size_t cid_id = (size_t)mpStats.acStats.at(cid).allocSize; + auto it = promotions_per_class_.find(cid); + if (it != promotions_per_class_.end()) { + it->second += promoted; + } else if (promoted > 0) { + promotions_per_class_[cid] = promoted; + } + } + + stats.numTraversals.inc(); + stats.numPromotedItems.add(promotions); + // stats.totalClasses.add(classes.size()); +} + +template + BackgroundPromotionStats BackgroundPromoter::getStats() const noexcept { + BackgroundPromotionStats promoStats; + promoStats.numPromotedItems = stats.numPromotedItems.get(); + promoStats.runCount = stats.numTraversals.get(); + + return promoStats; + } + + template + std::map BackgroundPromoter::getClassStats() const noexcept { + return promotions_per_class_; + } + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundPromoter.h b/cachelib/allocator/BackgroundPromoter.h new file mode 100644 index 0000000000..93f592586e --- /dev/null +++ b/cachelib/allocator/BackgroundPromoter.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/PeriodicWorker.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" +#include "cachelib/common/AtomicCounter.h" + + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the promotion. +template +struct BackgroundPromoterAPIWrapper { + + static size_t traverseAndPromoteItems(C& cache, + unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + return cache.traverseAndPromoteItems(tid,pid,cid,batch); + } +}; + +struct BackgroundPromoterStats { + // items evicted + AtomicCounter numPromotedItems{0}; + + // traversals + AtomicCounter numTraversals{0}; + + // total class size + AtomicCounter totalClasses{0}; + + // item eviction size + AtomicCounter promotionSize{0}; +}; + +template +class BackgroundPromoter : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param target_free the target amount of memory to keep free in + // this tier + // @param tier id memory tier to perform promotin from + BackgroundPromoter(Cache& cache, + std::shared_ptr strategy); + // TODO: use separate strategy for eviction and promotion + + ~BackgroundPromoter() override; + + // TODO + BackgroundPromotionStats getStats() const noexcept; + std::map getClassStats() const noexcept; + + void setAssignedMemory(std::vector> &&assignedMemory); + + private: + std::map promotions_per_class_; + + // cache allocator's interface for evicting + + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(); + + BackgroundPromoterStats stats; + + std::vector> assignedMemory_; + folly::DistributedMutex mutex; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundPromoter-inl.h" diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index b00302086b..8dc0166ecf 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -35,6 +35,7 @@ add_library (cachelib_allocator CCacheManager.cpp ContainerTypes.cpp FreeMemStrategy.cpp + FreeThresholdStrategy.cpp HitsPerSlabStrategy.cpp LruTailAgeStrategy.cpp MarginalHitsOptimizeStrategy.cpp diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index ffbff0289e..5c7b463f62 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -93,6 +93,12 @@ class CacheBase { // // @param poolId The pool id to query virtual const MemoryPool& getPool(PoolId poolId) const = 0; + + // Get the reference to a memory pool using a tier id, for stats purposes + // + // @param poolId The pool id to query + // @param tierId The tier of the pool id + virtual const MemoryPool& getPoolByTid(PoolId poolId, TierId tid) const = 0; // Get Pool specific stats (regular pools). This includes stats from the // Memory Pool and also the cache. diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 59f8b1cc43..24278f3691 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -340,6 +340,18 @@ void CacheAllocator::initWorkers() { config_.poolOptimizeStrategy, config_.ccacheOptimizeStepSizePercent); } + + if (config_.backgroundEvictorEnabled()) { + startNewBackgroundEvictor(config_.backgroundEvictorInterval, + config_.backgroundEvictorStrategy, + config_.backgroundEvictorThreads); + } + + if (config_.backgroundPromoterEnabled()) { + startNewBackgroundPromoter(config_.backgroundPromoterInterval, + config_.backgroundPromoterStrategy, + config_.backgroundPromoterThreads); + } } template @@ -362,7 +374,24 @@ CacheAllocator::allocate(PoolId poolId, creationTime = util::getCurrentTimeSec(); } return allocateInternal(poolId, key, size, creationTime, - ttlSecs == 0 ? 0 : creationTime + ttlSecs); + ttlSecs == 0 ? 0 : creationTime + ttlSecs, false); +} + +template +bool CacheAllocator::shouldWakeupBgEvictor(TierId tid, PoolId pid, ClassId cid) +{ + // TODO: should we also work on lower tiers? should we have separate set of params? + if (tid == 1) return false; + return getAllocationClassStats(tid, pid, cid).approxFreePercent <= config_.lowEvictionAcWatermark; +} + +template +size_t CacheAllocator::backgroundWorkerId(TierId tid, PoolId pid, ClassId cid, size_t numWorkers) +{ + XDCHECK(numWorkers); + + // TODO: came up with some better sharding (use some hashing) + return (tid + pid + cid) % numWorkers; } template @@ -372,7 +401,8 @@ CacheAllocator::allocateInternalTier(TierId tid, typename Item::Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime) { + uint32_t expiryTime, + bool fromEvictorThread) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -387,13 +417,18 @@ CacheAllocator::allocateInternalTier(TierId tid, (*stats_.allocAttempts)[pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); + + if (backgroundEvictor_.size() && !fromEvictorThread && (memory == nullptr || shouldWakeupBgEvictor(tid, pid, cid))) { + backgroundEvictor_[backgroundWorkerId(tid, pid, cid, backgroundEvictor_.size())]->wakeUp(); + } + // TODO: Today disableEviction means do not evict from memory (DRAM). // Should we support eviction between memory tiers (e.g. from DRAM to PMEM)? if (memory == nullptr && !config_.disableEviction) { memory = findEviction(tid, pid, cid); } - ItemHandle handle; + WriteHandle handle; if (memory != nullptr) { // At this point, we have a valid memory allocation that is ready for use. // Ensure that when we abort from here under any circumstances, we free up @@ -430,18 +465,71 @@ CacheAllocator::allocateInternalTier(TierId tid, } template -typename CacheAllocator::WriteHandle -CacheAllocator::allocateInternal(PoolId pid, +TierId +CacheAllocator::getTargetTierForItem(PoolId pid, typename Item::Key key, uint32_t size, uint32_t creationTime, uint32_t expiryTime) { - auto tid = 0; /* TODO: consult admission policy */ - for(TierId tid = 0; tid < numTiers_; ++tid) { - auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime); - if (handle) return handle; + if (numTiers_ == 1) + return 0; + + if (config_.forceAllocationTier != UINT64_MAX) { + return config_.forceAllocationTier; } - return {}; + + const TierId defaultTargetTier = 0; + + const auto requiredSize = Item::getRequiredSize(key, size); + const auto cid = allocator_[defaultTargetTier]->getAllocationClassId(pid, requiredSize); + + auto freePercentage = getAllocationClassStats(defaultTargetTier, pid, cid).approxFreePercent; + + // TODO: COULD we implement BG worker which would move slabs around + // so that there is similar amount of free space in each pool/ac. + // Should this be responsibility of BG evictor? + + if (freePercentage >= config_.maxAcAllocationWatermark) + return defaultTargetTier; + + if (freePercentage <= config_.minAcAllocationWatermark) + return defaultTargetTier + 1; + + // TODO: we can even think about creating different allocation classes for PMEM + // and we could look at possible fragmentation when deciding where to put the item + if (config_.sizeThresholdPolicy) + return requiredSize < config_.sizeThresholdPolicy ? defaultTargetTier : defaultTargetTier + 1; + + // TODO: (e.g. always put chained items to PMEM) + // if (chainedItemsPolicy) + // return item.isChainedItem() ? defaultTargetTier + 1 : defaultTargetTier; + + // TODO: + // if (expiryTimePolicy) + // return (expiryTime - creationTime) < expiryTimePolicy ? defaultTargetTier : defaultTargetTier + 1; + + // TODO: + // if (keyPolicy) // this can be based on key length or some other properties + // return getTargetTierForKey(key); + + // TODO: + // if (compressabilityPolicy) // if compresses well store in PMEM? latency will be higher anyway + // return TODO; + + // TODO: only works for 2 tiers + return (folly::Random::rand32() % 100) < config_.defaultTierChancePercentage ? defaultTargetTier : defaultTargetTier + 1; +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::allocateInternal(PoolId pid, + typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime, + bool fromEvictorThread) { + auto tid = getTargetTierForItem(pid, key, size, creationTime, expiryTime); + return allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromEvictorThread); } template @@ -1398,6 +1486,63 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, return true; } +template +bool CacheAllocator::moveRegularItemForPromotion(Item& oldItem, + ItemHandle& newItemHdl) { + XDCHECK(config_.moveCb); + util::LatencyTracker tracker{stats_.moveRegularLatency_}; + + if (!oldItem.isAccessible() || oldItem.isExpired()) { + return false; + } + + XDCHECK_EQ(newItemHdl->getSize(), oldItem.getSize()); + + if (config_.moveCb) { + // Execute the move callback. We cannot make any guarantees about the + // consistency of the old item beyond this point, because the callback can + // do more than a simple memcpy() e.g. update external references. If there + // are any remaining handles to the old item, it is the caller's + // responsibility to invalidate them. The move can only fail after this + // statement if the old item has been removed or replaced, in which case it + // should be fine for it to be left in an inconsistent state. + config_.moveCb(oldItem, *newItemHdl, nullptr); + } else { + std::memcpy(newItemHdl->getWritableMemory(), oldItem.getMemory(), + oldItem.getSize()); + } + + auto predicate = [this](const Item& item) { + // if inclusive cache is allowed, replace even if there are active users. + return config_.numDuplicateElements > 0 || item.getRefCount() == 0; + }; + if (!accessContainer_->replaceIf(oldItem, *newItemHdl, predicate)) { + return false; + } + + // Inside the MM container's lock, this checks if the old item exists to + // make sure that no other thread removed it, and only then replaces it. + if (!replaceInMMContainer(oldItem, *newItemHdl)) { + accessContainer_->remove(*newItemHdl); + return false; + } + + // Replacing into the MM container was successful, but someone could have + // called insertOrReplace() or remove() before or after the + // replaceInMMContainer() operation, which would invalidate newItemHdl. + if (!newItemHdl->isAccessible()) { + removeFromMMContainer(*newItemHdl); + return false; + } + + // no one can add or remove chained items at this point + if (oldItem.hasChainedItem()) { + throw std::runtime_error("Not supported"); + } + newItemHdl.unmarkNascent(); + return true; +} + template bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, ItemHandle& newItemHdl) { @@ -1605,21 +1750,35 @@ bool CacheAllocator::shouldWriteToNvmCacheExclusive( return true; } +template +bool CacheAllocator::shouldEvictToNextMemoryTier( + TierId sourceTierId, TierId targetTierId, PoolId pid, Item& item) +{ + if (config_.disableEvictionToMemory) + return false; + + // TODO: implement more advanced admission policies for memory tiers + return true; +} + template typename CacheAllocator::WriteHandle CacheAllocator::tryEvictToNextMemoryTier( - TierId tid, PoolId pid, Item& item) { - if(item.isChainedItem()) return {}; // TODO: We do not support ChainedItem yet + TierId tid, PoolId pid, Item& item, bool fromEvictorThread) { if(item.isExpired()) return acquire(&item); - TierId nextTier = tid; // TODO - calculate this based on some admission policy + TierId nextTier = tid; while (++nextTier < numTiers_) { // try to evict down to the next memory tiers + if (!shouldEvictToNextMemoryTier(tid, nextTier, pid, item)) + continue; + // allocateInternal might trigger another eviction auto newItemHdl = allocateInternalTier(nextTier, pid, item.getKey(), item.getSize(), item.getCreationTime(), - item.getExpiryTime()); + item.getExpiryTime(), + fromEvictorThread); if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); @@ -1631,12 +1790,48 @@ CacheAllocator::tryEvictToNextMemoryTier( return {}; } +template +bool +CacheAllocator::tryPromoteToNextMemoryTier( + TierId tid, PoolId pid, Item& item, bool fromEvictorThread) { + TierId nextTier = tid; + while (nextTier > 0) { // try to evict down to the next memory tiers + auto toPromoteTier = nextTier - 1; + --nextTier; + + // allocateInternal might trigger another eviction + auto newItemHdl = allocateInternalTier(toPromoteTier, pid, + item.getKey(), + item.getSize(), + item.getCreationTime(), + item.getExpiryTime(), + fromEvictorThread); + + if (newItemHdl) { + XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); + if (moveRegularItemForPromotion(item, newItemHdl)) { + return true; + } + } + } + + return false; +} + template typename CacheAllocator::WriteHandle -CacheAllocator::tryEvictToNextMemoryTier(Item& item) { +CacheAllocator::tryEvictToNextMemoryTier(Item& item, bool fromEvictorThread) { + auto tid = getTierId(item); + auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; + return tryEvictToNextMemoryTier(tid, pid, item, fromEvictorThread); +} + +template +bool +CacheAllocator::tryPromoteToNextMemoryTier(Item& item, bool fromBgThread) { auto tid = getTierId(item); auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; - return tryEvictToNextMemoryTier(tid, pid, item); + return tryPromoteToNextMemoryTier(tid, pid, item, fromBgThread); } template @@ -2294,6 +2489,16 @@ PoolId CacheAllocator::addPool( setRebalanceStrategy(pid, std::move(rebalanceStrategy)); setResizeStrategy(pid, std::move(resizeStrategy)); + if (backgroundEvictor_.size()) { + for (size_t id = 0; id < backgroundEvictor_.size(); id++) + backgroundEvictor_[id]->setAssignedMemory(getAssignedMemoryToBgWorker(id, backgroundEvictor_.size(), 0)); + } + + if (backgroundPromoter_.size()) { + for (size_t id = 0; id < backgroundPromoter_.size(); id++) + backgroundPromoter_[id]->setAssignedMemory(getAssignedMemoryToBgWorker(id, backgroundPromoter_.size(), 1)); + } + return pid; } @@ -2358,6 +2563,10 @@ void CacheAllocator::createMMContainers(const PoolId pid, .getAllocsPerSlab() : 0); for (TierId tid = 0; tid < numTiers_; tid++) { + if constexpr (std::is_same_v || std::is_same_v) { + config.lruInsertionPointSpec = config_.memoryTierConfigs[tid].lruInsertionPointSpec ; + config.markUsefulChance = config_.memoryTierConfigs[tid].markUsefulChance; + } mmContainers_[tid][pid][cid].reset(new MMContainer(config, compressor_)); } } @@ -2412,7 +2621,7 @@ std::set CacheAllocator::getRegularPoolIds() const { folly::SharedMutex::ReadHolder r(poolsResizeAndRebalanceLock_); // TODO - get rid of the duplication - right now, each tier // holds pool objects with mostly the same info - return filterCompactCachePools(allocator_[0]->getPoolIds()); + return filterCompactCachePools(allocator_[currentTier()]->getPoolIds()); } template @@ -2824,7 +3033,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { oldItem.getKey(), oldItem.getSize(), oldItem.getCreationTime(), - oldItem.getExpiryTime()); + oldItem.getExpiryTime(), + false); if (!newItemHdl) { return {}; } @@ -2957,14 +3167,14 @@ void CacheAllocator::evictForSlabRelease( template typename CacheAllocator::ItemHandle CacheAllocator::evictNormalItem(Item& item, - bool skipIfTokenInvalid) { + bool skipIfTokenInvalid, bool fromEvictorThread) { XDCHECK(item.isMoving()); if (item.isOnlyMoving()) { return ItemHandle{}; } - auto evictHandle = tryEvictToNextMemoryTier(item); + auto evictHandle = tryEvictToNextMemoryTier(item, fromEvictorThread); if(evictHandle) return evictHandle; auto predicate = [](const Item& it) { return it.getRefCount() == 0; }; @@ -3349,6 +3559,8 @@ bool CacheAllocator::stopWorkers(std::chrono::seconds timeout) { success &= stopPoolResizer(timeout); success &= stopMemMonitor(timeout); success &= stopReaper(timeout); + success &= stopBackgroundEvictor(timeout); + success &= stopBackgroundPromoter(timeout); return success; } @@ -3629,6 +3841,8 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false; ret.nvmUpTime = currTime - getNVMCacheCreationTime(); ret.reaperStats = getReaperStats(); + ret.evictionStats = getBackgroundEvictorStats(); + ret.promotionStats = getBackgroundPromoterStats(); ret.numActiveHandles = getNumActiveHandles(); return ret; @@ -3732,6 +3946,7 @@ bool CacheAllocator::startNewPoolRebalancer( freeAllocThreshold); } + template bool CacheAllocator::startNewPoolResizer( std::chrono::milliseconds interval, @@ -3769,6 +3984,64 @@ bool CacheAllocator::startNewReaper( return startNewWorker("Reaper", reaper_, interval, reaperThrottleConfig); } +template +auto CacheAllocator::getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers, TierId tid) +{ + std::vector> asssignedMemory; + // TODO: for now, only evict from tier 0 + auto pools = filterCompactCachePools(allocator_[tid]->getPoolIds()); + for (const auto pid : pools) { + const auto& mpStats = getPoolByTid(pid,tid).getStats(); + for (const auto cid : mpStats.classIds) { + if (backgroundWorkerId(tid, pid, cid, numWorkers) == evictorId) { + asssignedMemory.emplace_back(tid, pid, cid); + } + } + } + return asssignedMemory; +} + +template +bool CacheAllocator::startNewBackgroundEvictor( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + backgroundEvictor_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i], interval, strategy); + result = result && ret; + + if (result) { + backgroundEvictor_[i]->setAssignedMemory(getAssignedMemoryToBgWorker(i, backgroundEvictor_.size(), 0)); + } + } + return result; +} + +template +bool CacheAllocator::startNewBackgroundPromoter( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + XDCHECK(numTiers_ > 1); + backgroundPromoter_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundPromoter" + std::to_string(i), backgroundPromoter_[i], interval, strategy); + result = result && ret; + + if (result) { + backgroundPromoter_[i]->setAssignedMemory(getAssignedMemoryToBgWorker(i, backgroundPromoter_.size(), 1)); + } + } + return result; +} + template bool CacheAllocator::stopPoolRebalancer( std::chrono::seconds timeout) { @@ -3796,6 +4069,28 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { return stopWorker("Reaper", reaper_, timeout); } +template +bool CacheAllocator::stopBackgroundEvictor( + std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundEvictor_.size(); i++) { + auto ret = stopWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i], timeout); + result = result && ret; + } + return result; +} + +template +bool CacheAllocator::stopBackgroundPromoter( + std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundPromoter_.size(); i++) { + auto ret = stopWorker("BackgroundPromoter" + std::to_string(i), backgroundPromoter_[i], timeout); + result = result && ret; + } + return result; +} + template bool CacheAllocator::cleanupStrayShmSegments( const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector& config */) { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 81ce90d189..5605b3bbf1 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -36,7 +36,8 @@ #include #include #pragma GCC diagnostic pop - +#include "cachelib/allocator/BackgroundEvictor.h" +#include "cachelib/allocator/BackgroundPromoter.h" #include "cachelib/allocator/CCacheManager.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/CacheAllocatorConfig.h" @@ -695,6 +696,8 @@ class CacheAllocator : public CacheBase { std::shared_ptr resizeStrategy = nullptr, bool ensureProvisionable = false); + auto getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers, TierId tid); + // update an existing pool's config // // @param pid pool id for the pool to be updated @@ -945,6 +948,11 @@ class CacheAllocator : public CacheBase { // @param reaperThrottleConfig throttling config bool startNewReaper(std::chrono::milliseconds interval, util::Throttler::Config reaperThrottleConfig); + + bool startNewBackgroundEvictor(std::chrono::milliseconds interval, + std::shared_ptr strategy, size_t threads); + bool startNewBackgroundPromoter(std::chrono::milliseconds interval, + std::shared_ptr strategy, size_t threads); // Stop existing workers with a timeout bool stopPoolRebalancer(std::chrono::seconds timeout = std::chrono::seconds{ @@ -954,6 +962,8 @@ class CacheAllocator : public CacheBase { 0}); bool stopMemMonitor(std::chrono::seconds timeout = std::chrono::seconds{0}); bool stopReaper(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundEvictor(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundPromoter(std::chrono::seconds timeout = std::chrono::seconds{0}); // Set pool optimization to either true or false // @@ -988,6 +998,10 @@ class CacheAllocator : public CacheBase { const MemoryPool& getPool(PoolId pid) const override final { return allocator_[currentTier()]->getPool(pid); } + + const MemoryPool& getPoolByTid(PoolId pid, TierId tid) const override final { + return allocator_[tid]->getPool(pid); + } // calculate the number of slabs to be advised/reclaimed in each pool PoolAdviseReclaimData calcNumSlabsToAdviseReclaim() override final { @@ -1034,6 +1048,59 @@ class CacheAllocator : public CacheBase { auto stats = reaper_ ? reaper_->getStats() : ReaperStats{}; return stats; } + + // returns the background evictor + BackgroundEvictionStats getBackgroundEvictorStats() const { + auto stats = BackgroundEvictionStats{}; + for (auto &bg : backgroundEvictor_) + stats += bg->getStats(); + return stats; + } + + BackgroundPromotionStats getBackgroundPromoterStats() const { + auto stats = BackgroundPromotionStats{}; + for (auto &bg : backgroundPromoter_) + stats += bg->getStats(); + return stats; + } + + std::map getBackgroundEvictorClassStats() const { + auto stats = std::map(); + + for (auto &bg : backgroundEvictor_) { + for (const auto entry : bg->getClassStats()) { + auto cid = entry.first; + auto count = entry.second; + auto it = stats.find(cid); + if ( it != stats.end() ) { + it->second += count; + } else { + stats[cid] = count; + } + } + } + + return stats; + } + + std::map getBackgroundPromoterClassStats() const { + auto stats = std::map(); + + for (auto &bg : backgroundPromoter_) { + for (const auto entry : bg->getClassStats()) { + auto cid = entry.first; + auto count = entry.second; + auto it = stats.find(cid); + if ( it != stats.end() ) { + it->second += count; + } else { + stats[cid] = count; + } + } + } + + return stats; + } // return the LruType of an item typename MMType::LruType getItemLruType(const Item& item) const; @@ -1181,6 +1248,9 @@ class CacheAllocator : public CacheBase { // gives a relative offset to a pointer within the cache. uint64_t getItemPtrAsOffset(const void* ptr); + bool shouldWakeupBgEvictor(TierId tid, PoolId pid, ClassId cid); + size_t backgroundWorkerId(TierId tid, PoolId pid, ClassId cid, size_t numWorkers); + // this ensures that we dont introduce any more hidden fields like vtable by // inheriting from the Hooks and their bool interface. static_assert((sizeof(typename MMType::template Hook) + @@ -1222,6 +1292,11 @@ class CacheAllocator : public CacheBase { // allocator and executes the necessary callbacks. no-op if it is nullptr. FOLLY_ALWAYS_INLINE void release(Item* it, bool isNascent); + TierId getTargetTierForItem(PoolId pid, typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime); + // This is the last step in item release. We also use this for the eviction // scenario where we have to do everything, but not release the allocation // to the allocator and instead recycle it for another new allocation. If @@ -1326,7 +1401,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromEvictorThread); // create a new cache allocation on specific memory tier. // For description see allocateInternal. @@ -1337,7 +1413,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromEvictorThread); // Allocate a chained item // @@ -1423,6 +1500,7 @@ class CacheAllocator : public CacheBase { // @return true If the move was completed, and the containers were updated // successfully. bool moveRegularItem(Item& oldItem, ItemHandle& newItemHdl); + bool moveRegularItemForPromotion(Item& oldItem, ItemHandle& newItemHdl); // template class for viewAsChainedAllocs that takes either ReadHandle or // WriteHandle @@ -1577,7 +1655,8 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the item. This will be the last // handle to the item. On failure an empty handle. - WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item); + WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item, bool fromEvictorThread); + bool tryPromoteToNextMemoryTier(TierId tid, PoolId pid, Item& item, bool fromEvictorThread); // Try to move the item down to the next memory tier // @@ -1585,7 +1664,11 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the item. This will be the last // handle to the item. On failure an empty handle. - WriteHandle tryEvictToNextMemoryTier(Item& item); + WriteHandle tryEvictToNextMemoryTier(Item& item, bool fromEvictorThread); + bool tryPromoteToNextMemoryTier(Item& item, bool fromEvictorThread); + + bool shouldEvictToNextMemoryTier(TierId sourceTierId, + TierId targetTierId, PoolId pid, Item& item); size_t memoryTierSize(TierId tid) const; @@ -1714,7 +1797,7 @@ class CacheAllocator : public CacheBase { // // @return last handle for corresponding to item on success. empty handle on // failure. caller can retry if needed. - ItemHandle evictNormalItem(Item& item, bool skipIfTokenInvalid = false); + ItemHandle evictNormalItem(Item& item, bool skipIfTokenInvalid = false, bool fromEvictorThread = false); // Helper function to evict a child item for slab release // As a side effect, the parent item is also evicted @@ -1742,6 +1825,133 @@ class CacheAllocator : public CacheBase { folly::annotate_ignore_thread_sanitizer_guard g(__FILE__, __LINE__); allocator_[currentTier()]->forEachAllocation(std::forward(f)); } + + // exposed for the background evictor to iterate through the memory and evict + // in batch. This should improve insertion path for tiered memory config + size_t traverseAndEvictItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + auto& mmContainer = getMMContainer(tid, pid, cid); + size_t evictions = 0; + size_t evictionCandidates = 0; + std::vector candidates; + candidates.reserve(batch); + + size_t tries = 0; + mmContainer.withEvictionIterator([&tries, &candidates, &batch, this](auto &&itr){ + while (candidates.size() < batch && (config_.evictionHotnessThreshold == 0 || tries < config_.evictionHotnessThreshold) && itr) { + tries++; + Item* candidate = itr.get(); + XDCHECK(candidate); + + if (candidate->isChainedItem()) { + throw std::runtime_error("Not supported for chained items"); + } + + if (candidate->getRefCount() == 0 && candidate->markMoving()) { + candidates.push_back(candidate); + } + + ++itr; + } + }); + + for (Item *candidate : candidates) { + auto toReleaseHandle = + evictNormalItem(*candidate, true /* skipIfTokenInvalid */, true /* from BG thread */); + auto ref = candidate->unmarkMoving(); + + if (toReleaseHandle || ref == 0u) { + if (candidate->hasChainedItem()) { + (*stats_.chainedItemEvictions)[pid][cid].inc(); + } else { + (*stats_.regularItemEvictions)[pid][cid].inc(); + } + + evictions++; + } else { + if (candidate->hasChainedItem()) { + stats_.evictFailParentAC.inc(); + } else { + stats_.evictFailAC.inc(); + } + } + + if (toReleaseHandle) { + XDCHECK(toReleaseHandle.get() == candidate); + XDCHECK_EQ(1u, toReleaseHandle->getRefCount()); + + // We manually release the item here because we don't want to + // invoke the Item Handle's destructor which will be decrementing + // an already zero refcount, which will throw exception + auto& itemToRelease = *toReleaseHandle.release(); + + // Decrementing the refcount because we want to recycle the item + const auto ref = decRef(itemToRelease); + XDCHECK_EQ(0u, ref); + + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } else if (ref == 0u) { + // it's safe to recycle the item here as there are no more + // references and the item could not been marked as moving + // by other thread since it's detached from MMContainer. + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } + } + + return evictions; + } + + size_t traverseAndPromoteItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + auto& mmContainer = getMMContainer(tid, pid, cid); + size_t promotions = 0; + std::vector candidates; + candidates.reserve(batch); + + size_t tries = 0; + + mmContainer.withPromotionIterator([&tries, &candidates, &batch, this](auto &&itr){ + while (candidates.size() < batch && (config_.evictionHotnessThreshold == 0 || tries < config_.evictionHotnessThreshold) && itr) { + tries++; + Item* candidate = itr.get(); + XDCHECK(candidate); + + if (candidate->isChainedItem()) { + throw std::runtime_error("Not supported for chained items"); + } + + // if (candidate->getRefCount() == 0 && candidate->markMoving()) { + // candidates.push_back(candidate); + // } + + // TODO: only allow it for read-only items? + // or implement mvcc + if (!candidate->isExpired() && candidate->markMoving()) { + candidates.push_back(candidate); + } + + ++itr; + } + }); + + for (Item *candidate : candidates) { + auto promoted = tryPromoteToNextMemoryTier(*candidate, true); + auto ref = candidate->unmarkMoving(); + if (promoted) + promotions++; + + if (ref == 0u) { + // stats_.promotionMoveSuccess.inc(); + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } + } + + return promotions; + } // returns true if nvmcache is enabled and we should write this item to // nvmcache. @@ -2050,6 +2260,10 @@ class CacheAllocator : public CacheBase { // free memory monitor std::unique_ptr memMonitor_; + + // background evictor + std::vector>> backgroundEvictor_; + std::vector>> backgroundPromoter_; // check whether a pool is a slabs pool std::array isCompactCachePool_{}; @@ -2105,6 +2319,8 @@ class CacheAllocator : public CacheBase { // Make this friend to give access to acquire and release friend ReadHandle; friend ReaperAPIWrapper; + friend BackgroundEvictorAPIWrapper; + friend BackgroundPromoterAPIWrapper; friend class CacheAPIWrapperForNvm; friend class FbInternalRuntimeUpdateWrapper; diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index ca51deb94c..08dd1f381e 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -32,6 +32,7 @@ #include "cachelib/allocator/NvmAdmissionPolicy.h" #include "cachelib/allocator/PoolOptimizeStrategy.h" #include "cachelib/allocator/RebalanceStrategy.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" #include "cachelib/allocator/Util.h" #include "cachelib/common/EventInterface.h" #include "cachelib/common/Throttler.h" @@ -266,6 +267,16 @@ class CacheAllocatorConfig { std::chrono::seconds regularInterval, std::chrono::seconds ccacheInterval, uint32_t ccacheStepSizePercent); + + // Enable the background evictor - scans a tier to look for objects + // to evict to the next tier + CacheAllocatorConfig& enableBackgroundEvictor( + std::shared_ptr backgroundEvictorStrategy, + std::chrono::milliseconds regularInterval, size_t threads); + + CacheAllocatorConfig& enableBackgroundPromoter( + std::shared_ptr backgroundEvictorStrategy, + std::chrono::milliseconds regularInterval, size_t threads); // This enables an optimization for Pool rebalancing and resizing. // The rough idea is to ensure only the least useful items are evicted when @@ -337,6 +348,17 @@ class CacheAllocatorConfig { compactCacheOptimizeInterval.count() > 0) && poolOptimizeStrategy != nullptr; } + + // @return whether background evictor thread is enabled + bool backgroundEvictorEnabled() const noexcept { + return backgroundEvictorInterval.count() > 0 && + backgroundEvictorStrategy != nullptr; + } + + bool backgroundPromoterEnabled() const noexcept { + return backgroundPromoterInterval.count() > 0 && + backgroundPromoterStrategy != nullptr; + } // @return whether memory monitor is enabled bool memMonitoringEnabled() const noexcept { @@ -433,6 +455,13 @@ class CacheAllocatorConfig { // time interval to sleep between iterators of rebalancing the pools. std::chrono::milliseconds poolRebalanceInterval{std::chrono::seconds{1}}; + + // time interval to sleep between runs of the background evictor + std::chrono::milliseconds backgroundEvictorInterval{std::chrono::milliseconds{1000}}; + std::chrono::milliseconds backgroundPromoterInterval{std::chrono::milliseconds{1000}}; + + size_t backgroundEvictorThreads{1}; + size_t backgroundPromoterThreads{1}; // Free slabs pro-actively if the ratio of number of freeallocs to // the number of allocs per slab in a slab class is above this @@ -444,6 +473,10 @@ class CacheAllocatorConfig { // rebalance to avoid alloc fialures. std::shared_ptr defaultPoolRebalanceStrategy{ new RebalanceStrategy{}}; + + // rebalance to avoid alloc fialures. + std::shared_ptr backgroundEvictorStrategy; + std::shared_ptr backgroundPromoterStrategy; // time interval to sleep between iterations of pool size optimization, // for regular pools and compact caches @@ -585,6 +618,28 @@ class CacheAllocatorConfig { // skip promote children items in chained when parent fail to promote bool skipPromoteChildrenWhenParentFailed{false}; + bool disableEvictionToMemory{false}; + + double promotionAcWatermark{4.0}; + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + double minAcAllocationWatermark{0.0}; + double maxAcAllocationWatermark{0.0}; + uint64_t sizeThresholdPolicy{0}; + double defaultTierChancePercentage{50.0}; + // TODO: default could be based on ratio + + double numDuplicateElements{0.0}; // inclusivness of the cache + double syncPromotion{0.0}; // can promotion be done synchronously in user thread + + uint64_t evictorThreads{1}; + uint64_t promoterThreads{1}; + + uint64_t evictionHotnessThreshold{40}; + uint64_t promotionHotnessThreshold{10}; + + uint64_t forceAllocationTier{UINT64_MAX}; + friend CacheT; private: @@ -933,6 +988,26 @@ CacheAllocatorConfig& CacheAllocatorConfig::enablePoolRebalancing( return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundEvictor( + std::shared_ptr strategy, + std::chrono::milliseconds interval, size_t evictorThreads) { + backgroundEvictorStrategy = strategy; + backgroundEvictorInterval = interval; + backgroundEvictorThreads = evictorThreads; + return *this; +} + +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundPromoter( + std::shared_ptr strategy, + std::chrono::milliseconds interval, size_t promoterThreads) { + backgroundPromoterStrategy = strategy; + backgroundPromoterInterval = interval; + backgroundPromoterThreads = promoterThreads; + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::enablePoolResizing( std::shared_ptr resizeStrategy, diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index a24b13d35e..ccdf3b623b 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -296,6 +296,43 @@ struct ReaperStats { uint64_t avgTraversalTimeMs{0}; }; +// Eviction Stats +struct BackgroundEvictionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t numEvictedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t runCount{0}; + + // total number of classes + uint64_t totalClasses{0}; + + // eviction size + uint64_t evictionSize{0}; + + BackgroundEvictionStats& operator+=(const BackgroundEvictionStats& rhs) { + numEvictedItems += rhs.numEvictedItems; + runCount += rhs.runCount; + totalClasses += rhs.totalClasses; + evictionSize += rhs.evictionSize; + return *this; + } +}; + +struct BackgroundPromotionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t numPromotedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t runCount{0}; + + BackgroundPromotionStats& operator+=(const BackgroundPromotionStats& rhs) { + numPromotedItems += rhs.numPromotedItems; + runCount += rhs.runCount; + return *this; + } +}; + // CacheMetadata type to export struct CacheMetadata { // allocator_version @@ -316,6 +353,11 @@ struct Stats; // Stats that apply globally in cache and // the ones that are aggregated over all pools struct GlobalCacheStats { + // background eviction stats + BackgroundEvictionStats evictionStats; + + BackgroundPromotionStats promotionStats; + // number of calls to CacheAllocator::find uint64_t numCacheGets{0}; diff --git a/cachelib/allocator/FreeThresholdStrategy.cpp b/cachelib/allocator/FreeThresholdStrategy.cpp new file mode 100644 index 0000000000..5b73bfd5f3 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/FreeThresholdStrategy.h" + +#include + +namespace facebook { +namespace cachelib { + + + +FreeThresholdStrategy::FreeThresholdStrategy(double lowEvictionAcWatermark, double highEvictionAcWatermark, uint64_t evictionHotnessThreshold) + : lowEvictionAcWatermark(lowEvictionAcWatermark), highEvictionAcWatermark(highEvictionAcWatermark), evictionHotnessThreshold(evictionHotnessThreshold) {} + +size_t FreeThresholdStrategy::calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid) { + auto stats = cache.getAllocationClassStats(tid, pid, cid); + if (stats.approxFreePercent >= highEvictionAcWatermark) + return 0; + + auto toFreeMemPercent = highEvictionAcWatermark - stats.approxFreePercent; + auto toFreeItems = static_cast(toFreeMemPercent * stats.memorySize / stats.allocSize); + + return toFreeItems; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/FreeThresholdStrategy.h b/cachelib/allocator/FreeThresholdStrategy.h new file mode 100644 index 0000000000..b59a1cc974 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" + +namespace facebook { +namespace cachelib { + + +// Base class for background eviction strategy. +class FreeThresholdStrategy : public BackgroundEvictorStrategy { + +public: + FreeThresholdStrategy(double lowEvictionAcWatermark, double highEvictionAcWatermark, uint64_t evictionHotnessThreshold); + ~FreeThresholdStrategy() {} + + size_t calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid); +private: + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + uint64_t evictionHotnessThreshold{40}; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/MM2Q-inl.h b/cachelib/allocator/MM2Q-inl.h index e791d6c6c3..469a3b6a84 100644 --- a/cachelib/allocator/MM2Q-inl.h +++ b/cachelib/allocator/MM2Q-inl.h @@ -14,6 +14,8 @@ * limitations under the License. */ +#include + namespace facebook { namespace cachelib { @@ -104,6 +106,10 @@ bool MM2Q::Container::recordAccess(T& node, return false; } + // TODO: % 100 is not very accurate + if (config_.markUsefulChance < 100.0 && folly::Random::rand32() % 100 >= config_.markUsefulChance) + return false; + return lruMutex_->lock_combine(func); } return false; @@ -211,15 +217,32 @@ void MM2Q::Container::rebalance() noexcept { template T::*HookPtr> bool MM2Q::Container::add(T& node) noexcept { const auto currTime = static_cast