From e7446242d407b4e38d4d75af438bf1f73f1abe88 Mon Sep 17 00:00:00 2001 From: Finn Roblin Date: Mon, 15 Jul 2024 12:13:28 -0700 Subject: [PATCH 1/6] support k-nn nested field benchmark Signed-off-by: Finn Roblin --- osbenchmark/utils/dataset.py | 4 + osbenchmark/workload/params.py | 170 ++++++++++++++++++++---- small-nested-works.hdf5 | Bin 0 -> 135744 bytes tests/utils/dataset_helper.py | 35 +++++ tests/workload/params_test.py | 227 ++++++++++++++++++++++++++++++++- 5 files changed, 407 insertions(+), 29 deletions(-) create mode 100644 small-nested-works.hdf5 diff --git a/osbenchmark/utils/dataset.py b/osbenchmark/utils/dataset.py index d8ad2b74f..4786c22d2 100644 --- a/osbenchmark/utils/dataset.py +++ b/osbenchmark/utils/dataset.py @@ -24,6 +24,7 @@ class Context(Enum): INDEX = 1 QUERY = 2 NEIGHBORS = 3 + PARENTS = 4 class DataSet(ABC): @@ -140,6 +141,9 @@ def parse_context(context: Context) -> str: if context == Context.QUERY: return "test" + + if context == Context.PARENTS: + return "parents" # used in nested benchmarks to get the parent document id associated with each vector. raise Exception("Unsupported context") diff --git a/osbenchmark/workload/params.py b/osbenchmark/workload/params.py index 59ebe27d1..8f99ad518 100644 --- a/osbenchmark/workload/params.py +++ b/osbenchmark/workload/params.py @@ -13,7 +13,7 @@ # not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -33,7 +33,7 @@ import time from abc import ABC, abstractmethod from enum import Enum -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional import numpy as np @@ -884,6 +884,7 @@ class VectorDataSetPartitionParamSource(ParamSource): def __init__(self, workload, params, context: Context, **kwargs): super().__init__(workload, params, **kwargs) self.field_name: str = parse_string_parameter("field", params) + self.is_nested = "." in self.field_name # in base class because used for both bulk ingest and queries. self.context = context self.data_set_format = parse_string_parameter("data_set_format", params) self.data_set_path = parse_string_parameter("data_set_path", params, "") @@ -1155,6 +1156,16 @@ def _build_vector_search_query_body(self, vector, efficient_filter=None) -> dict query.update({ "filter": efficient_filter, }) + + if self.is_nested: + outer_field_name, _inner_field_name = self.field_name.split(".") + return { + "nested": { + "path": outer_field_name, + "query": {"knn": {self.field_name: query}}, + } + } + return { "knn": { self.field_name: query, @@ -1177,13 +1188,49 @@ class BulkVectorsFromDataSetParamSource(VectorDataSetPartitionParamSource): def __init__(self, workload, params, **kwargs): super().__init__(workload, params, Context.INDEX, **kwargs) self.bulk_size: int = parse_int_parameter("bulk_size", params) - self.retries: int = parse_int_parameter("retries", params, - self.DEFAULT_RETRIES) + self.retries: int = parse_int_parameter("retries", params, self.DEFAULT_RETRIES) self.index_name: str = parse_string_parameter("index", params) self.id_field_name: str = parse_string_parameter( - self.PARAMS_NAME_ID_FIELD_NAME, params, self.DEFAULT_ID_FIELD_NAME) + self.PARAMS_NAME_ID_FIELD_NAME, params, self.DEFAULT_ID_FIELD_NAME + ) + + self.action_buffer = None + self.num_nested_vectors = 10 + + self.parent_data_set_path = parse_string_parameter( + "parents_data_set_path", params, self.data_set_path + ) + + self.parent_data_set_format = self.data_set_format + + self.parent_data_set_corpus = self.data_set_corpus - def bulk_transform(self, partition: np.ndarray, action) -> List[Dict[str, Any]]: + self.logger = logging.getLogger(__name__) + + def partition( + self, partition_index, total_partitions, should_nest=False + ): # TODO should_nest is a hack. + partition = super().partition(partition_index, total_partitions) + if self.parent_data_set_corpus and not self.parent_data_set_path: + parent_data_set_path = self._get_corpora_file_paths( + self.parent_data_set_corpus, self.parent_data_set_format + ) + self._validate_data_set_corpus(parent_data_set_path) + self.parent_data_set_path = parent_data_set_path[0] + if not self.parent_data_set_path: + self.parent_data_set_path = self.data_set_path + # add neighbor instance to partition + if self.is_nested: + partition.parent_data_set = get_data_set( + self.parent_data_set_format, self.parent_data_set_path, Context.PARENTS + ) + partition.parent_data_set.seek(partition.offset) + + return partition + + def bulk_transform( + self, partition: np.ndarray, action, parents_ids: Optional[np.ndarray] + ) -> List[Dict[str, Any]]: """Partitions and transforms a list of vectors into OpenSearch's bulk injection format. Args: @@ -1194,18 +1241,81 @@ def bulk_transform(self, partition: np.ndarray, action) -> List[Dict[str, Any]]: An array of transformed vectors in bulk format. """ actions = [] - _ = [ - actions.extend([action(self.id_field_name, i + self.current), None]) - for i in range(len(partition)) - ] - bulk_contents = [] + + if not self.is_nested: + _ = [ + actions.extend([action(self.id_field_name, i + self.current), None]) + for i in range(len(partition)) + ] + bulk_contents = [] + + add_id_field_to_body = self.id_field_name != self.DEFAULT_ID_FIELD_NAME + for vec, identifier in zip( + partition.tolist(), range(self.current, self.current + len(partition)) + ): + row = {self.field_name: vec} + if add_id_field_to_body: + row.update({self.id_field_name: identifier}) + bulk_contents.append(row) + + actions[1::2] = bulk_contents + + self.logger.info("Actions: %s", actions) + return actions + + self.logger.debug("Bulk transform called with a nested field.") + + outer_field_name, inner_field_name = self.field_name.split(".") + add_id_field_to_body = self.id_field_name != self.DEFAULT_ID_FIELD_NAME - for vec, identifier in zip(partition.tolist(), range(self.current, self.current + len(partition))): - row = {self.field_name: vec} + + if self.action_buffer is None: + first_index_of_parent_ids = 0 + self.action_buffer = {outer_field_name: []} + self.action_parent_id = parents_ids[first_index_of_parent_ids] if add_id_field_to_body: - row.update({self.id_field_name: identifier}) - bulk_contents.append(row) - actions[1::2] = bulk_contents + self.action_buffer.update({self.id_field_name: self.action_parent_id}) + + part_list = partition.tolist() + for i in range(len(partition)): + + nested = {inner_field_name: part_list[i]} + + current_parent_id = parents_ids[i] + + if self.action_parent_id == current_parent_id: + self.action_buffer[outer_field_name].append(nested) + else: + # flush action buffer + actions.extend( + [ + action(self.id_field_name, self.action_parent_id), + self.action_buffer, + ] + ) + + self.current += len(self.action_buffer[outer_field_name]) + + self.action_buffer = {outer_field_name: []} + if add_id_field_to_body: + + self.action_buffer.update({self.id_field_name: current_parent_id}) + + self.action_buffer[outer_field_name].append(nested) + + self.action_parent_id = current_parent_id + + max_position = self.offset + self.num_vectors + if ( + self.current + len(self.action_buffer[outer_field_name]) + self.bulk_size + >= max_position + ): + # final flush of remaining vectors in the last partition (for the last client) + self.current += len(self.action_buffer[outer_field_name]) + actions.extend( + [action(self.id_field_name, self.action_parent_id), self.action_buffer] + ) + return actions def params(self): @@ -1217,29 +1327,33 @@ def params(self): def action(id_field_name, doc_id): # support only index operation - bulk_action = 'index' - metadata = { - '_index': self.index_name - } + bulk_action = "index" + metadata = {"_index": self.index_name} # Add id field to metadata only if it is _id if id_field_name == self.DEFAULT_ID_FIELD_NAME: metadata.update({id_field_name: doc_id}) return {bulk_action: metadata} remaining_vectors_in_partition = self.num_vectors + self.offset - self.current - # update bulk size if number of vectors to read is less than actual bulk size + bulk_size = min(self.bulk_size, remaining_vectors_in_partition) + partition = self.data_set.read(bulk_size) - body = self.bulk_transform(partition, action) + + if self.is_nested: + parent_ids = self.parent_data_set.read(bulk_size) + else: + parent_ids = None + + body = self.bulk_transform(partition, action, parent_ids) size = len(body) // 2 - self.current += size + + if not self.is_nested: + # in the nested case, we may have irregular number of vectors ingested, so we calculate self.current within bulk_transform method when self.is_nested. + self.current += size self.percent_completed = self.current / self.total - return { - "body": body, - "retries": self.retries, - "size": size - } + return {"body": body, "retries": self.retries, "size": size} def get_target(workload, params): diff --git a/small-nested-works.hdf5 b/small-nested-works.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..00b07c0d0b4f31dbee9e66936ad97b0c307759c0 GIT binary patch literal 135744 zcmeFaUx;1Vz2~>>tC>-5G?_DrU_@xhdC-FbdN>by&_hFhqv!@bh~$v;gC6?f+@OIT zXokZb1@zF6g%F}*ggKbzQW#+tqUjW-F@@7H!Zc>nG)-ZerZ9~uOk*~UX$sSr!kCM= zTyFCDti4ug>vYMEzg3C<_`4<4{Kl`U@tvx7s@@yZyYH0$*L9a_{9o4o+j(#Nw}16l9l5Q4)3vNy zoExo|r)%v0um9hE0=NF^UoYL`e%)xhKPUfdyLlsV@8A6Uds_GY=CA+lzeE4^y?cN2 zZ~y!Mf$+$$|N3wK&A+?%@40ECT&en1>O0*(8FOpFce=6SJKr4xcWYs)dfWdCZoZlS za{iz3ZYlHbU!Cv%)&E%Rj_&6R|K0D74bRs%2{rtGA&c@h^|}w>#qCqM%Ud3``C$?aut6M-`EE9(RG77@fmudO^+tpbGTpD zx(we%?T6O3&R!SDF<%VC4~QKRI|lE7j#v}dd6!u0ZBD?SL$@C0tbo1c#SRGkOl|^Q z#f;cza69NAHb-06fi-vp=M374eeIV)kr>aU$G->$_HVd7*E;>*?F~3V8t**mJZw z+R8rqIp_liJqMwUxUUoZpHbspH`tfZ?jd2z=NS7vCN?up%%1ic;MA^puX1bjj93Q} z{BQBEfjxY+Tscqrtgz*U{SWbffnK76+775+Bi8bAhFjnna5iUr27ZX`In1zw8sqL`4if%J&2!)Ln&UlKf&rKlpa%o!DmHND zxt9A**w)MU;0iJI0Dgyk4wz$&dDiEe`vbTce&7pbh_3sbc#m$+xWg9u?%|*RmtFS> z`aeY5w@Hv=&jEcz+`J9Gl45*sqvkt{{JuP#7(fp^*Q-E3gZ5lI#PzJ(g`bMKa>iHa z!0#F`-S_X{ZUV)g<~UD}&R_}Wv%Ciq@YyfHRD!$@;5>&WJ+Z+)z;&kcnlr#(CjV{V zy*X3J@!q`-?BiT3bG7006I#E-&>56|=ttB&1dUfu=Wwn+$b*oJajupp{}?W#wJ%^l zLmvR|!+DUX$-uq^*vIhp72f%tn9qEuxgAt2EzV(cUDJ=!Do z2H3~$;S_c2bqKeBYjOqOnCm$IRE8MipTN04CC9k$9&I1>0GwH^FPzza3ETM2jq$tN zVJqdl;Ul{4CvYEwotl4wc0Y=Feh=5xcb(7iQ}s(ujCUaun;FM1{V^|->mD;Oek$1J ztSax>JqL5^Cz4}y0zD_!8H?bDHrNLP+@3!11e{qc!9ReG_t1ONB%xx9oohc&3RS56~NXU}vCxdWQI}aZId9ALBgc48|v%Ij_^t8qd<)ZwYZ{yk=go zF1}~nfu+5O_24|dXJg*`peNRWLvrrouh5?VE_wzMob_)I^L;o67Wkf>IVWHahT6xu zwdKn*s&;Q2+@s!gF9Z9pK#((l-w~T46BG7P%lWh6Z^0KJD&M1pYp;QO>+rP)v~|84 zJ$42Oxb`Drp6fog=j8pkjGm*dnSraoXX-PzW}x<@^6pLcQ3cV8b8O2wVd}RaJ`|%=~Qy;XM6{E2h?I-A%6e@Bw%m* z`RzRcz9a5C({~TfbC?5rcz>P%&m+OR7yEc7_R8QgeA|1N1MU+05jX-Xa_rfp)Z@GM z9&jz~p1LB~`_yz`Q919ZXVk^`eUeYu&gIPZG=B*eU;w2rI>>V$=fDndtqvr(1lIv$ zhrm7Wf+jO;>zv~zDElOT0Io9!89b1ye!wYe^UWFHF981@;9AaHgdES``EG;zz&mem z#d=}>bI=DtInZWt#vopNs8-Yb4$gjS^D6Qrzx3o=lW&aQv=up1vBvr4=mYex!BkR= zbE(ZyPsRCM-}nH0W*MzSa31)6>pFA1U&Mq`_5k<%#`r!Sk@FUI7cVf!AMgYHCy?zo zmB0bLy6!V+Bjv0DgZPqK^%ZmwYv)(KFU~;U71#&vsqkpm_kP?0=RvEZTak$Id%K3S zzkQmN_1Uw1yUIVX2Yw%D0Y4+=z5--op63DZ&aBbq$eG`P%fy{)msrLRaDsgd_Q4Vu zYht~5p6{I-<9Z3d2XkUKi8V2{sP%I0%fCNuu-kqzInLvG_-*lCAAyI!wVNdJI_uD` z;qw!oMNe!1%Dcv_1M=5E4t!vLV~22opN!knGr``eHU1nKp3ysw;;zn_kRgr z*i$&KzWvR47yB)+0(ZePVhIc&(>K5Zdk@Tjy;q=}4XclHQ2P{o2s-nqGuQQgPV8fF z4{Y#1Mcd1~M4on!b{_AYvcN9CPopDB==6_)*kkhk-dyZ$@%~-__PPejT9c#{qoa1; zo7yAr%}u~r-iDi_h3j}e;sE>#xc+6}StxyQe)#6DfX~95W9$J=$x+n!2l40d4?#~n zfW6+q&)ACoP5cfxll!vg9<~zUAN+s#J^D8j4yNS$jID7dpKlZMJ);9+o@W9BG5776 zXeabO$k>YW*sI)=x^MGN$g#h3Cj265U_Wbi;ZHde;Oy6<-+(^{j19&?1`B(l#pN60 z-@E6;cCcMjJHXHIz1Ob)8t|MlH36J|!0*w{@KfyTph-`T&q%#SuYi5MJNB7jXSgfa zuG8TM{<6jgZ2OC4)t-+%iopB3T7aXcH3K-o-y{AN=@oPCUQY_P3=pSXK){za)@!~^p4o?D(Hw6 zF%QW1#JpGbeh7A}y^qdtO`P4I+2%b3)|c}oXN7$XJb(Av5nE#4#P0|sm<1J$LJCO%LT+aT#fja{G;37B%yTlXx2JN~% zwr4N5#_#ZzH}U;WD?^NT;X{1;y5146&NX}ZHTE2P4}6VnKkGAGJ9}f|F>vkEv!snZ z?VW+WeJ6YLRG#C1pZHVY{8O=4j`18_V+U-@0N-Y?zO6sibL#F?IO9|Tzo&=!E9|@E z*=HcOM4#aAqLn$?pRIBk9N^!_U!yZP0X|QAD)yU!bHuFeKo2}m#Wm%8E^5#IGVndL zx3%7TXYg6uTfMB@2JQ3kY>f>u-bs6CyWRqnb*ujl+`Y;NF@J8%&5=g}+x6`G0B9c* zOHt^qM?b4MatYg5ClGrX7uSLN@cf*wkFl4}BY_Qg4RplY^XjqPV?xi+&g=fBGrFeV zUZFjJ+t{9ud5_R*u!H>p+H*>EjjRS8vA5vXX!Bglxtf^c*{y*0%d_Z-84qCn4BRDt z4g3_m4%$9Ca@@Npg6|ye*Ba|jfY0IyaDMmQk*7qAcO{6m`|X@lajit2=hJffno~G< z#T?`2_`LVAGqC{1d`>;~f_Q*=h3ES_;LO$=&*WwFTf}^Cl^MQ!c@y7RZlT`=?JN?p zEBLPEemD3jN@+QB`*qbfnz4Iu=f?Q=w{^=$%Ow!lzlL9;{jS`sc82SSdFQl+=Xx2u z4&-`b_8id8;(O-|8GnT~r|~`ZVddP9wXXFPoDjEfpcC-`9heh)A6R=IUHZ%62U;w! zol$AFb)NN?KrY2Nm+No93TXcxeh)O;d4k*~13CU2{}7nF3xe1JB=U~Y``|akJNzZK zy|_kP1IvmYK7kI-93j7|d1X(8E)n<64#Z2$8U_Ct`0RS%dCNzP zYwW|FCuU#&Zs`2qM|%b#$Qyvq@c}T;yL1hhI|EB_0unKwm+^rb_p9$&udzLc1-3IK z@EH4Ra22eKQPTtCt`oqzOsoTHXOeT~IktOSg1&Oku{(3oyX3fD#&72YTh_Ti`~mP8 z_qC?wIRBf(Ptcx&`*5vHpALA2YQ-A4J_w!sciCAoam9W^ zjA!V%cy@Py_bm}uF5~CG00Zzox^HvO0iT0C?G^AVIM-hQpW}US3E2NFFee`5IkWpM zf`1+Es|x2Gh}rXPVmsEDTY0$-91?51xyd*&XL|x3fF;Pp3r|juE$3PvgATum>lkma z{~X_ao?s{N82dKJ)NSxRE5-grVtmHll{d(np$B5lZ;t)}ao6x(XMFqZp>xgiXWZwY z$A5n;l?(-{Cp2E?&&GkMBAB6z==LxdL9i zB*cBbQ}G$v+d6I6?_-?XJlAr~ufQ&FM(vDNa*X%iv-=Hkxg!wx0oS3O%{~j@v-jB; z*Eip_dvX&nFM&(MguG{MeunRWu>m-%f9K5BOrE7bOV&Vn2IqRNm1A7Pc@lPj2XNLG z!8gaX5;za%bG->V;;u0#zN#^GIZrryF2No+Fb5QYyGow@c7gM(fw|74J)M0Ye+F(* z^98V`Nv2l#Ei>1_7tCiUhy{{h@}(9XfQQu2xa;{+u((&bNQO)>7H zS?i?r*K(opb#p54J^aFrwxIXxrofcKvDX!qFoqKPGH+8&HGzC*12XZ+`0!aJDpr{cZy%-pDT=mt@*7}&#urT<5O;mgU6BjW#GY@EV=QBPh8?z& zqJq9k59fPu1bhZ5c9xdbDTtm>kbB8b?Z- z!}(5sj!kAGW$Xi=W3;WA;V*!_x5fUR`-dQ5p9;2h-j5D=KeBce@0c@mF$Q+=Hxu4> z@g6YeA{fkrPc^?(VgD2Eaf9nk#hIO7ZBDsM(}aobJ7-R+JzOtR^E8hA8a2jzznyU^ z_ORw-a<;jWNB)oCygwcGK)&bdS_wM?rOk7RJm0M~=t5=`dG~DG-?iSB2c-{OA20rn zx?k7%9=z{c#$SOwu))6!dh$K99yCeBzRO;O_hi7I$~9u{!F7KO%v<9}jQyhZLc5lC zAcFzh*=OK&eC?hZ^&BMZ4y>^4ne``PoqLuuzsWxK2CORQe$4Ga0H50o7~eplpGiJT+u@18jb zf>2N7nA$JlJxlMTIj4eczSxI5+Tz`(2x9IO|}nwaaI5O*KieqWoch)vID3FjGf(K23W_p^d~ z3cd!~FG-BwuaAj8JcCd0-g(#1?tpp#893j0-k}bCmSp1p0lfFpJ@|YT=d%7y{5^6! zC-1qryVx_}8O?!yM{W=Hv4wqS*N4d&VVkJQOUjFky(Z|B*q+yZ+l zu5$vuf%hBi96pPI+Owo5K7Cd#$GH;qo=cFkEj_&7hXm)`=JeI>(B3&`HLkdZa9-!L zW-9OA;GG24wwft#FM0QKmgE?J=9PPhJ(b(kJpqT*&Vjk7f`0^OZ;_@PNzOVDTcYKC zmY;(TzetSxzC`Q?;2rP?1o0*-{2u?MJ>zF^0nWZ1_EZ9YR@dBfEqv=U+Wg0$%mlZ_ z_UCqJ2N?B{F$0odZR;sP`TI1bC-jlHkq|TTuG}`gn_<>k^>^t_ zp2`8f&twPK)0u~;-;r8lT}1He;<^47l-ajQe4o=- zAjKJ^A$s~;pQGK!x~?;?Yqe*fFXwr>AK#(I*+yG<9`dbTZYm4-@4`PrZ;N$_xV6qZ z?f*U2D}Af(&{H|YH^0p{+v*?mQ(WIZ-nadl)7JUS9-xo<{7S|6hjUqXAO0-aC2nt} z^&36@jPV^l^)c4ux>mrMFZ{XX{G}c{!8P$rT>k_;5TBu^G9y0|FST&132&db;5&4a z0sAxu+kQPT?+f4@B_A$>9_ai2|2^>gsb6NVTK_6>@8dVXXEqi0Q060eh;6SXeT?t_ zhIp9;zXR56$U6Z3ob-;K2Rp<&bI_OZ?RA2EY(DVc37qpLuwO^+fIb8R_7PfsA&2xa z{yx_H60Cn*BF}Sq3eJJ4tnuyh(0J`#_~3g`IeEpOokJ9}%;O!rQn}~OQD>=GfMjo+xu&$LDVCZR`u6JTLqKDC+CP zegC)L!4A$oKL69_Lt2v^;;u0VDQcAbV&i+KmiU2fA7`0?4~hQ*IMY+`B{(F$0YUB> z?Rw7YUVjho{u1yWXcs|S_a?DST?fqd-BCW)Zn^j=_yb?w*$>ncet=)%`@47PgI&&x zGUb;wZp~s+H?m``b$higqA@RP&q0_w6?;0bcj!|vU{A$--(_>2f$41K89%WmMo!){ z+s-q$$q(sw3%p8vMn4C>k3+niWncH|EXN>k@f&LVoh@y}OBZ72;QmnInRYD?-D?wj z41W)JCf>vLY+=jK`uB-B<4xfGUD&6J=XEskTimt22lpQ6u`>u@+R0%#d9gzKRH*F)5;5NI z-81S>h~1&iXVdbnX_Dce!?~ZSxPNh2`SaFK#JdnNJ|CY+5q$S~O&@&;?Q_(BhTp@_ z?FR;d?=x`ygg%G%U3{burwlRvZ#~<5i<<9qQ>^?49>It0=x}^x+V$xgdES+cTjjzJEkT%*J9vHjgAp^aNRpnpxw{u_MrnwT@g-vvFmM@~;niKu%j-gxe=*%6m3 zg1&x(yDH#3lY}0Kmowp9ymyDdJMkOv1lXtLc-MqEx6tnMU2p+(wNH<3a*nt??Qh&V z-{lS3K6ilUmEep$#eQTA+q1LQIp^pA&MWkffzQV|FM&jl40@m~-oG(^d+a>`pQ~rk z!(|Y}-on2K?gM*W0UcZiCvWVBK<#5(v4dhnRo&w+P&Er+!8Ji%WA^D?l<>!8C9 zFw~fPOyqrt&i1JM3fuc|WDNZQ@LZZa$KD4Epxudyxc0Wp>EZM8>|L{q7a0lLXa62> z&wdY`xvM$u%jfnClv?yv&{bZ&p{I57K0j+R@c|@YztR&ucYV+kU!jHj`8nJt;FsVt zU>~L2BW&NB+u*bpvBCwtu-9OzWEo4S=+<=KFBrk_bY4m;oqoskhnAXetBjQ?_<~f8)h7k@I}Kn`^i`=#H2Y!5_pD zdX6r4qQ;iSeO^8XVl#YW);vQy>k)bdN)Bmi`3*69$!QxCt=~;*w3f;jM_{kU%<~@k9(UMEALDyCCnvC7+x`o1OrE&|c7n5~In90q=jp4`7ZIF0KcWq*`%%2MeU2$J{9bq{6L>2uIYIuczgXN|Gwqf zOvUHoS$W3t2NQmYJy;8TZrft6@0EFqYn)1q=kXQ%F1$VL?|R$f8TfPNJunq(JhOAu zSsVCM$@s>_iXJ}eY5yMn_r#t8pQpZX&UUuyftWK|JKaMN_t}05{CkIY+Vh^uU1A&H zS$qFp)26P8wtX}DCfpi(0}_4{ZRh_O-t~R~yh}=o@te}&AA%Kl0$is_kmFp(UT=VhiB!jrvK{XP#s17w9?h0LG@`&%?`L z1?QRG0v+}_^jpB6xvmj0o|!s>4~do8$?fn(0zR_@{C#f+Gzqu?*xUIswt1e7Vr-4w z1G$WT=SKbB;dfv~Y^eMbIM?tQl>JuTXEh_o-%Fg&o=t+B4BA@qrM4yva^@5F-gy3v zx93%29opycN8nzaDO0E3CAQE9pJ4l3?PZ?;19_c5?7JcMEg^P+cSqPK#kVrbf0t9< z_g}qN^Xw_~zk&04`W-vLFKeN@+RwbXKBDcO?6)N5ed^H>)#@O zAvY0yXYl*kM7CkV}c~?OGPdCQ%?p`78>ww&;JR)`zbTNryKO^ro z;vWM0I%_5#=7Aah9ISzyvwTIKxpNTaO+??b@H=K*6JyQyxi(4U9KhWJC+5fa+|_~a zI-o+d`CKLzcZf#_ho*O80ReOVygqK%*b(uMavz-CC9uU)62E|Hg;P% z_P9daoRxrg&7Db}{#$U)XZ!%TR#dp0br;B4qwk_Ws&>%dJnQCQ4}1WZ(e7`CeGc^G zDT&zo_{V71O;N(-k2i@Mn^k0dXAW}a=s$pqz-O@p1MvVEBz$fAn&-M5zW2;I0$Xve zFNpc`a*1vJ4z}lb5A8X0#FQPhGnIO5`*t9O>{zoPz9ROx#_c2Ty6*yOJg-B05>FL# z{9DAmgQYLNcl^0|6WPVL)_kAwW{W?-xBdNcDDD>=1%wXS>@!n#pm=8 z{tC$SnTkE!)5ma09=nTa#_0FNPT)Q?ukmVP1KK|B-+MKkE5mJw8w>LrFcsq|#`E4G z@7j$QXIm0lPObMWeE-f8`#gYiy|cvoUGC8tq`yZW!4DwE%L#|X4(NMMkmv6lN|~Lx zR`ALKEid{Q-|-HunA20|JJ(~U7}p7OtMly6lHJ)CSE-mCA+$GFxSzr$|gUX&c;-SJyEH)kUDu%-vz zKWCiEcR0TTV#clSCcNkKC2&tGVD9VYRJmDg`Hb!F`enwJQ1xD|^&NMuI2hu z-o9TEa}S9+bFbhxSr9LR{yXAwIcjX8_TjDFgD>Yk;r!0-fcL?NU|Wt|8$`VPGqL4) zx8)S?uYZs741{OqyD2|_+i*{S=j|PS4?F_qcIZpMGo9nx_f4>ntCC~PG5=ljK$(xY z`|62%hW_62Tj1J$pOuL5-+w;F_gnHdaGu8?;5slP-?MlKJj)+~TOfdIE#U{Cc5eX| zz_mAM>*U;rHQH;m``8DU!N5A!?g7tX2Rzq?^X$4hhqIr6i z+T5$4iE(4szzMv2u}2g8JJUtrIhZ?@h_TN1#^>eS9kC_}dzYAe2LfNo!AwByS$Ow; zf*$avl4?I^-oD$@ybS*~XcB=X5TAGJGe*Ye>zTV>`?}{oUYuKpzXo!i{Q_LD4>`(H z><>W@pMe8lO%vz+2+WCOihJ#WanHpwn)Wt#kN5_-Ki79ZXNkS64fFs!i(_=qqlx=#|9g_N z`k4ES8oz<&_4rfKpToP?9dbJSsW_|WY)-(<&;#1MjBU@|8{=NvKAUznpM`lxHTRxA z=%`OXF~5iR{qUQx0$s?A=Q&z0{`6c=YR`W#cS~H$Z_``}KEs z&sy2$JNWBx3Ec%6Y|Ad3KUWioGkD*fhjQfgXYkor?f;p1_@4Vs?2Yj%&LHP`tl)kL z+*6Y_yWf_tK~HX1(KvIx8_p!>I#aRET;bUSIYrQNp0DTeYv4V|#1->>hm<>LbKO(K z_)IhURN(Am%`@}^`~$S&eK9_j;=@0Kv+q9k9Hbx%1X$pgH3^w77sQ;^_!VHyx1`oR zf>(Twp6^*=UwhpJD{@4NNoKXp8?e75Zvegh@XJ0hPX+E6Ze4lf?roa4)F;pLIRvh~ zJ)ijz<9iqIiM**y`!&1iuGTgG8|v+)81q^F9(ayF0DliQTm8+Q5qk^nGU&s|DE9EX zyAL0=E~2sHGqGnU5bYtxd*%7w#=i*cGvLby{8T)b2gLpR$fB-s%Hw}Z+&bs5$Htr( z@h-7nfos6|uA)6(`y_HM;X6l%ZCpRF^%ZTuJ$_#vf_9FDpxOX|%ZsHvp;2seF1^57L zOHa=G#Qk>KZve}Rvu3crd{2J)hHp2-k9fs@&Iso_a6tVD_?83_epa3*Cuwg<4g&B` z;ML{#YfSgq#kg*u-G{sw@Lw~ZnEP3won3B@ZN9LF_u2RcorrCV&q&^Put<#WtM|FX z_C0HNDdT7Q6poq;a>N1*a-ah;a#zdn)U%U#~Hn0C-<;wdrNpPM+IQ2zz7ZVcGGxg_dBJPakJ$pH&)b#M?I>&%M72@tI zqo1Swb_I5mz_(@zC#Gj(j(&=ojrOch{Tgm+fVWrKM-AFj-=-s>J|6&4By(har`EIt;^9OFSNaW2QPNP zBQQ+z_Tf8xd$v5k)vjHnYS+6>+~2i*uLt5monqYu;5Q>h36(zr`wlVrIS6WcVl%XB z{04js60sua*YVpNzC-VTj(B;d%#g8-9f4JyO>OUv*g2q`$jiVp8_?~%+=F>-9p`8q zOtG!^-F1$IyuIzo`eKOs`)G}ub9!E+_VDI!z?*Qd0pBxYO+1%l;JJ7&rkpwM<2kqv zZUXnwhs+B}#(?LktkHLC?i}0aVIOlM#=Ev(b3F(9U#U95Pi0A-Gy4rOZ-Kv^<5{@B zfXl$=^|#ux}D`55EOI^CkhGfwjJKzPD|Smg6kuT!J&F$KDom?!Y@^LI;>i zit*=*J+6b-xA--6&XCD7XZvSSBK{8X%fS7;bUrQwZ;id(-#suD&qv<*9;1CHr{ewm z*ckjV*k1oCod2%i`MhS{M26tZh<&f3%tuUd&VBmbg?k$$>}SB72okWJ+5QhePb`4v zQTB*^fbH*UJ$hTr@x65IMDF&v48+{aufSQdp{M*guw83@4(<|Mxt9j;3$(YyGw$Q; zJ_GSNKYvcwaDT*__&%LX_V*?J^p04sRf*=hrKO6pPvK8 z86RMKSDGxVonq{h(c2>DdAg2sY>V;E8GyMj3Hf_B#`i1WoO7BpoAmo7dFGrYnfQN1 z{~rC4=!1Rw_`7#!KYaJvo=M;v&+3V^9N){s$s8>u#rXH5Tkzg(-v#e%>ucVTK6yPj z2XF48_9}VA2VlMLSH_;!clc}KFU7i1JS*Qz|9g~|(XQ2@r{X&{2hM*7ZSKZA#GX0E zcgT1DIk-uVF}XnR6LV(omiD%oW6uPqEt*`QPA<>`@dY^_U^|n~@@w!IxQ`CrXW;oA z0_)y~JHhVZ#1h-N67~+ZFusf42l7RTIqx?&UYwo$GyEpI*sgI>YwYt5u|O-X{U%U+ zuF-mrq0>KejNgdwk>fMmmINQ*61+WMB|SMi@ak>(g!tRwEJ-o`@8A93z_<7KG!M=_ z1lnh4&Qu2Co|EG2@-N-9fB=7_)@O5;Y--&?K3>k~TTkLMY@gFqT(8ZP$=ToH4_VjW zPws(N3N_}PC4G$VMWD|T->b_Y!@soen!JKrf{WYdGHJGA{pr1$P~$r^*^_H{-aYFD z7(fT6Yy0!SoD!>iSzr4sA^#8HTN2~@abb(IZ-R3T?^Iw<<$(AhI0l}N`L4hH{Ehp( zywl$OZSn7v*4?jMS(ln7$CdA*MP57yb5GA5enD<|w^(l~8+_xLoO9$@ldu)#KeNU*NOXl?AxU5ulBIdwyY=IS?~50YOLFDJ)5qLh8WM;edS4i_t(L_ zMeGRf$7uIyOoEF=`4_xf{tSH=Xs4KFVB`1XeE{E|!L8ujTL%JUa;EEQ%bREa zL-T5m>-){g=<{fK&#cKD`wimz=xgX{UvqYeTO(%h`rFdi9KR!vfa}lEYjZ&#iF=inJ|z9Y2K$M|>p>%=aCb0873zKQqV z-r|tDgr4%|_34QBruIZY%z+ZwyP6qbh-hWLkXDc6DJsNMHpvPJAkM;$fhze8_&(Ay7o^2oFOpV{x zWyUk+u*ZzJcPi1N$@|3peZha~R_FgCK8Ptobf|Jd=Kl_sjb3Ny&^!WC_0`DD2 zlXBMZ)@a|OkLNrU<9)p7W$sku=EPQD54g{PSO-dfOhr4#`1j^7Z;Uni(;V}D0e1uz zHOJUg%)bxkdp5+B;f@pYtW%AvTmL0<$v3ok>)`z6JDWMX!1LRdAm_9P_Ee1fEKem) zt#U`@Z%+9aboo2PEx6mj?_soBwD!jN*n7h6vCU2BIp~NfzT=+Twz#)7@aNrGdy;du zi)Ym481GPj1{ZKa?Jh9ykL0Ka&;i$Vy}&*|yUr4r=if7i>JRYtFjw36-JIz&(zag* zAE3w9Paxwj(AL<$)g|Ib_}+>0;1ghtF}1Qr`|YsS9uXt2e+T$}T>-w^VGX?f@8bKj zTip{=9$>rACMm{e*%K2!8_#%yJ-|8FW#ISaLy(D0j z%~|4ExJLg9-ZcX5RlGgz-;uK|*6rRH`59|$OM-t2?-_b7Q*m9-Y#_(H%@%)%KclvT zbN!Po-ugK)@0`5*oXTT-|8w`ZYY+3M@)>dK?9t9(%70w>6yv`UdM92I;@0|nyiXZ_ zDjVv51lqoqdOe;fT8X!k7B zV5PHaG&P*zTE}e667w)YrUrX?BJeK z^DdCL-&EXN>s{heXGAM2YW#b&`|JX2iSKrc57=qa|Ma=Xc(zyRk+G*@&d0=W0^i$V zi@!|lDaiDVEza8Q{Y-mj^5>IzrkwWaeAu3?d+{8X*m9>5zGjGcr{zt^U^WGoVX#HLPP5eW0v{Q_0ywnH&Q{cO?Zyq^QG57Dl`@pjq zws?Kt_Z9H>xdaZ#>x2>`JZsPAJ$QTW0_)7px&q0r$(e`xD&XtMT+q1t3e1<)-Fc-MaBVyM0jEz~hChl7Lo{|5XoY&AD zzH3hBHRfL3v+G|b&v}jYa0yuZF|b$Q+us>8c8B(x*Dn4Sz#QQ_+r%9A zXS~GT1>VIBFV16I_a2B^)1jLfb3cAZ25isQvweo1${gQ)W^&B+tejil`~Guq6%6>! zm(?I+`@HsmXKx>8nxUH@n>vZ>_&$B=iaR65Df9BF5h}XZYs53uiB9?cjtx6=UU&i@6yJigd+?{7CeT--Q#5lS3d>uQ8e@lpM$aw}Xl5>dm z&MZNO8$iPL?wkOB5A`m#XJ7ge|I`?|)s}eu<2s!CS-?-F^avtw_7z9lIq=33`lFij z9@_b=cTM|x*80ZnvjRcQ0Dg+?&y+dbROF{M?Kw_!?DZ?qk=xh&8GVY4FTVjDc9Yle zjr$Gx0PMRidWe?sLc6~44LRB`32}RDpGnIJ_~TdbK|i0*(Y`LoXZA8(A`9C`DbxQ_3Q@3;NeTk{3E@4zYM-q8-dQTFhUCVY?W zIW>73-`_v1wYR_XMKpHI-4Gv$se5ZDd^?9-bAZ1~Xqyw0ygw25p8E`n!1=q2-|U3# z8JE3bEB19(zr*HT2W_n}qy8D=Go8_Q3H&ZD&>Qd%*xtMS%E`CAC+g&d&qdu4o62Qk zK3nIOPg{K8%ijjBIh9rI>AgF_7Cre*+{*zm=TY0=^P5V<_}#e)=krha(=)h0%(^Qe z;7$cQg9L`E7dO8Bed?-sEX(aS1k+Z%b^5C2|$-lQXDSU>~@4AB8SICm!J2v#*?YD8mmR z0cVi2Z@XV}>=}@Euf=BXJ$`_x$az0r2cD{ zSAp8zuAc<6dNw=Q&bp87d0YUlCExOYK+Jc^S^YWP#OG+g9vw0M+g6X>mL7*2-JIz(183f8X+Xxo>9-VxozA^xJzH!@q^U0-3xy{tR^3N{X@9 zF8n||@EeCUN{st(|K^vRO4-+*O-gTKU5t6{zeoEaG0%7^*&J)(%sW6&C1Uh?a^&0n zwNL9+`qn&SOL9-^h+DVs8Z`&2ND6*7++cRl!Fwli&0C<|>yaGN$9RYBJp)-xqVQ+J7oB{_OI8_`U`hh$YZr z&x!AX4Orr@)ZjAqLok?!*xT7Q#3IIhukjPMoPB&(S0EEBf-W}NzW3n%STW#RV{KRS z_V8!G`)QtM)x%9?&<8i+{Mq2#);P;P_?Vb;+SfJZl(G(-eSZmz_fe?w$04y3V9dIA zu&$}_eihch_;^YE|fTh1C@KPGuTb7LENt9|ZMvG+M* zkAd%4ChsM|KLCed0DZi~mY%*BM{t35jp=jp_klkF_vZco*80RwXC%)%KNWu_INuyx z0X^_c`xy7x&c+(!eq$0jO=dNAnV9!`3HpEq3pn2;YrZ1>A$SCq#9Z$l=*i3G6SIeL z_Y%NZ$wk}Cc&mXeGQ`;5cPM~s^~9PaY}d`yI=B2ltglFDWADTHGt2!B;TWrlCfK%RB}p3`mR^l+)-1315nP29Wnmug&lD%Ki%2>&JUET)ob{Y5zUJpj*Y zDy}y-7GsX*)MPM5t>>7r-@=}Xxz6VKoRH&s+pm{Hd60ZK;vN z{+1APPu6;;z9qz6$3FdyZ~t#>GF%5-!{@gTrt%|v&o_~i!M2=(`vbTM)@Q9xK@h)I z^ZedT#h;(IfO~L%eN5T0xBWA5zq8wNM9nwE&H3mIzT}f5@3Tnpa)(3W_kruj7H4L_ zbzlL1*81i+_Y-nl<1EQhIrO$9Vm%08pI3=Joc~oK_p0@+Ikq;&b6jDU^&}?ZJMLbm z@8BsNwM8u2T@bN0C!3;GS<5Z`ap7jVz8 zo3xy##D5R0Q;bh#UgJJ*`?!y7i5SmEUX&ihjL+)&z7KL&C-vsNf&Ue7{Y-uWFG*eB z{rfFy^SqRUW3Pww@jSmJ)cG9Ur+v0%P0ZfTdY0JRv$Myy=A$6TxQ}D{2X+(d2K?RH z`zP>zQ#)dEJ@F-epp}*%F+Ow8#&zFE{{Sp%UP-QhByyj@xwhEeL$Zb(Yg>+g?{Tdn zaQijK*c~{3zG@54yX?!HCQ%Kvweo$7RZ4Z8c&UYJ>*bnnd<|y6^XbSbQuzgOB?llD z=q7EpBjR#p&lBEWr*qTC9xL)XQ8~}F?JE%*fM*owKzr_|GZQ-%Z1>zaYqWdfT}4?- zyGn|&uX* zZ+s5y6U27V+j0)x{+`uzwwAZ1M*dy+*FlcPps_CzllOP4Yrtn_pAGtL{4gIZ@W0YW z>u<|NIP(X1-yP?k$`E*P<@*9>|EZ+F0~t7T@d-7F82`I4`*3q$-cUKcqZ?FL+r8+;lTDA zxI}wztTU%cCLZ`H#v1djFGB1p+=Yr2x{3V*-gOh&Z;HK3E&iOiSlbKV+>gllHTWg) z875*G44@N;`}xz2@n@^Oo9tt|e(UMqKYbti80R)Gu*IDC6|kWu&;v204|0jAi(r?# zT>Vhr7;`>Gui(ows?W*2C2|&M*Y|9iWa4dJ*KqCTo8wIW-{bK69AV^TNin_`_O;$; z`I5kAun%_+&b*g|_zFC(oVio+Os#o_?#T;y_h#Mze11=XbI#$7ZHxInpZ4sw`5Bzg zw+Gu2@nW{Ne#s%;0ec&__v_#*VEkJ`&J(z=L7c@CTf%i`afGuxJ=$|6^eyRQJgY?B zS+YZa&vJbRAJlr!S9Oi;de`=6??7GxO`NAlFgpA`JzQgftwi_-K+N_3{|p;)THTDi z`?cn|HDTt<;!O6GZ*?zcjQk_|#m!AONiS6FSCvQXE z-@^~sXA1FO!P{?*_Ab8$&VfT>3BHN%t>?4`3+!9^@IBZg_DB2cC6jB8cW@5g0f|@#HpCCX2`EC$wLC9#zK4Aobl8d5&#*mz z=M+8hj`+IPyGA)@eF5M0l(FwtzQ;aGg1Ggk^GuSK^c&z4II6v+t#RMZb`N;o#-{Q* z@gD+vIO_`ZlbmyK(>?jk335(_+)KoK&wVC;0t2yA!T!3&Gx`lMm2w8ep3(DN^f#cL zOIeqAriZgTODB)7$VZI-y#wpr>u2ysXhr@Q+nJVNhV7kopK|8Q`8?*pxW2Z0k9Kbh z^b$D3RebFsUaar?WKJe_AA1cHd#s2DXmSP4zu&mFzgL&_8;Lfzk8xJHZGH|vGY2@! z33>%OQ6*x$tNS(Xy}SsVM@cc>;{*8TztK`N~=B)8&#P`q(pq`3#p4~OzJADVF7&Gi+@7u)u2K@jyi!(M! z*f)tCfOkQP#*F$rF>AzE;P)UV^*&!Y=R5?S?*ZC%k3gc|09^AJl>0zVPb?e9b}paa zTVNnIl^L-e;M$*oFTgF(BmxV6WpA(mfj>uYOF7pV&&i)B%Sn!0gFXDePv)~b73=U5 zx(9wov`+=wbM(7(7Yz8@GOK<4S!(~yv_Hv9{D|>+%X>Cw$#vrL=5Na}T#AZ%$6Joi z!@4!PCudvyE_;U;&Ve@XC8@sg1$-YbvBmKombx>zAm;jKJsXRc)ZPQH67SUik)#;= zXY_~kdIWmx1Oj+}4}mkTw6}Vm;1A?AS>Ow$+>a>c=^rIlb9^r@)3YPTn)BERy$3RQ zjPG6c{>nWkW=)Xaq0ODEC(q6EOh7U2yF8Tx@;@;KeD`9Dx5o9~BktaFjD6Y~YG;rW z+u|KG?mAb_$a$m2cHpPq&1oOkf2a1C#!qc{-)+}vb2YAqGw#1hZose1iN=o4@hY(* z@tc|PI}mUgc%R&>--bhAOmXgg z{58DsA;#DS+xQ-F&+5mZeNPiPJ)AiMwy@7ra=rl0;GR$5dpKo=ZI0SL^7<*p-wBKz z*L>rO+I=nI@n@>oq8K2;#p90fHrRd5`Giw z4vD=^{WTqh;bJCHL;IBvu5ad&1tA6D)c92;3wc( zZ-Pv$iTok4>mbPK@Vi>?3=e_#&hMGX@B;|o%spE1ERVnk;5OKAz2JrzbJxV|;hA+c z=9Ym9}@E$IVbLO9H=R?Vmpg| zlfd>_TmTEW4!?=@o~5(--pqjY<~Yj=`#jupV@NwI>�r-udSHJgxEW+arKE12F!l z8{_W=`b~OlIi;^TWiK(-$~T$Adsb@WQ_kMb=^mZ|&qccfD>!-Y-*M%F`Xw?Pm1J$Om*vm80DFQ0+4DG}qpFUWZ&YjPV`68PhY znCH1yxhe1d6F33R+0k!~ehNJA2k47nARfRy{1IQcR!1yV^k`*4{QK}HbW*>=0dND{~Bzz>el$4ojAeP_Zf85*e8L&ZsPs(><44!HHjDh=Cd-6 zU(T)aKBvdj1vr%$tv2I8&iR`2J@gE}HwOe5fPK=9@&AUE`%T2AVvYUw(GTIh)6UaR z^5irA01_zg){U{QL%z%3Z+58hED~7MdV9ME`3-hQ?R1YB|2BNU1-5>Ne+=wrpAsY9 z>gLAa{oZxh$`1A(wlVv;uIFxF*F7TE$BQ1;*xR{8>up^7D*4`Nxu~^rMPfXo#@S~o z2|f|~kUD2ohIp}t{WHGL;wNAa=r73)bdw9jUE?|NY#$Mg9q-;Du~cJM$bS>)yS`ix z=H{@be|J9sfp6|(&|xc{wfp=n_yO?w=76Z!A?A5H;~MleZr>yPPvAQc#M}P%iTm@S z^|kkJKq6-aXTE39V_V})?*pHe{d@*P8aPd*;#iRN&=&M_sS2+btwfC92-oR=YxTt3hjkBu>&P$B9ozuyFaH_nfwkr&IOPOBVqC|2wgBE$pHD~J z{-Gw%T%V_RWrf`nR|fnw{yz8syO#rVV2!-F!q^hO+%I(FT&L`XoV^+3>Y3wIKQ=G2@K=?8R6k)ywg30_-Y8VF>pauk zajIw3-5TZ}(1ivx%zsfAy3ob3U5+atm_<<+x)8&;(1k7pbCN033KYq~snBZB#YPBG z1Yr(FgdzyD5keF}cn-#x!icaDMijxALkOV=!fZq^MKIwIOsH-`dY)(R{r-Dv)qf)Q zjOiW*f8?#T*7v>N`+aNewf8wyc9MaZ-+f1U<~789R$`0KJkP-A@*^Pkx+A;}xUUJQGc0rJ4V+7<2_>Pjca_yHjL(aZ;h#cYz(M3KHoX^0%K2(_rQ1d zkoYA=^ZpdH#LT}9I_3rFbMC0_0G#*w*z>@shWEhVsSE37&Tr)~Kf(V2*v|ef^CAyZ z!#l8~&g&iKjd9G}H&}bt*FjsRnre)? ztna|Fj_1&^eu|jrnTwmuKLQ;b{oXHovu43}017b7MJk6bxoz$ea`+ppYD$#dQa<~{NB z+p+eU{snjv*h5denK{RE-YMf5_?WnQw?Q}aPVk?D1JL4ome>(^26!IwH;g^84mh_j z;rn$LIJdunxc5kn2QyrIh6k+e?+kcn)O`-L+Z~pfU51H#18_=O6tZ%>WyPbUvHqP<6j}HI%-1V~$b={}B;)9uE zo+ih0@4y=~PYuo3s8z=0`0Ve&`Fl0s53}Fs_wkL-55M~>tC)Ar+NbzyJI9w;PtE|w znh^UrR>zm-_qqL$*g3HNP{#1?kMz4W2G;uu>7u%cdxq{^6tDqZ znV$c<)HU9-)^=KDdX6pefq8xI*YTgClRR^^1AIA`oZDM)@8e(J>$|L5Z~{Ic-jy+Z z{KH!M?Mu{lzKWeP_uQJYupY4Xx#hVgqONs3^Y2o#F%Q$1{W)&%`wj7(^F80PJ_7|_ z%zK-AeMsy8G$qk#Z%6oge5Z_8;1}0mkT=J+uE8VX{+t^3LfAg!wZvj_wxB8a!7;Hi z&*gj1+`HQPJJ100+_PG5FfU-kyfe<+xZfVX1@>aUIgd;DT6hJxhpWJSzX~*YeVMd6 z@}9!4j%Odc$-EHTf;aVp1JJTQ0pq?HOCiQ+uLJP+JY)JAW0)L$_rWc&1%-HrU)*Ni z0`p%crvB#)&JFx6Ft4yaWXyg?E^E*6Jvi37JM(rrfVITNA;GNu}`8gwg9ly_IQ^z2xuWk>o1D~tU+L*ca z@ChjRV&~|!tUtsrPq^09>B=}A-|Y{G9f4gB#0oGcd+Oy2t?)m@J_0^B*DV;0@d^Cq z9uVtUA2XiV54eK=obi1|tu6ckR_&M_Yw<+tb z&DhiU&HEW+ZGD;cdzsk1Ik&^7e*oUww}JBpbZ~P1r_4{lQ^c)v42|1T0$s-`g|GvBEnH#I`NQckoB*$-OyQ*_#UU&FjXZ|c0 zjrBXs&3Qo|>$MH6xA@I*zt$-D3ga8BUzvTo9AdZd`MX~M58?IrHjFXv%O{{I6HbY_ z|8dQ*w!7vN^Bn5xQv7V)oFozD?Y-Ibfc?&h?d%HtN5~Ix%zpnYG$S z%=?)qTh4%Qz^CS?;GDJRb_|M``Eq;Kufb984v3i>8_M*&Z|H;n2IEhFT5I!}d>_AB zL-~APC&au1#_VO>3!Da=vHqty_9UY|>l^`1u4gQ-lZt#av4!V)>l}X1I=ZjHxrg5zpO5Ik55NGg14qVL zd&cj9Gw?c4OFJNboAn#Ovlz-$!}>jQc^#YrG}qQqqZAzA26LaQeW~#ZaK9(42jce7 z%Vm54|NCIeyunw1dpHFh^AB(9>cov^|!nHXAupiIi&*#`NV?opV z=Q76Em+$TYkN`K_!+QAd4+tfE0d;udY%s%H;ku@ z$G|-M9M6K${t7vc2gHo~Gy2XPb1xmZh0kZ&!4>M-=hMLZbqahhI%3)(@%LE!9aF26 zsYb`TW!~Vk$Ikg|p8E%&FB7gZH~$FUON{RWpXCGA2gzqNZhxQqW}X|pjac`MFi|<==%>DP|nNxuMq;LB&cEI=?F!w$%?!DeHKLGBxhgU!c ztli9`XKBpb(HJjXn|>Df^Q;gXfLa~n38THe3e;}#HOxCk@3`2q_8Am>SC|jX?dt|} z>x_Budg6bB&;E{Q7<&l5kKb4aCsvqyw)(|Wz`Ydwv7Wj2*ym-?%WM|yGW!QE?eVw}@Fku~(Rzr=A@5 z*^$?S4gQ}1L>t7mYV@d#XJu0}_E%lIy9pGE4g&(Zk ziji+YqaQeU|rv}HGS6Z?XN(=7duC>3ico?BN+kHC|*K&xLqlo&9&LJ?kfoSJs#5_Z@q)=Gvld?qBT5wbn7Pj|b+K zDcibg*|X2HW!}xPi80+5bL`1)@+-hQc!;lsJIEKT{jR?Uo(2u@{`6o2V)vYvX;l~t zYoF04;12U{jx~&@tj~eZ@@-(Qe1Ff;KfuY^Ie!j+4}J#hN82*q1$|)w{+w80JTV`% zpaZ@SN3MO=*aPP3+yw6Zkag@m*2X>r_LQ8aOxo7)tn_#Ivac=N0XV)!{1Wh)HDIS< z5&Y_lo>)Pn5n7qf(|*JG0dqBrI>hbu0BHIif@9!0`mC*cYFuE}_Kdur#$2nXuPYFM z@0;~D=CCec7suy$xo>&nUS^;5pTqYKum{&Ixnpg{0o)xUrl^PuzUJ2cLp- zV(&4o&s}bT_Z|E>lb@%F!!B!9~^X0&M|nNxoh*YKlgKrZ_8Lfi$=rfx&7^oqEBdLdTxoG!j-?S ztJY0&eZKZ(-yM0@9DvVu{l4sS;S}&5arN!%9(;9<7}utboL51|+%u4)R*Qz`CFdC2 z1wFn7d<@UL2A{T_$Du6W6YtVFetqiY+z}Nz7pDn- z8w}#PnX|hSP>lNw?-G;KG4A4J`Z=Ut%iKMvskwfKxjC;hx?Y>>fjsZ3d$X?rpJ(@D^l{!?Iog_t_N)6Ums+&sUb<2exA?TPty{2v0J_s(CY znrGyh@PxQV+y!mp0|>C40^^C-`` zxjv)utjUSFkB)c)UZ~>M^*>`hz~9dtn3I+1_fq>hB7Pg#M?a%;gfkkWyaA~9Be>az z=aqS3?lWw`K+aQ)KL$PXoiTSWZ{k<$(0qKcEo=Rvfisk8|2?C3=2PJReH!;#@U=7W zj_2?i{!4SLCw3P-*Ddo6b9?kIJ_P=b<@{~NW5xlF^^aNWv$r#FWsaNEz;VCwTIT9K zP5g*)V0|$A<#qGgbNCFr0*q^F^kw=cxG&?U#0oxP{E&6eT)#EkN5^^q4RC({+Vl<3 z*MnAgKgj8T-`fX3{TD%#I8)z6&EAa*&-EEFlu4&!+z|I1pU3YRJOcM-pE>g7sUw^> z!1{kh>?Pp51?ND%dGOYq+m}ht@fFr@05uA0`#NRqK04OwTc=@u z!05SqZu*YF5%afz^Fv0pj5nYs=QjQ`umNiNPQ(iF`}jNq&%!fq@trbP&)lB1F~5xFmewm<-9#ND%fTJHvPpTiY!%K8DL{uXY|q-DJU z-zBD=z1y>#20VdJ9b@(_-_d9Bo!M{9=hega@9md?yaPsKuY-0bD#IY5jX>Cug$&YKYz8CmYlV@ zFLT}lAA&n@)>g{2a`epAi@D$1tUGhcv^V?O&Kie|*1n6+Jv3srp-j0gV+V}y{_|h* zT7iFUhj9KGxM$7!_Ff7Lfq4ez>gyTLh#PY~-Vbuj|2zEF(-*T}-;?t>R-c7=e@5(M z;F)XoWL{gQyP{rU-U0hK23xt{Nw}|qj`=0V(N8F5t~KTT5Dezcw$` z%a~U96BzT~X&&z3_;-c@zwe;BYP55n>)Sv}{W9~rGS&WsweQU7%p1w`A9CI#<~{eh zoy)-&Q@^lo!9Nn0^K;O!4hH50UdMRAXrHdznN!~fGDjQF0&Wi)>&zM(M&VhQ+mw;s z^o-smeV+YY;-{ctE?4Wz^i8mj@tioT1LA9I;m8xdE7n@ueb%qxe+Ycvj;i>;`tR}E zhu@=@!GKn*W3Fa9k1rC>vpK}SVXdaP>3m*$Zhp7cCRdK{g7Z^yYU}WMC;KuzXM0+k z&)>S%dz<+K#->a?U1$BDfcf`;KP$%PG7G%H`Uk+;C&W(}8**d%2G9WWo+NHg(T5FX z`i2~{w#G3reOgzhZ<_V3y#+mp4b0yq_7rH!#D9zVz}&jGnQy^{d3r9y4)MJQo&%jR zd@=j;`4)UG09k8!ofZD8efZzDK-|)IJ{XO4!!@N~-)eQ0sG6_VhFpHt-uz;MDdme(S31???xg?w9zv_z?eX z;4|y?XmsSfPi&w0ZoCGp@mS9!_2-$(D&J`LYPza9HbO~0F^ zkSm9AZGK;^-Qt()Ue-3?x8|$xj;okF@4@rL)OqYq5cfOKkXOp|4e1!yc86H6kJ0<( zdTnajlfCcLH+GrWUBOph@HT#Io7qu zL(svg%{=SN*=Ik!HvRAKKg+n!UYh+k7}Y7|ah5%J2FCZ@7yaHh$NE`&#`<4mJlfOa z?q?zKAA|KdujkvjzV|7X_=cG8p+D~*RyE{!{=P?^VQr(Y)ZS^mdv1Hep4&eF&-J2x zW#9Yi7J5lvd*sKTx8`4TuRf)&ai6>6akaOcyU*pKbJ>vdI`Ms`-T*wmt~|~t_;N3; zzwcg{`v48&`<|PsKG%Jwek;&8k;8m#`t`|E-*KOLul<=i@@K+&?)lyXK99$&Ous+) zoviIPeBbRiiTxw!;jYbl_zwP$z>h#v*3oHhAFmL*0@|v+=kL#4$1`J9{v%@EKhO7n zWdGy$WmUqzllc91|Nj4F=&F3F8*=XwH?}?#=g0UDKu_#X8ISB?W$N2^DO3F;=4cM88pv_~K8xDa@!r+Fhi84y-`}b4@5yh< z^mAR#zo*>?qRnRHx9@t|CSANWMdmg{>0^VireP7$9*>{NF`p>a`%*ym{ zb?;vgYl!>qIM((6-5!4nK7v!%H^;bl*!p*>{u-`)c0I_a_sL0cmmYc68C#x9kDjgMBhVn{%%kFWcJyEz1HUWwPm_v?Vh;&E&jhD ze*;eUaNJk=JB3r5@2PR`bSZU|`g@+Mth?y?9oiT7yY?dR{dyBL#f-&#@1Nve`fWJ` z`!sxa-zL_SB`>oU|F^*lz~@?<+8+};u}9|e*Y*i(`+EcU{_fMbjQ;b? zjCKD%$M=ZZ9dYY>R%>hV7kt)s+-JU<-kDmvA=hU*Fgm{q$1x^f%~l_y^*2?GQBw{m z^7fcLd*3&}@pPr(dHD^p56ybF;2gOw(|4t3Ew5!g0KcVYtPg>?>b?%fUWglOSX;|G zbIiKKzhUed8_)vp%Ll;RQ)0!~$~t_1M$Bh)1bohi_}VhT9@iYde;ttTethl+pfJA; zv{I(M++poE$a@<*!zWtco*ptL)j^}GXqQ$SNd(PS#hm7wrc7*_Qdog#^glAuxIr@BG)x8X| zCpDgA{WQ44d??fXxX0Xc@>}V*ti_)kH4ljOK&_ZHjz9s_ZU|CaBM=d4a`?(r1(tPUCXnRzX6 zzxybo*;;VDQ z@p*XHN{MF~?ZfZ=Lpav2&G*>9mmV?h+qc?hU;w4m*^}S=@%ikD8x!g{)^=;I@9$>* z4mi{rtT*JoO>CdtC*~ftwv5B){_wu|-g>6iU%$gUB6+@h#(n<#?3Da{X3YY|XKXLL zf3KO@$y>yQ&WR(oC-#=1_2?_KhnnZxXJ=C{HBzLPqod@eLD{O%dE z{sz>x%PG^ppXYs8^SlF|v)@9$<7+!-ZT*I^?p>aHe-Di3Q{@^zn)5z@^9XD~ZC$~G z-#TNh6~UVETg#ufYjfWNzW0IW@R*h9XRY^eV7|`|$ny-2f#0#(w#2`S-}nD)PYvIZ zSBcB}rL9cQ%l;eYwLQY`??^9$Zcp4i|Fdz=wY{Kj`r7y0d=?#$voEf;7^v-NKWn>A z?mq!z4;V`om#g+|V*AWIzkSvo%q^d1of3Nz91@f3dTph!B6b!3O>m?}W#{-_uku29 zNX+N1U4c`dL0E}YnxX-MAa*artP}eX=pYKEqXJ6d<{u|a4a0;B- z?h}88n9rkUUfY;c@EQaAYTVHe26>gWWvah1pLuQe_Ook*E{NYHrVVA%xihav=lq8~ zy@g+`OMBw_Kbzxq{~dLWdzX&wp|U={GW~t_V5ML1#h#k=Us=Ud>ky6C@O$6eRgSqG zYrmO3pMCYW#C#8GGw&R~_qR6devJPwj9rlLzWgqFo=1E1?ZxNX5bsJ|f#-$aeBXm+ zPdwk;ftd5!?veKkH2m*b{*?w)a4*&#^Bsvd;E?%~#7{tR&FFpiJJGOSo4x)6{!c;6Jf>zT)4n_>YsquJYkP$M zDSYm=VQkAfx;(!;_lNK{z*={Jn*O`TIdjk1Scisl_wV=&qxy2i+P=;D0WtHuyDx*B z=N6wmjU!X{>UrBkTPAJw)Hdgc(dX(p4b1%pj%UWaf$!cuCv)Bca@0Hl=ASXztA6{w z4|?*}CjT8c_FKRsur__K5K-%inHR>Nq3Qc9&oh&! zzYxE}*fKVZ9}>R<6X{h%jylXU^Dd`A1x+?)J^c^zL=z;BJqK+ALYKAHCr z7=OUnm8ova=)4C7|J7>~Zz~7LEyUJl%sbLD+DpTzST_wZJpbkz^QykFEq?pE zP0U{1Yf+Q&Z}Fdkn6VSqFM`XUzBdhd>*r=q+99K7e+o9BFF3(CvbJv4$8+2q`?Bwt z{mLugCVt}w=B{igQ;j!>eFvCR@HzL48|E?n!diak{th&(W5!xAfa}-r0y*bZ&4zd< z@c$X{wu&FIz6(ykF?a%wbMKc>^V{H*d5cc==`%T)x&FK!X0Fd<EP?Pzh~h1GuR$``~^H>?l;4pe+EYFImh@pai7gw z;1cLqx5iiIzTPHwz-X<9jw>r=x-Z@}e=eqXDe2MrqL6!7<;%1G0r+m;06W8X2CTgW z?$NXG-g>TYf;;$Q$Mf88$$tWl`^>&RojG;fyu+2|bF6(1eVJ+-{}#9lXT4W-1{ymQdw*Q{skhVe4x=qt?CFjxM%FJu1O z=-&YMaYB4;_GrH^fG2=w{6k>HB~}=3!X4y-UO)Y;&HLj%?fU`qM$}fO z^YFbY%tZ_DGV2Dk@y*(vy?VxuDn0wNX6k*eA9Na^?*PvDJ}?&jTAA*JXE5MbOYVTb z2j;v^>?c6|Q`QasSmzuZ;x|6bSl?0Og}8fogSb3(W3KIgpWVlwcvsK?4Zap!BCiGd z?73$=p84iHWNw}3nV&LlfSjK7Q$TCW^i6CSYg6+zc-|FxJ45H?*>8=VImhq4x&`v= zkMP@92jts(Q|2s2`#J;TnazG{2X~37_X{v2k96l_4skVpOnj_KtYPiDXKXMLkJcgUg5P@9>4DGT7}(2(b;3}58=H4 zyu&?lZG&GsV)UDL0POp7R;E3Ampt1i7`H3VL)OtrUQbSg-@Bsj8{in&hi0#B`TW_* z`!*QwKDjQ_TZn`bEL_ zIb)qYJxyGbzt8NcBQ})Bqj&KCko>p7couNi_C9{!HLa9!_}m|d9=?C;l>i-;u(+0nYn*-ZGAR zWla48dj#se2CjqjB40Zr?!Pw<))Oo1%5-ns%OUU%Sfj_Uj(HvHhV{;7p7+A{^A+H? z-L;nb>SV3Xcfk?3Kl_dK#XqNe@&++^-jC7Bv~Cz{Q~O(RKLs7}78Iae!Z(1=Kx`-z zuP4tMe?G_3tGpi03;6#AJcMVgO)nBw)|Kh^M00BMnOUO++Pa2oYx}L!y7G9uP*bG7 zdOqXYjO%}g_&HdgYq&?m&wz37U2WDm2G0@e8EZ4I0b?(#J1*~J)|KP4J2bycD2(d1 zt{MN5xIFK?-|ay^R?39S_-^SZ{uJXG;|7@1!P#d%XX9lR@0ojlo}OcM-uGw!6O3DU zwb_H3TW~=90cZ;>&^gZLdAPs3#+jEwKw$Fz>KY@6ui)!}x-Wqr1JAt~ED>kjyz;raW_cIJ3q_nZTt!yCjdgG1&g`r)6lF68#$2J->GYjciSpUcC#GW}fW z89fVemH4+ofny))Igfc(-eqe!%CV2OOuyTFhVLi9KDVG@ZolUFEH|tj55Yrl3f5-K zXEc=Qee>+s*5iK-pPW0!SjP%;YyLBSfB!gQe!w^yu}0v#5A4VM0iVy!`})GXhP5;O z{&x)I+V8lhM9X|e#Ozz1T5I!pKLw77cc3Zj=#=MoUAV7)t|8w(AL9EC7?^uj#s@US zi};@gp09lOZ4a>m=L3A^ZNVdO#9F`4`ZnvAT{F6N4{FM7zyQX*Gj`IVpE8$!!gw%qjQ4OmV4pq@V~5}rct4D3ZJF-P6Zq`sgt))w#f+=*U3^c1kAa-E zxd(fAiP%-JVXc)(10Z zU~QiL+D{Koz;mn{Vrv_pA%5!*aEy=V$M}G?chZ>Wtr;82^bYC&3$O*gzdin#_58k^ zF}mjt9J4Mfj(f^_!0$R|=lCP|FM@)%k}6JpjZXs^xOBjO$Y;9y>F8Bd9Oo~d2n?Tq<3@%O=P@DSeG{H|>9 z4e+cl*WaxQYpp92Ms4Pvx9`sBoYOG=IcxjSl0VN z3Fyo8tu@{;_g<{ev1Prs0nQ0J_FAOyI>E-H=tW-bZ8v2 zb`%9P0n@b>%kg{B&he3}b2k0{@AnOi`|QnY)9>%z-+lHB@luG){p{nY?VcQaur_1)pc674jc2Kc3f!ew1?xl?K6LlN53hLGx|B+%ysHJyTSh^9NzUcr36U4_D^R@Y0 z?yFq@WAC`OviyAi6ph0AF~jH22mi z#17}2?iw7Ro*cgwr@(Ku&)~0#{Q~?5q>fz8=j87Q2cUqSwVbd>}SK?u0eu)1ZzrSy;wPnfO;J3ze;2f^}SWo;O>pS{D$GR|j-ujKVAlBjY zXR3GA9Alz@2E27`I-l|9D08hX)A_gX2j=#8!2Aek`qX+7zVFX_V2t74nR7VS4g7gA zo+o~_Utz5k=H8R=JU4UOVsL_I;(K5pJ1guLpE@mb5p#YOznWXM*h7bJ)Mb4a7;E)m zPcgp^ykDN>5g2O~g24Cmbx@eA)iOGN08h<^*qLLjOz);Wcg)8g7-I#lcc9=?&rzLI zurUU#=Q$sPp15dc$dPZqni`&M2h^75o=%v*#pqrSf%}eiaI}Z`jQjV9y;i1wAO37l z+&p_3&x9U4hheVq5_99<2bV!XXKknO?A1Ph1fBp}b48gn2S?zy_-}!G9DUYe{t%xv zIyl03&!{;X8=nz#d(?(9ox=&SWB8*VpXOey^}RXf*(6@d#P@#ax9@>bZcl6g)=wSe zKKC6p{xsanVEnntxaP|5;k<$0zApjwYAbMj_r49}sac!49lR^V?xArG3h41YgU@{y z=CLEzH}HE`4nR{9nR@J5pTY6W>?>ydtE+YVj@I_wyh~mOjvWgD;Cw^OK83xh^(xTs zI@ZH?zkd(B@8-tLS+C=7%cQp@UYM`VzJ3P&60FbP`F8F@-2>)@T<_2L92if4&)+^g z_xsGfFFm}N=e+^$`w+MneNCD4)b?F#^danH(gIU;s! zocY1bHLoqx_v9-60^G-dubDXmWAr~D{%zoXdO2VK_J0aG=4Bp@6=2=l*QS2$!}s9~ z-!X{!{qw)$F7C|v?nk^x>=EP9isQMOzXje?=XZg*|}%YB~`yC)A{2aGlIe0y!u zwjX=mGCqOt3V0o~@XE~Zi9Mwc9Dt7X01DWG2J~>9x31y1R(qq5`2Y@>>+gW){EKVT zzeDM3!6Cj=a59fSCH@28xeapYvoQ9+b9xTHImS9*ee>P3XK@qUVJ;_eIS0Vn;x0I1 z-GBmKV*M15^DN_QK;B2do^LVAFVs`_J}~Dx=$RWEh?(bExo$vN@gK2P(>~Qa2Ih9a ze@8G@fIV!1dp%_B^D)+f2E73oQ_mi(sfIEAqJWll>^|!c;60dQJ>wPDt!sE{-D53p z0~&AwZi6kjL(IGBx`m&(dniEvU2ql7Lq_+Y`Rv3{roC>6Tg$y1F`Byt>e$-=E)ny4 zOypuu+v8shgE2Ys{sBR=!nGQs>F9D%3s8~+TPgFE=VOTO1?7Djao_;co- z>nq?xU@iAo&=t-Hpys=vW8Q*~nV&E=uHkhvT;C@4J+Ngy;6G$MVr>uhB?f%%NsW8p z_1S;M=v`39xMy>l@c`UqF24Zt9b5Dp&;#e5-!cAEIkUfGJZJq7c#oe3$3Xo;Ean;f z2{5kzkoX4Ji@KNbpMs8<^B(_Q;C_#0j=U{%e`XfO6VM5Ag!PT-i^=!zBt7^`@T76L zF*Ut2Z{oM-W1xopwP3?sTmn6L;qjrez7sQ?i=B%wN6W6%!MZ-Ea)^iPx$QyuTVVz#gvv2n&PaAtI)4TGCF(A)9 zt~q7aDsYVpe-(yT|Oy_;#+VW?!cF&$BoJ4eQYuJ;(I% z6tM%=j(!)M_i#Sc4@P7CedgnSXIK0UeEU6PT*q5-e5T%;b7HXp|7Cpk{45xV-2q3y z+Na>*yl&v`&Ih0O%y?}h2mhTkd~xJSHY)V7S{vzlY-A2Kgx;n@l_};PjDH3ky`$q<5Kj*cbN6Umxc9pi=Mc;K5w6~dF`BIX3zXNe8%4Z*Ma8q@;hPA(bDhS-k%{h z*4J71A8;SPKku%BeRhoB`W@Wb)curL?w~c?{{ZBVJutfNfqV7eabE`m{#aq&&#)iQ z*LUxEu+OY>pV&RdwVmSY%j9vN$Dm_vZTI&Od;~6oLwxq9l`?&&TgHOl`zE&+_?m#J zX5Rb6egfJuy2d|#%(|J^V~^-qTg{htpbk;j`uz&Sn)aXf(h(qy7n4TYB_N(_06>wj(s3f#ZEF#FMv=>$%#) z$f<;Sa{mYZuCp&D)!&``JK?^#xrfFWh4*K@SJKBk?~7;T^Qd(SdDib4>zMgBu1)cd z+H{e-+pWVT4joPefuF&@N z;N3RfmYK)6Ht)=5aQsF_XB~H5%KUF=b%3?`m&xtm^q@9wiRSyxZ_&J#){^`oj2AAZEWWxCT!GfA+}p zOouYDHq1MGV;*B+-Rg7XjLq}Aa{^*Hd*3Pdt){$j-zBW+?=Z&R1O+8z2HsAkjs9{~(fX}}l+hfmMbAAdo;6AuS+?w{&F&`X7 z8R_F6a*VqdbB=(PJZn6}=UVKnOux5$K>P@3Exa=44`u4b(fe^5|1t1cpUcCqZJGD@ ztn(a@*OoCfa`gLm-JgSjSZ7QySI-F}5;ueO7OQmKxU?t)z4KOlY%-X{KLIB1(^+?aRf z5FE^L>s(@PUGsc53S6z;lfF-g|1(h2c+DB`8$Sjg0PjU96K~7>c#g~YF>w3^(BFYU zE@yZ=r$Hg+{+u7oyhBEN z6nDk!S3~}Fe4gzF_`W* zW7Jm|dq>bS|BP6}*syMa`k#O^@ESOs^I(j&VJ+u2@VRbTTQg>F&*3ix4{*P7U3*s6 z>YalR@Q=MQ?`E{bkMX|*PJw6M5$i!Ab|0JrdHP39W8lWM=~>yM>vr~=_dGGbL*5PJ z4KdC9Q`T?J@k2)U(-nfC750sp_cn1k2h7ziWxAu|8L;kI3(w$Ha2@pEuYliJ>v)Hb zKm$B4d$PXNW{vg+zkA=snLhx>K(2885cJFoaP&QD7_}o}o{v21y+O>s_o#cB`S>iU74zqv zIn5mV*Z!Sh6>rMayZpkuz&vU$Qt(95-&w>3O zFv=bGJ99Q_m8o{mcm&72#EkhKd1SPLe1=agd9`hcy$Id~^2S`a1=z<4=+N{oDo(K7U@cYaVW` zFVk5*NxUaMu-4i#o%JjDp8#4Reg;aJ;CILOtzoTswj1JY8JFWX;LSOH!gvYzE*USt zQR~X|P4x~I;QQ6%iw&9M-+&GrgC>5aJ@eKzu+I;!E&Ah~W&JGh?%V=Czt|!3pW(B& zW8mG=rfgefQP%-i>!;&TY$duIH>b;1I~W4$s~^*8$F2*L?pz_7EI{LR|BC z>@zhV%(`v)JY!(}0eRN!m|LgFaSrqg_q{fG{jBMCMH|XEJoblNW1eko`t93u8IASY z+^5gCulzZWQ#9S5rtio(NPG;;f9UIo`Ew#U>$qCR-z0v*SldvhK76<3)#mR8%(tLr z?eD+o!#Z2mev`}@n^4C_%^uGHY?U^^gvmbe`F*S^NcW)4L-hfsRD~$cM>Azoi2TsSjnK8zk6ZhxZ z`=D{Hl?hkz6+y1=*1_=}{=?Z%JfPn5 zto=Sb0&-gTsnIdl=d)S+oC}|qaW%BAOuzH$Z<$-?&^g?mwfmFr-EZ-)tq^|&f6Vxk ztZxAOGp==I`ev%-vppxaS!t>5d7a@qVl41u9dqj+5!;#rvM)JL6I+{Wds5$=uHp}6 z`W>pr{{lSknAoz8Irm;1;C~yu0X(0k%nU|*D2#`zJlEfY`}e?SGla)nzH9HTIGW?` zS^fh?tt+ejG538pEI5q$~FxxD4!J%pKxA2TbZ(!}+8TUneG)GaU8fVPaEc4pB1HRoF z{~_yR&@%6Udd@X@#6>o%X4rF z4&?xIULxkGu6%ht`3=}GmnYw6wPo}fk8$P?n0uG+F(1!>97i!Q_bh%kL!Eb+>)#pk z4L*NP$om+y_y$mbIR`*}>l8-)M~u;Z$XwGu_Atjjfp7g2)^~vM+pJSdUrfCRK+P?% zZc`GS#z*+N+1E3^0`C&g=UQ0HwP*Socp8l7fv*MLX>~qkyaIMctOwTiJk;IV!^)0X ztF>Vi?%SR(!!3+$na0A{fdR<#y*>p8%nxT?v$7*%?%SHTfH^xWlV*qS8DbslMzHp- zJ%!KQC&9f{UV)>Y@4j)*N?qsXYNbrtw^g}J%+8STT~wNmEyne_qqfY=>I{Vh0g4n||t*MM;xozoxg!#dVC zUR#gP|Npc5VshO_C*bWf_c{>qB8$|&Um*cSL$Wr zLz(s{&pYs#5kDq>+~=OS{txgw+T+>|=e(EAt77tw@Tu*+@t$}d`z$&S_Qbm~{r3Z` zzuglb@Od})?|rt!&%h_w_CF(^%d^-1-8aASjo%-o|Mx%m z{clV)u6}JAo94@zH2>inH*fyIzjyhE|Na}_xHbB}d~h(w|7G6SzcSNSdcS+~kLGH> z`+K);edC)lzWmk4d8@j=bMw}J7ztnd7xn(|5B_LW7tw@0Ys! z^j3A3d$qf#yGgUs+tt0gdmamK-?@Bsw~vLls=L$s>aP8Tx6)hs+r5qpZ{N9G-Gz6P zz5Cd0qgi;X{+7DGo%^@&mRqvSzPbzVCe0FE-jAz$_Oasa>V9n}vvj&VkL5mpb*H!R z_UZlB9=#QBcmKY++wVTTrS9@QU3jbKwtK!`-o2L#Z+UJ@2VdXamwmkD{C;b9U-t19 zy(PNLmb%M#--^xAMHM;j+eSLS%3vW4>Mfu9^oEP4zzuog#zPk%= zxd)5hS9bT{U*_#T&&|SHRF>KDd>7unx>t9C`*_Q9yRwJ3R9l|M*LU}2(OY=S{rlQ3 zyn8U~pXbZ{JIKK-yw!WY<9%)SJQm*aJXUpg`&f93-ojgcKUjFn`7P)2mEC<_cuU=d zx7@#lx7@4co-X(A3wV3ZZ>6`K%bxu%`&gdL)!jaJ53pJEmUG$3HT!tWJ^0!#yqm0^ z?^5^6JHdswM3?jX%I-M+W!`V?;Vt`J`dfYuE_$oywmZLlddq!TI^D-x&TrYr@|9ZZ zF1+QOm;16ip!N4^_kCIJ-@;q;mb%N&^S!*?&#kZT-lK)LI+tDD-Dk2{c+0sg_j%tw zmj0IW`|_T;3vb!S*LUT1lhwJb&TrS?!rR;5?mb#~OIOQ%S$+=g8eP?0_Padag}3Zu z)n7hq&B9xq^X~qAWhc1sR{PjJk1y}hTX;*|uk7}_=q>#%`~BLUx=VkH-tzn3qPLvi z!u##r_hsQN{Vf^G-=p^NmV2=Jd(ZASfJJZF$MXBo@|(i4kEQO?-@;q;7T)r6b>Z#% zzAS%V_?LOh?@bGDsk`iB`MxZ?rK^Rv+=GR;?031R%kL!%Z{I$a@AJa@jpko9zu~V_ z;~x%BkH1{~#&1lH{OvR2Z#J8ex_N?s)f|~+{PSP2GUH2M`qG&A#H{n1ztR0)lhXD# zN8;{}lilpslV6{}uTS9DC-CbN`1J|=`UL*#p1}08dH3T#zXtq29=|lV|En7FOT({f zyfpXss~Z2+toN%L|GQc5S2h0kv)-?2oPOua?|%R9$A3Nd_p2KJ&8+vU8Xy1t_}Q_- zZ%n`DHu=3c_3HJ1`R~j952tss{IBy?=Ar%k^1spQzy5hNfAjx`^T>OX`;qm3`TZ#W zasFoi&!4{n|KGSD>FL%t{_vLbTi^Vh-}68J{heF4zWIB<`$xDZ#_{ib^Bcc^>kpgX z|HkkAyH|hr5576Ov;L!R{vW@O!tOu(T=~EGjhnyo+Ac8X!~Z7yZ~or@Ip^f(+Vr+g zHTV7b%iibY=J{p)oBTc3TL07bf7|_i_{zU+uKefu2iC=ZFaFQ7MJeNJnf!YE^$Gm? z1pXVJKz^?0_s{&Em)}?N9ltd9ncuVW`&+(4`F;2zy;>*w(Mx{xqMtGOH7|WfC%Wk) zeP>_sColf!haa8jrf&L4pUI1EotHY{CqH$fn?9nGb@IcHe(I*b=tnPgq8FX$Mkl(_ ziS9*u;YTO>(Th&%rH_krYk%~j6W!>fe)O~7^b@`0)q2s%{=P^r`KgyW$xEH|oBGjD zz4RGg{53yy!msP3pY#!a?XUAvH}#@l_Zgk&W`C&{-C8d?(arwCi%$H}PhR?pest1L z>eu|}gdhFXOMd!@e)JNLPW&}Ldg(KD(ogC~C%Vx~-RML=deMn)^s){=`H9#0sdG^u z=_~Q@(`WQkuhvgLb^YkYpZ%m>bdw+bi|U7$`ZYiG>N=^P{X{={;YTn0)T#STo#;g; z{ODYi7yZ;tKhcX$bQ2Fh=axF@C%UN{y;>*w$&X(2qZgg%)_S#0`b=JQq8pvmtM%fK zZgkRT^e@t__3Hdu|CiNE->IAavajgooYQaWr%w8-`Oyn6`O#0^^by_YWu3g}*ShKN zBHiePmwsx!`*Ndf{dN(M#RzCw;{q-T0#yesrQ+ z_mzIWNH6)dZt`p0^by_kmwuy{{-S%4Ud|`{S|{~uos0U2&PDaoU+UC4sgr%Bew`Qn z@S{`jr#`28oqlVb^q0EnBl@YIb$IcIU+bn`^y_}Y4=?fXQ!o0doBgML&M)~{M?d`N zre5@-AHCF#UU=~*KYFQC>!fb_$r#dbo#-b&deKij{^%w@ zywpwKsTbY4kNBgL{M3t1>ZMNlh+cTf4?p_ROFzj^-RMRy{O}S_e)8%%sh@tLSND

&q;Bd&zt*ekMJIj!vU=$^eMUF+v##r>uk;(8 z)K5R@BmS)G{P0t^u9rSiCw0S1e&W$jUiKILx}U706aDm?yy&KG^iwx=lAm?*!cTr( zH+7Vk8bqRPxMkhy5S{1x^=zi)_Td0UiwR&=tQ^HN!|1pesoi>){S23 zMz_|BZsO4kFLhEkeWzaZQ!n|6ho5+Kk{A8dPu=vDF}gJ``KeRur;pUnI`!+k=%r5b z(pUPZ`wB05(T`sGif;OdZuIKB@RA?>)JuMJ>;A$|e)OYzkxu%q^}>&C;?YZ9>ZHHa zPoLRu_=!g+y2($y)Q^6xmw0rO7k=s_@1lC?FZGj`zM_|U(Mi43Po3x`KkLL(CwZwC zz0^tF)C;fnM<@BI8=bmt`bj+c$qzquq95J#k$prjdGUvzcyyAVda08>(pU1LQ|n)( zSL;MS_0nJRQ!jbpr_Z{6`bu8;I;j(W^inVN(^qujkACu^ zm;B_VUh1b_{K=1Q>O?2HsTbW^FTCj1yy!(g^`e_T>i&`!et5}?Uh3C%qnCcek3Txe zkAA(6)K5Pb>D4;Xk6!Ylmwq!QzviXy=tMVtr0?u2{^Z3U{qUm`-PBD#=`(rJt@BbR z{N$%jbkj$4vQB>Z(NEp<7yam^PV}M^-RML&I?=sIFZ}34KYGzgz4URBZtag=bfO!b z)Q^7ln|`8~yjm|h+20rGB|r62CwZxpep5gCsh2*(i@)ZlPWW}5^pifqul;pi>ZV@w z>pr6s-Rv**qFd`lC%V~Rc+rVJ`pHXQ(T`60N&T81o$#ZdddW{8(T`r@(TTt2M=yP* zPWnmx=tMVqsT-Z>M=v_jjb7H_CqMBzKXoqZBYhec$`r>-Bp__LqXi*E9x ze^LGLQorV>UR@{kv!CckFZ}3*pE`BFsS~~Egdd%Y@}i%*=_h*8iEiTI=iE{!{X{o) zqgU%hKl#y%e)OUf-CD2KNuSA!PIRM_dbM8s(Tz^}jQ&NswO*ZH>;JNP={t4PU-lK< zoOAk3{nSZ+H9vacB|rM9n?9l&y{wZL{aQEuU8Ebm@X}AMm;BUA{q&c9qn9!M@GsJj zZuC+weMP5^M=!kWKYFQ~{iLt>qZ@zp!jDdL>%P*@7wILx)=hq`n?9nO{?c#s(qD8h z(#!dTU+biPt#eTy(YdHz`b(W!Cv~!~)UWfRAAWS|{nY1FuhVa>lm1dSeMCR?vkou* z@N3=Fi+*$9c-PDU-^rM%$(F-sBZERX$xl4` z$;!wchQa|hHM7P#U|Iv+J`iWlZM>o9W zN4KsQ-C8gC(Mx}+6P@VRI;orf!jEq1)w#h;iXRMrtj2?e(EJZ@$eIm zPV%Ck`l*|~GDf%NB|mj){q&LgS*L!T7roR;UiwNObzk8{FZ$6-U(rn;(T!f67hdwC zpL)rUZrxw_$&Y??FVachwO;tqO+0$ZOP%zW`sp+K4L|YdL^t`Vm-^AK^%9S6^1@G@ zLf4qqL(_Un|k5Z{^%q>b)!?)O+Sf8Kl$OO zPV}RjKC+MKB`^N)6OT^vQ!jPWNBT-$bZY&J^lF{xr(XI?e(EJJ{PbDZPhZJPU(u`U zB)`^8z33%Byy&Li)Qf)li%#l9FZs!fe)7Xle(FXyb)%R3=!KWO=tMvH=_B=`SFe+o zI?+#l^y+%)=OVr2M=$!(iEjFjUiu6_I^m~Ybkl!yqF?LP`O!_CS}*yvZusdVy!4Se z(NBK#(_i+Pb##&+{p6=^>O?=f(F;Gk=%!Bi(Th%9H+tbEzwRq_qMLfPZuAmQUiyuG z>P0X0(qD9=7k~JPM<;c{k6!Afe)@_|{LxQd^pc;v)Jy%;i$D3%O`YgOH}#@h>xCEH znisw3r(Sf^N8MlY!VfQb(M$chZuG*-I{DG7`%b;ouk(`^o#cm~{PdHtjz>57(Tjfi z4KM4&!%u$nQ!jngyyzt!esrUgdKr@!UacFwLow@);h^g zUUZV5c=F?qZt~Js@>3`A@DqY$&O4g>eTtki_RDIk^J;k_Zyx1`9wGQ zH81h>ll@0O^^zZ6bdsOCUz8tS@}iTx=%;S_Nd1iQ*E)56bW=BZ(N8=&;YBC?)pfH@ zUU^cS7FUi{HZUaglp$**~}PUUi=} zKf0-x{P?32{p5uo{pcn?dEq55y3xBRKY7uMesrUk{95;y&9D22PU>WxI;op}l3&+L z-!(tH=tnnoq8I(}heddW|ntdkd=^j+ttUY#HP z@RMKb*7?axAE}?b=p--tsUKc+!i!GwqhIGoCwYlSFZ$7oPIQx(c>Ku=KYGzm-Hg#m zJp5Xxu9J1@grEHM6W!FQ>xLJdZES?$&X&0U*|=y=0`XD@X|->q;B}B7rpT6{OBe> zdEq5Lx~ZGI=q5kB=p-I~bfXjfpW;UzCR$%}sS>vi;^le}6l_3HfOMJIXDNk7R? zAK`}=-RLDRdUbxC7ro?%U+aXIdf`Pc{OF{9`p&w}i(c|mCpyVbJo(WJFL~)Fy2(#G zd5K3iI?<0#^5YLbI?+vjbnE=A!%JTDlNa68N!{edAH6y+{8~Tj@S_vm)QxU*(qHo9 Y4?p?QOMdFq`N<1EyyQhMb&~&o1685$iU0rr literal 0 HcmV?d00001 diff --git a/tests/utils/dataset_helper.py b/tests/utils/dataset_helper.py index d8de935a9..c13a27b39 100644 --- a/tests/utils/dataset_helper.py +++ b/tests/utils/dataset_helper.py @@ -193,6 +193,12 @@ def _build_data_set(self, context: DataSetBuildContext): # file with distance. context.vectors.tofile(f) +def create_parent_ids(num_vectors: int, group_size: int = 10) -> np.ndarray: + num_ids = (num_vectors + group_size - 1) // group_size # Calculate total number of different IDs needed + ids = np.arange(1, num_ids + 1) # Create an array of IDs starting from 1 + parent_ids = np.repeat(ids, group_size)[:num_vectors] # Repeat each ID 'group_size' times and trim to 'num_vectors' + return parent_ids + def create_random_2d_array(num_vectors: int, dimension: int) -> np.ndarray: rng = np.random.default_rng() @@ -239,6 +245,35 @@ def create_data_set( return data_set_path +def create_parent_data_set( + num_vectors: int, + dimension: int, + extension: str, + data_set_context: Context, + data_set_dir, + file_path: str = None +) -> str: + if file_path: + data_set_path = file_path + else: + file_name_base = ''.join(random.choice(string.ascii_letters) for _ in + range(DEFAULT_RANDOM_STRING_LENGTH)) + data_set_file_name = "{}.{}".format(file_name_base, extension) + data_set_path = os.path.join(data_set_dir, data_set_file_name) + context = DataSetBuildContext( + data_set_context, + create_parent_ids(num_vectors), + data_set_path) + + if extension == HDF5DataSet.FORMAT_NAME: + HDF5Builder().add_data_set_build_context(context).build() + else: + BigANNVectorBuilder().add_data_set_build_context(context).build() + + return data_set_path + + + def create_ground_truth( num_queries: int, k: int, diff --git a/tests/workload/params_test.py b/tests/workload/params_test.py index 6deb3161f..b5bbe7473 100644 --- a/tests/workload/params_test.py +++ b/tests/workload/params_test.py @@ -37,7 +37,7 @@ from osbenchmark.workload import params, workload from osbenchmark.workload.params import VectorDataSetPartitionParamSource, VectorSearchPartitionParamSource, \ BulkVectorsFromDataSetParamSource -from tests.utils.dataset_helper import create_data_set +from tests.utils.dataset_helper import create_data_set, create_parent_data_set from tests.utils.dataset_test import DEFAULT_NUM_VECTORS @@ -3119,3 +3119,228 @@ def _check_params( self.assertFalse(expected_id_field in req_body) continue self.assertTrue(expected_id_field in req_body) + +class BulkVectorsNestedCase(TestCase): + + + # TODO: figure out how to unit test the nested cases. + # basically create a nested field list with different vectors and partitions + # + + + DEFAULT_INDEX_NAME = "test-partition-index" + DEFAULT_VECTOR_FIELD_NAME = "nested.test-vector-field" + DEFAULT_CONTEXT = Context.INDEX + DEFAULT_TYPE = HDF5DataSet.FORMAT_NAME + DEFAULT_NUM_VECTORS = 10 + DEFAULT_DIMENSION = 10 + DEFAULT_RANDOM_STRING_LENGTH = 8 + DEFAULT_ID_FIELD_NAME = "_id" + # NESTED_FIELD_NAME = "nested.vector_field" + + def setUp(self) -> None: + self.data_set_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.data_set_dir) + + def _test_params_default(self, bulk_size, data_set_path, parent_data_set_path, num_vectors): + test_param_source_params = { + "index": self.DEFAULT_INDEX_NAME, + "field": self.DEFAULT_VECTOR_FIELD_NAME, + "data_set_format": self.DEFAULT_TYPE, + "data_set_path": data_set_path, + "bulk_size": bulk_size, + "id-field-name": self.DEFAULT_ID_FIELD_NAME, + } + bulk_param_source = BulkVectorsFromDataSetParamSource( + workload.Workload(name="unit-test"), test_param_source_params) + bulk_param_source.parent_data_set_path = parent_data_set_path + bulk_param_source_partition = bulk_param_source.partition(0, 1, should_nest=True) + # Check each payload returned + vectors_consumed = 0 + while vectors_consumed < num_vectors: + expected_num_vectors = min(num_vectors - vectors_consumed, bulk_size) + actual_params = bulk_param_source_partition.params() + expected_num_docs = len(actual_params["body"]) // 2 + + self._check_params_nested( + actual_params, + self.DEFAULT_INDEX_NAME, + self.DEFAULT_VECTOR_FIELD_NAME, + self.DEFAULT_DIMENSION, + expected_num_vectors, + expected_num_docs, + self.DEFAULT_ID_FIELD_NAME, + ) + vectors_consumed += expected_num_vectors + + # Assert last call creates stop iteration + with self.assertRaises(StopIteration): + bulk_param_source_partition.params() + + def test_params_default(self): + + bulk_sizes = [1, 3, 4, 10, 50] + + + num_vectors = 49 + # bulk_size = 10 + data_set_path = create_data_set( + num_vectors, + self.DEFAULT_DIMENSION, + self.DEFAULT_TYPE, + Context.INDEX, + self.data_set_dir + ) + parent_data_set_path = create_parent_data_set( + num_vectors, + self.DEFAULT_DIMENSION, + self.DEFAULT_TYPE, + Context.PARENTS, + self.data_set_dir + ) + + for bulk_size in bulk_sizes: + with self.subTest(bulk_size=bulk_size): + self._test_params_default(bulk_size, data_set_path, parent_data_set_path, num_vectors) + # test_param_source_params = { + # "index": self.DEFAULT_INDEX_NAME, + # "field": self.DEFAULT_VECTOR_FIELD_NAME, + # "data_set_format": self.DEFAULT_TYPE, + # "data_set_path": data_set_path, + # "bulk_size": bulk_size, + # "id-field-name": self.DEFAULT_ID_FIELD_NAME, + # } + # bulk_param_source = BulkVectorsFromDataSetParamSource( + # workload.Workload(name="unit-test"), test_param_source_params) + # bulk_param_source.parent_data_set_path = parent_data_set_path + # bulk_param_source_partition = bulk_param_source.partition(0, 1, should_nest=True) + # # Check each payload returned + # vectors_consumed = 0 + # while vectors_consumed < num_vectors: + # expected_num_vectors = min(num_vectors - vectors_consumed, bulk_size) + # # actual_params = bulk_param_source_partition.params() + + + # actual_params = bulk_param_source_partition.params() + # expected_num_docs = len(actual_params["body"]) // 2 + + # self._check_params_nested( + # actual_params, + # self.DEFAULT_INDEX_NAME, + # self.DEFAULT_VECTOR_FIELD_NAME, + # self.DEFAULT_DIMENSION, + # expected_num_vectors, + # expected_num_docs, # todo placeholder... but we expect 1 document per run. + # self.DEFAULT_ID_FIELD_NAME, + # ) + # vectors_consumed += expected_num_vectors + + # # Assert last call creates stop iteration + # with self.assertRaises(StopIteration): + # bulk_param_source_partition.params() + + def test_params_custom(self): + num_vectors = 49 + bulk_size = 15 + num_vectors_per_doc = 10 # 10 nested vectors per document + data_set_path = create_data_set( + num_vectors, + self.DEFAULT_DIMENSION, + self.DEFAULT_TYPE, + Context.INDEX, + self.data_set_dir + ) + + parent_data_set_path = create_parent_data_set( + num_vectors, + self.DEFAULT_DIMENSION, + self.DEFAULT_TYPE, + Context.PARENTS, + self.data_set_dir + ) + + test_param_source_params = { + "index": self.DEFAULT_INDEX_NAME, + "field": self.DEFAULT_VECTOR_FIELD_NAME, + "data_set_format": self.DEFAULT_TYPE, + "data_set_path": data_set_path, + "parents_data_set_path": parent_data_set_path, + "bulk_size": bulk_size, + "id-field-name": "id", + } + + # todo is it weird with the parent data set path? + bulk_param_source = BulkVectorsFromDataSetParamSource( + workload.Workload(name="unit-test"), test_param_source_params) + bulk_param_source.parent_data_set_path = parent_data_set_path + bulk_param_source_partition = bulk_param_source.partition(0, 1) + # Check each payload returned + vectors_consumed = 0 + while vectors_consumed < num_vectors: + # expected_num_vectors = 10, 30, 10, 9 (15, 15, 15, 4) + expected_num_vectors = min(num_vectors - vectors_consumed, bulk_size) + # expected_num_documents = min() + actual_params = bulk_param_source_partition.params() + expected_num_docs = len(actual_params["body"]) // 2 + self._check_params_nested( + actual_params, + self.DEFAULT_INDEX_NAME, + self.DEFAULT_VECTOR_FIELD_NAME, + self.DEFAULT_DIMENSION, + expected_num_vectors, + expected_num_docs, # todo placeholder... but we expect 1 document per run. + "id", + ) + vectors_consumed += expected_num_vectors + + # Assert last call creates stop iteration + # TODO: It's actually failing here. Figure out why the StopIteration isn't being called in params. + with self.assertRaises(StopIteration): + bulk_param_source_partition.params() + + def _check_params_nested( + self, + actual_params: dict, + expected_index: str, + expected_vector_field: str, + expected_dimension: int, + expected_num_vectors_in_payload: int, + expected_num_docs_in_payload: int, + expected_id_field: str, + ): + size = actual_params.get("size") + self.assertEqual(size, expected_num_docs_in_payload) + body = actual_params.get("body") + self.assertIsInstance(body, list) + self.assertEqual(len(body) // 2, expected_num_docs_in_payload) + + # Bulk payload has 2 parts: first one is the header and the second one + # is the body. The header will have the index name and the body will + # have the vector + for header, req_body in zip(*[iter(body)] * 2): + index = header.get("index") + self.assertIsInstance(index, dict) + + index_name = index.get("_index") + self.assertEqual(index_name, expected_index) + # here, need to iterate over all of the nested fields. + outer, inner = expected_vector_field.split(".") + vector_list = req_body.get(outer) + self.assertIsInstance(vector_list, list) + for vec in vector_list: + actual_vec = vec.get(inner) + self.assertIsInstance(actual_vec, list) + + self.assertEqual(len(actual_vec), expected_dimension) + + if expected_id_field in index: + self.assertEqual(self.DEFAULT_ID_FIELD_NAME, expected_id_field) + self.assertFalse(expected_id_field in req_body) + continue + self.assertTrue(expected_id_field in req_body) + + def test_nested_vector_query_body(self): + # assert that _build_vector_search_query_body returns the correct thing. + pass \ No newline at end of file From 95a677277c91c498b698d794c484ad9ae3c890c0 Mon Sep 17 00:00:00 2001 From: Finn Roblin Date: Mon, 15 Jul 2024 14:22:12 -0700 Subject: [PATCH 2/6] fix formatting + unit test VS nested query Signed-off-by: Finn Roblin --- osbenchmark/utils/dataset.py | 2 +- osbenchmark/workload/params.py | 9 +- tests/workload/params_test.py | 182 +++++++++++++++++++++------------ 3 files changed, 121 insertions(+), 72 deletions(-) diff --git a/osbenchmark/utils/dataset.py b/osbenchmark/utils/dataset.py index 4786c22d2..210b00a2a 100644 --- a/osbenchmark/utils/dataset.py +++ b/osbenchmark/utils/dataset.py @@ -141,7 +141,7 @@ def parse_context(context: Context) -> str: if context == Context.QUERY: return "test" - + if context == Context.PARENTS: return "parents" # used in nested benchmarks to get the parent document id associated with each vector. diff --git a/osbenchmark/workload/params.py b/osbenchmark/workload/params.py index 8f99ad518..e60733abe 100644 --- a/osbenchmark/workload/params.py +++ b/osbenchmark/workload/params.py @@ -13,7 +13,7 @@ # not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -1207,9 +1207,7 @@ def __init__(self, workload, params, **kwargs): self.logger = logging.getLogger(__name__) - def partition( - self, partition_index, total_partitions, should_nest=False - ): # TODO should_nest is a hack. + def partition(self, partition_index, total_partitions): partition = super().partition(partition_index, total_partitions) if self.parent_data_set_corpus and not self.parent_data_set_path: parent_data_set_path = self._get_corpora_file_paths( @@ -1349,7 +1347,8 @@ def action(id_field_name, doc_id): size = len(body) // 2 if not self.is_nested: - # in the nested case, we may have irregular number of vectors ingested, so we calculate self.current within bulk_transform method when self.is_nested. + # in the nested case, we may have irregular number of vectors ingested, + # so we calculate self.current within bulk_transform method when self.is_nested. self.current += size self.percent_completed = self.current / self.total diff --git a/tests/workload/params_test.py b/tests/workload/params_test.py index b5bbe7473..ad42802d1 100644 --- a/tests/workload/params_test.py +++ b/tests/workload/params_test.py @@ -13,7 +13,7 @@ # not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -3120,14 +3120,13 @@ def _check_params( continue self.assertTrue(expected_id_field in req_body) -class BulkVectorsNestedCase(TestCase): - + +class VectorsNestedCase(TestCase): # TODO: figure out how to unit test the nested cases. # basically create a nested field list with different vectors and partitions - # + # - DEFAULT_INDEX_NAME = "test-partition-index" DEFAULT_VECTOR_FIELD_NAME = "nested.test-vector-field" DEFAULT_CONTEXT = Context.INDEX @@ -3144,7 +3143,9 @@ def setUp(self) -> None: def tearDown(self): shutil.rmtree(self.data_set_dir) - def _test_params_default(self, bulk_size, data_set_path, parent_data_set_path, num_vectors): + def _test_params_default( + self, bulk_size, data_set_path, parent_data_set_path, num_vectors + ): test_param_source_params = { "index": self.DEFAULT_INDEX_NAME, "field": self.DEFAULT_VECTOR_FIELD_NAME, @@ -3154,9 +3155,10 @@ def _test_params_default(self, bulk_size, data_set_path, parent_data_set_path, n "id-field-name": self.DEFAULT_ID_FIELD_NAME, } bulk_param_source = BulkVectorsFromDataSetParamSource( - workload.Workload(name="unit-test"), test_param_source_params) + workload.Workload(name="unit-test"), test_param_source_params + ) bulk_param_source.parent_data_set_path = parent_data_set_path - bulk_param_source_partition = bulk_param_source.partition(0, 1, should_nest=True) + bulk_param_source_partition = bulk_param_source.partition(0, 1) # Check each payload returned vectors_consumed = 0 while vectors_consumed < num_vectors: @@ -3183,7 +3185,6 @@ def test_params_default(self): bulk_sizes = [1, 3, 4, 10, 50] - num_vectors = 49 # bulk_size = 10 data_set_path = create_data_set( @@ -3191,66 +3192,31 @@ def test_params_default(self): self.DEFAULT_DIMENSION, self.DEFAULT_TYPE, Context.INDEX, - self.data_set_dir + self.data_set_dir, ) parent_data_set_path = create_parent_data_set( num_vectors, self.DEFAULT_DIMENSION, self.DEFAULT_TYPE, Context.PARENTS, - self.data_set_dir + self.data_set_dir, ) for bulk_size in bulk_sizes: with self.subTest(bulk_size=bulk_size): - self._test_params_default(bulk_size, data_set_path, parent_data_set_path, num_vectors) - # test_param_source_params = { - # "index": self.DEFAULT_INDEX_NAME, - # "field": self.DEFAULT_VECTOR_FIELD_NAME, - # "data_set_format": self.DEFAULT_TYPE, - # "data_set_path": data_set_path, - # "bulk_size": bulk_size, - # "id-field-name": self.DEFAULT_ID_FIELD_NAME, - # } - # bulk_param_source = BulkVectorsFromDataSetParamSource( - # workload.Workload(name="unit-test"), test_param_source_params) - # bulk_param_source.parent_data_set_path = parent_data_set_path - # bulk_param_source_partition = bulk_param_source.partition(0, 1, should_nest=True) - # # Check each payload returned - # vectors_consumed = 0 - # while vectors_consumed < num_vectors: - # expected_num_vectors = min(num_vectors - vectors_consumed, bulk_size) - # # actual_params = bulk_param_source_partition.params() - - - # actual_params = bulk_param_source_partition.params() - # expected_num_docs = len(actual_params["body"]) // 2 - - # self._check_params_nested( - # actual_params, - # self.DEFAULT_INDEX_NAME, - # self.DEFAULT_VECTOR_FIELD_NAME, - # self.DEFAULT_DIMENSION, - # expected_num_vectors, - # expected_num_docs, # todo placeholder... but we expect 1 document per run. - # self.DEFAULT_ID_FIELD_NAME, - # ) - # vectors_consumed += expected_num_vectors - - # # Assert last call creates stop iteration - # with self.assertRaises(StopIteration): - # bulk_param_source_partition.params() + self._test_params_default( + bulk_size, data_set_path, parent_data_set_path, num_vectors + ) def test_params_custom(self): num_vectors = 49 bulk_size = 15 - num_vectors_per_doc = 10 # 10 nested vectors per document data_set_path = create_data_set( num_vectors, self.DEFAULT_DIMENSION, self.DEFAULT_TYPE, Context.INDEX, - self.data_set_dir + self.data_set_dir, ) parent_data_set_path = create_parent_data_set( @@ -3258,7 +3224,7 @@ def test_params_custom(self): self.DEFAULT_DIMENSION, self.DEFAULT_TYPE, Context.PARENTS, - self.data_set_dir + self.data_set_dir, ) test_param_source_params = { @@ -3271,9 +3237,10 @@ def test_params_custom(self): "id-field-name": "id", } - # todo is it weird with the parent data set path? + # todo is it weird with the parent data set path? bulk_param_source = BulkVectorsFromDataSetParamSource( - workload.Workload(name="unit-test"), test_param_source_params) + workload.Workload(name="unit-test"), test_param_source_params + ) bulk_param_source.parent_data_set_path = parent_data_set_path bulk_param_source_partition = bulk_param_source.partition(0, 1) # Check each payload returned @@ -3290,25 +3257,108 @@ def test_params_custom(self): self.DEFAULT_VECTOR_FIELD_NAME, self.DEFAULT_DIMENSION, expected_num_vectors, - expected_num_docs, # todo placeholder... but we expect 1 document per run. + expected_num_docs, "id", ) vectors_consumed += expected_num_vectors # Assert last call creates stop iteration - # TODO: It's actually failing here. Figure out why the StopIteration isn't being called in params. with self.assertRaises(StopIteration): bulk_param_source_partition.params() - def _check_params_nested( + def test_build_vector_search_query_body(self): + k = 12 + data_set_path = create_data_set( + self.DEFAULT_NUM_VECTORS, + self.DEFAULT_DIMENSION, + self.DEFAULT_TYPE, + Context.QUERY, + self.data_set_dir + ) + create_data_set( + self.DEFAULT_NUM_VECTORS, + self.DEFAULT_DIMENSION, + self.DEFAULT_TYPE, + Context.NEIGHBORS, + self.data_set_dir, + data_set_path + ) + + # Create a QueryVectorsFromDataSetParamSource with relevant params + test_param_source_params = { + "field": self.DEFAULT_VECTOR_FIELD_NAME, + "data_set_format": self.DEFAULT_TYPE, + "data_set_path": data_set_path, + "k": k + } + query_param_source = VectorSearchPartitionParamSource( + workload.Workload(name="unit-test"), + test_param_source_params, { + "index": self.DEFAULT_INDEX_NAME, + "request-params": {}, + } + ) + query_param_source_partition = query_param_source.partition(0, 1) + + # Check each + for _ in range(DEFAULT_NUM_VECTORS): + self._check_query_params( + query_param_source_partition.params(), + self.DEFAULT_VECTOR_FIELD_NAME, + self.DEFAULT_DIMENSION, + k, + ) + + # Assert last call creates stop iteration + with self.assertRaises(StopIteration): + query_param_source_partition.params() + + def _check_query_params( self, actual_params: dict, - expected_index: str, - expected_vector_field: str, + expected_field: str, expected_dimension: int, - expected_num_vectors_in_payload: int, - expected_num_docs_in_payload: int, - expected_id_field: str, + expected_k: int, + expected_size=None, + expected_filter=None, + ): + body = actual_params.get("body") + self.assertIsInstance(body, dict) + query = body.get("query") + self.assertIsInstance(query, dict) + nested = query.get("nested") + self.assertIsInstance(nested, dict) + + outer, _inner = expected_field.split(".") + + path = nested.get("path") + self.assertEqual(path, outer) + + query_knn = nested.get("query").get("knn") + + field = query_knn.get(expected_field) + self.assertIsInstance(field, dict) + vector = field.get("vector") + self.assertIsInstance(vector, np.ndarray) + self.assertEqual(len(list(vector)), expected_dimension) + k = field.get("k") + self.assertEqual(k, expected_k) + neighbor = actual_params.get("neighbors") + self.assertIsInstance(neighbor, list) + self.assertEqual(len(neighbor), expected_dimension) + size = body.get("size") + self.assertEqual(size, expected_size if expected_size else expected_k) + self.assertEqual(field.get("filter"), expected_filter) + + def _check_params_nested( + self, + actual_params: dict, + expected_index: str, + expected_vector_field: str, + expected_dimension: int, + _expected_num_vectors_in_payload: int, + expected_num_docs_in_payload: int, + expected_id_field: str, ): size = actual_params.get("size") self.assertEqual(size, expected_num_docs_in_payload) @@ -3325,16 +3375,16 @@ def _check_params_nested( index_name = index.get("_index") self.assertEqual(index_name, expected_index) - # here, need to iterate over all of the nested fields. + # here, need to iterate over all of the nested fields. outer, inner = expected_vector_field.split(".") vector_list = req_body.get(outer) self.assertIsInstance(vector_list, list) for vec in vector_list: actual_vec = vec.get(inner) self.assertIsInstance(actual_vec, list) - + self.assertEqual(len(actual_vec), expected_dimension) - + if expected_id_field in index: self.assertEqual(self.DEFAULT_ID_FIELD_NAME, expected_id_field) self.assertFalse(expected_id_field in req_body) @@ -3342,5 +3392,5 @@ def _check_params_nested( self.assertTrue(expected_id_field in req_body) def test_nested_vector_query_body(self): - # assert that _build_vector_search_query_body returns the correct thing. - pass \ No newline at end of file + # assert that _build_vector_search_query_body returns the correct thing. + pass From ebe022e7b29eda3f4533f1fff4f888b1e1125339 Mon Sep 17 00:00:00 2001 From: Finn Roblin Date: Tue, 16 Jul 2024 11:53:30 -0700 Subject: [PATCH 3/6] Fixed apache license formatting Signed-off-by: Finn Roblin --- tests/workload/params_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/workload/params_test.py b/tests/workload/params_test.py index ad42802d1..267f47439 100644 --- a/tests/workload/params_test.py +++ b/tests/workload/params_test.py @@ -13,7 +13,7 @@ # not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an From 9c2348c4a12ffe7502667e472dd068a39584abbe Mon Sep 17 00:00:00 2001 From: Finn Roblin Date: Mon, 22 Jul 2024 10:37:12 -0700 Subject: [PATCH 4/6] addressed Vijay feedback Signed-off-by: Finn Roblin --- osbenchmark/workload/params.py | 82 +++++++++++++++++++++------------- tests/workload/params_test.py | 27 ++++++++--- 2 files changed, 71 insertions(+), 38 deletions(-) diff --git a/osbenchmark/workload/params.py b/osbenchmark/workload/params.py index e60733abe..e91e0e96d 100644 --- a/osbenchmark/workload/params.py +++ b/osbenchmark/workload/params.py @@ -33,7 +33,7 @@ import time from abc import ABC, abstractmethod from enum import Enum -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Tuple import numpy as np @@ -884,7 +884,8 @@ class VectorDataSetPartitionParamSource(ParamSource): def __init__(self, workload, params, context: Context, **kwargs): super().__init__(workload, params, **kwargs) self.field_name: str = parse_string_parameter("field", params) - self.is_nested = "." in self.field_name # in base class because used for both bulk ingest and queries. + self.NESTED_FIELD_SEPARATOR = "." + self.is_nested = self.NESTED_FIELD_SEPARATOR in self.field_name # in base class because used for both bulk ingest and queries. self.context = context self.data_set_format = parse_string_parameter("data_set_format", params) self.data_set_path = parse_string_parameter("data_set_path", params, "") @@ -980,6 +981,15 @@ def partition(self, partition_index, total_partitions): partition_x.current = partition_x.offset return partition_x + def get_split_fields(self) -> Tuple[str, str]: + fields_as_array = self.field_name.split(self.NESTED_FIELD_SEPARATOR) + if len(fields_as_array) != 2: + raise ValueError( + f"Field name {self.field_name} is not a nested field name" + ) + return fields_as_array[0], fields_as_array[1] + + @abstractmethod def params(self): """ @@ -1157,20 +1167,22 @@ def _build_vector_search_query_body(self, vector, efficient_filter=None) -> dict "filter": efficient_filter, }) + knn_search_query = { + "knn": { + self.field_name: query, + }, + } + if self.is_nested: - outer_field_name, _inner_field_name = self.field_name.split(".") + outer_field_name, _inner_field_name = self.get_split_fields() return { "nested": { "path": outer_field_name, - "query": {"knn": {self.field_name: query}}, + "query": knn_search_query } } - return { - "knn": { - self.field_name: query, - }, - } + return knn_search_query class BulkVectorsFromDataSetParamSource(VectorDataSetPartitionParamSource): @@ -1226,6 +1238,33 @@ def partition(self, partition_index, total_partitions): return partition + def bulk_transform_non_nested(self, partition: np.ndarray, action) -> List[Dict[str, Any]]: + """ + Create bulk ingest actions for data with a non-nested field. + """ + actions = [] + + _ = [ + actions.extend([action(self.id_field_name, i + self.current), None]) + for i in range(len(partition)) + ] + bulk_contents = [] + + add_id_field_to_body = self.id_field_name != self.DEFAULT_ID_FIELD_NAME + for vec, identifier in zip( + partition.tolist(), range(self.current, self.current + len(partition)) + ): + row = {self.field_name: vec} + if add_id_field_to_body: + row.update({self.id_field_name: identifier}) + bulk_contents.append(row) + + actions[1::2] = bulk_contents + + self.logger.info("Actions: %s", actions) + return actions + + def bulk_transform( self, partition: np.ndarray, action, parents_ids: Optional[np.ndarray] ) -> List[Dict[str, Any]]: @@ -1238,32 +1277,13 @@ def bulk_transform( Returns: An array of transformed vectors in bulk format. """ - actions = [] if not self.is_nested: - _ = [ - actions.extend([action(self.id_field_name, i + self.current), None]) - for i in range(len(partition)) - ] - bulk_contents = [] + return self.bulk_transform_non_nested(partition, action) - add_id_field_to_body = self.id_field_name != self.DEFAULT_ID_FIELD_NAME - for vec, identifier in zip( - partition.tolist(), range(self.current, self.current + len(partition)) - ): - row = {self.field_name: vec} - if add_id_field_to_body: - row.update({self.id_field_name: identifier}) - bulk_contents.append(row) - - actions[1::2] = bulk_contents - - self.logger.info("Actions: %s", actions) - return actions - - self.logger.debug("Bulk transform called with a nested field.") + actions = [] - outer_field_name, inner_field_name = self.field_name.split(".") + outer_field_name, inner_field_name = self.get_split_fields() add_id_field_to_body = self.id_field_name != self.DEFAULT_ID_FIELD_NAME diff --git a/tests/workload/params_test.py b/tests/workload/params_test.py index 267f47439..9e9ea2a8b 100644 --- a/tests/workload/params_test.py +++ b/tests/workload/params_test.py @@ -3122,11 +3122,6 @@ def _check_params( class VectorsNestedCase(TestCase): - - # TODO: figure out how to unit test the nested cases. - # basically create a nested field list with different vectors and partitions - # - DEFAULT_INDEX_NAME = "test-partition-index" DEFAULT_VECTOR_FIELD_NAME = "nested.test-vector-field" DEFAULT_CONTEXT = Context.INDEX @@ -3135,7 +3130,6 @@ class VectorsNestedCase(TestCase): DEFAULT_DIMENSION = 10 DEFAULT_RANDOM_STRING_LENGTH = 8 DEFAULT_ID_FIELD_NAME = "_id" - # NESTED_FIELD_NAME = "nested.vector_field" def setUp(self) -> None: self.data_set_dir = tempfile.mkdtemp() @@ -3143,6 +3137,25 @@ def setUp(self) -> None: def tearDown(self): shutil.rmtree(self.data_set_dir) + def test_invalid_nesting_scheme(self): + # Test with 0 "." in the vector field, with 2 "." in the vector field, and with a different separator. + invalid_nesting_schemes = ["a", "a.b.c", "a.b.c.d"] + for nesting_scheme in invalid_nesting_schemes: + with self.subTest(nesting_scheme=nesting_scheme): + bulk_param_source = BulkVectorsFromDataSetParamSource( + workload.Workload(name="unit-test"), + { + "index": self.DEFAULT_INDEX_NAME, + "field": nesting_scheme, + "data_set_format": self.DEFAULT_TYPE, + "data_set_path": "path", + "bulk_size": 10, + "id-field-name": self.DEFAULT_ID_FIELD_NAME, + }, + ) + with self.assertRaises(ValueError): + bulk_param_source.get_split_fields() + def _test_params_default( self, bulk_size, data_set_path, parent_data_set_path, num_vectors ): @@ -3284,7 +3297,7 @@ def test_build_vector_search_query_body(self): data_set_path ) - # Create a QueryVectorsFromDataSetParamSource with relevant params + # Create a QueryVectorsFromDataSetParamSource with relevant params test_param_source_params = { "field": self.DEFAULT_VECTOR_FIELD_NAME, "data_set_format": self.DEFAULT_TYPE, From 26b0d8dc2074ee55e1b7fb879f98403ae0696935 Mon Sep 17 00:00:00 2001 From: Finn Roblin Date: Mon, 22 Jul 2024 11:08:55 -0700 Subject: [PATCH 5/6] Update PARENTS enum to not conflict with Junqui's PR #546 Signed-off-by: Finn Roblin --- osbenchmark/utils/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osbenchmark/utils/dataset.py b/osbenchmark/utils/dataset.py index 210b00a2a..5c15f6217 100644 --- a/osbenchmark/utils/dataset.py +++ b/osbenchmark/utils/dataset.py @@ -24,7 +24,7 @@ class Context(Enum): INDEX = 1 QUERY = 2 NEIGHBORS = 3 - PARENTS = 4 + PARENTS = 6 class DataSet(ABC): From acb9e94d91f412a0ffc7c6aac4ceb58dff8efc6b Mon Sep 17 00:00:00 2001 From: Finn Roblin Date: Mon, 22 Jul 2024 11:11:54 -0700 Subject: [PATCH 6/6] Add TODO to support more levels of nesting in the future Signed-off-by: Finn Roblin --- osbenchmark/workload/params.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/osbenchmark/workload/params.py b/osbenchmark/workload/params.py index e91e0e96d..24c68466b 100644 --- a/osbenchmark/workload/params.py +++ b/osbenchmark/workload/params.py @@ -983,9 +983,12 @@ def partition(self, partition_index, total_partitions): def get_split_fields(self) -> Tuple[str, str]: fields_as_array = self.field_name.split(self.NESTED_FIELD_SEPARATOR) + + # TODO: Add support to multiple levels of nesting if a future benchmark requires it. + if len(fields_as_array) != 2: raise ValueError( - f"Field name {self.field_name} is not a nested field name" + f"Field name {self.field_name} is not a nested field name. Currently we support only 1 level of nesting." ) return fields_as_array[0], fields_as_array[1]