From 5537644106c92282c8f15ec22f68c05b30bbf901 Mon Sep 17 00:00:00 2001 From: nitya Date: Tue, 17 Dec 2024 21:06:56 +0000 Subject: [PATCH] Deployed 06fa7fb with MkDocs version: 1.6.1 --- 0-Workshop/4-Evaluate/03/index.html | 126 +++++++++++++++++++++ 0-Workshop/img/evaluation-model-update.png | Bin 0 -> 94518 bytes search/search_index.json | 2 +- 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100755 0-Workshop/img/evaluation-model-update.png diff --git a/0-Workshop/4-Evaluate/03/index.html b/0-Workshop/4-Evaluate/03/index.html index a93631a..e93427e 100755 --- a/0-Workshop/4-Evaluate/03/index.html +++ b/0-Workshop/4-Evaluate/03/index.html @@ -1095,6 +1095,17 @@ + + @@ -1105,6 +1116,43 @@ + + + + @@ -1348,6 +1396,32 @@ + + + @@ -1365,6 +1439,58 @@

4.3 Configure Evaluation Model

+

Recall that in the last section, the evaluation script identified an evaluator_model that will serve as the judge AI for this assessment.

+
+

1. Specifying Evaluator Model

+

In this workshop, we are reusing the same model for both chat_completion and evaluation roles, but you can choose to separate the two by:

+ +
+

HOMEWORK: Try deploying a gpt-4 model for evaluations. How do results differ?

+
+
src/api/evaluation.py
 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
    # ----------------------------------------------
+    # 2. Define Evaluator Model to use
+    # ----------------------------------------------
+    evaluator_model = {
+        "azure_endpoint": connection.endpoint_url,
+        "azure_deployment": os.environ["EVALUATION_MODEL"],
+        "api_version": "2024-06-01",
+        "api_key": connection.key,
+    }
+
+    groundedness = GroundednessEvaluator(evaluator_model)
+
+
+

2. Configuring Evaluator Model

+

Let's take a look at the core evaluate function that executes the workflow. This function will run an assessment once for each record (in data), for each evaluator (in evaluators). This requires a lot of calls to the identified evaluation model - which will require a higher token capacity for efficient completion.

+

Note: The current script uses a single evaluator (for Groundedness). Adding additional evaluators will increase the number of calls made to the default model, so make sure you configure quota to adjust for that accordingly.=

+
+

Update the model quota in Azure AI Foundry if execution has rate limit issues

+

Take these steps to view and update your model quota.

+ +
+
+Click to expand and see a screenshot of the update dialog +

Evaluation

+
diff --git a/0-Workshop/img/evaluation-model-update.png b/0-Workshop/img/evaluation-model-update.png new file mode 100755 index 0000000000000000000000000000000000000000..b2064528f59138fb5b30c8559763a74def2ff6ae GIT binary patch literal 94518 zcmb5VWl&sE*ENU+hXhT6y9EgDZb2I-0fM_bG&B~1ySoM_1PQLe-66Ppehv-zJ1PFYwxx8x)Exs@;I39F%b|Da1=htXdoaUg&-gxGoinN-)ULR zR6{_({AD98t)?I?O{M1QWNBk>fq?KSA|(w?Eoq5(@Elqk_8kGSh`@)Sh(JQ|us{h- z9YY~dO$r<3jc{Ev5^H2QkAr@b56&bKvsY}hHdO&HFI|}*!m_$wAz;c01c+4cX#Dkvh2Kn5!>H^h4leG{ zt!PiOQ&T4-o5@Y2>Er@0>cvE#g>(A4YzCBgzWkKIn9Q{pxS(jqmSnNJdHE9i;$Tx= z*dmG@FR6%s|DEbLkH&9tjhQF9u{b>JOfw2A2LI3`8L=(5u?gq6jGgE?wS6+T&lLXn zIdc*(f~uQ91yk>gu^PdBOYhK$+K-|#bxz-=aySP+Y=LkKlbs{PBSw=C59mTFv0adO&m@#^VQ*d))iH+=X;4 zF3l9VSDagk-jXF*<{Xl?HFUXIV75yu^AN?qmNaQ@{th)op~D|GHU(d${=xM|dX6-A*G#}bWr1xRAzBYF)zaGx#z83J5~lTE`B=$XmpRb zD-3)1gOasDX)S5_XLGbYqz}1*HKW1JRizD439O@{zHFm=k}n zIivjgl-o&#{yoULhO9;s{}ZxQ4dzeOED13tl$CCFH4NK7{%UBkfoAzQ*+>W77S80? z--I`*Uk969;714H8RLJWB9jR%i{~oBIFe+DrX(B z!lS^kps!4~3}F2sGqR*WWjfZH(p?}k#xTM#j$KN(@OY}~FzQhMQ1bfiJ!xpPdT+Wh z*+*q%LWVcLF_Yq$d&&ChjoD4kj6*poYd-&^`HNr|cGE3qmQ-t3BfBWNXn7)NPw9y+ z9NW87Z96A zH}v`%9rrsGQdmu=l5AX0{)f_CwGpO4W`5>sb9AnIt~0I=88=+oT#Od4 z%m*yW>xkul;|>u~CHE#!BdUGH6UUCCNRTAo}MoJt@DDn*paYxPe zaCS2w!;lb&{I2irAmJ|IPa+z^I-&|fRn~AK^~n23<$kSx(|)r^FCuXcHKRYD|9q|S zkevPYj7p$bq4-PDpctvxvRGI#FU4jgcZ7LFIC&+RvNB&gT06Dur^ZsHtHNGNMSH~$ z7X1c^n{+TT@8|K)Y%&fp%l-|H4K5xy9!?%XJ|Xw0H@fd5-~W0){J!|T6^|TmdFDi> z$hgM%?@SJBOrBS4DLSfcpGaplW+Bt~Hm9@T>C8F4DcpsLdB?y0Kc(jij_D4Q!S3Mb zBU|vR!(a0bbAI{qQ$LRBk52y{&ty&UPsPn`6mFD=o93JU;Thtz=J7WF#OKawVDs6| zdj;K2hF_6$*=)`Zn{?$Z<2&7V8$3ZO_A1^CzRLH+ro_t}yBt5i3K%PLS)^NL@MYp= zGH1fF_A(nXIt+{2m@KDbWxqysW~`)_W>S6(u%@=!vP-a=Ho5#evzX}c#R9kzX}@RD z&|R?xj%kSCUGS;C5hblq2^=TRQq8Jv^lX$`VOUWY{@NzcX4{6?Dk7S0DAQ#cSL^kA z!#H5%IVreGx}=vgtcy&AT$7BHOrDtZ!#=xMjrWkkO2l6Mq{4}^Tddpj(wCFMy&l(c z4>ndk}OAT7Vf|)51dTweLl5_-{$>y-@xjb|MNR zQ=o!TA;>G=Bwlk{TTSg42XCfycd>P|V;E{Q?0yNC9a@h6rDl5=ITbpEXX0qmZ-Up0 z(<{4`^CmfbG5kI};#1sE;wP86!ZHu_g&*35!0A)Y=JfFNV9p&S(X=x?Au>CHEJ|Om zjPAM<3*EHJ4}h=!KByL`cKw$E$_C~J(yJsouPak|DJG+)&%&9Q6v>!>H(99!$S;bZOhct3$7L<-@SaJ8Da3eBSB_u4H#a2O-K?|ujy03Cr$p8pUJvhj)+2JAj9{+_qB>;LP7 zn*Gf#WIx0r#3*u6cEP{yuKcmI+yP_7C(Nx0l;MCRDYa#zN!d^jjcxoarr>eMq z=GzRkP3NRl&+@Cz@2qD1dVPa4qjkdq{o0OgB@2zc9G0evecJ|b)jWZ(fbTuD^~|9{ zy_)$a?2-HY!1tjY+VB^=ka{|JHo5$ABGDK0%jmFcj#5kTv-ia@^)t20*H3HxK5h5+ zuNp$P<(k#!1bhT;ozLII(VnW5s61p6WnsmA`%(CV!$0QX=8Ut;ENAd&FiZ6?r_8aS zk+5E>s#1S&IdQq~bo$hf(GTG1Dd1WEXnDI*ps80{p~{-gE+#FK<;kfa%$wCs8vjaXEVlC-}4YE>=pRnTL83p{LhZRfDyP~GIY;@-4c z(=t6{wleE8duHid?dK%FvE9l2#Qt>P1YB(eWdR@VbN&eXdG(&JlZKPO-q=|sZOQX_ z84v!6mW0PI)+55={o$(lBw{&fc_YT(iFhRvfainlDURcJ?;I|A4T*~u@5DY3r4?uO z{RM0H?{nRqD_gKWk1FN}HNrGh?Hqa0yqyFugjNF{_wIF}pXU}YNVl@en`FhY2YP$^Ty|N0dMfbYp*FxYMdM>EHsQB0>N=<4Ctn#I9Z@eh8Tr5++7 z*XDN=gmns}Kr$SJ1`>qrnWavjf&3+>pm#_w5mvtE?|!2izaSu(*dbhWAxg;7Bix6J z(Gv$6V{Ut))TE@CkKO#?e!8!>q1wQj=2r29?pj>{vQGE9Mj!IH({&nNy=1?j`fg)v zfzwpU;SH>S#b*Ue6%_;)_%%8LVwepA3j7KY{&^4oz{NNtS%7D*EUsW%MI>%9F+^=k%?sirQ1CLfAkpoG4@ z{%nV(pCwGdcy)o#d?>!h_9OdT^V;9P<2Q$EgC56AqaNo=D}q}qtsY4gT(}8MIH=g| z6{fkI6*`>oU{}%u^)Qo+FREg&j0L6`_aT{mU4 zIob`yk?bG|Te}F{?*poXtaRC`|G`Wz_vcA7H*MLriWa-f})i1d|Igyt8Nmp+N zhMsjaUX3Er7>^)eY*rv(XvWwHz>e}OFRYeA#Ozotj7O!~LfNR2 z2Q2&s7%iA(9x>hWh+&wwn)2!FaAp?*F>8ZuSF}&AS~V|!KVM$;|Cm~Ru|%;dDZm~+F5*eaZX$mOP7f@|3FlrfNcJ0YU5`vO_~xg+Ln=E;J`sXjb$ zS&==9uVi{J9nBQ(Mzus+Y-LANQszPyOfbkE|jK zv*j7I`!v@vM;7N1>YsQ2d~C-@DS3V}>1n21joXNu-|Fi(ZYQ@C?gWj|yv{Ko&$bMG z{#E4*0PE%h(Oh!CksWs?F52DgKDsF&NUI>U=slgzPl|@mw70Z~wlKzI*VNjD7fa@l z`}8^S!EYcIAA46ZA~O}l95QxI__9`KMFyQg@g<;L@DuThNu2Kc&gl7J5Hz@QfFPJ% zN>nlA$N25S!06)<=Io2L;~;8EEe!r$uGxQqRW*jiJKV_43s=2GyM!uRVLqgML$G zabftdkN%j^UIp`gY8zR@lA-s^YAoyN4;6XR{>ZFJB6-VHWo+2NiW~of+>3jSa9nff zQu8;{Eu$P~RwS$6F6(^>G7HgVa?>|1PZbQ!C2MJP=jV3LIHR*q@Pjm3jIY!?@@O0I z1^`SdZ=OwE>!=gC5h5BN<3DIAPDb#=2o(JMBUGXnJ-?9@`qbpwwdKuhsmj(4O}b~K z_w;D1UZKh{=np6q%d3|qa^$)y-|jebERpV&11|ICL`pxsrp5sK%Xi|AfU_v&BE3f< zWZzK=+5fOXcMdD)g>QzA+Q5E93J@i`y{v} zccrU~oo3ry56C?1B)v+Zu>XLrFq(?w3)q-!7*;d^B2QW879#X`=63?xsG$`>8v8aQ ze_6{p7K7wzRhs97Jv0bAkFfSP%_1FMX71{83hRsUne~UM)n#UBO*kJo>+by zblQmNcPZ2AYd<+IW(q7h`x^twNtr$cE89p(QCLVF!qS$98Z#5`9T&?*q^YC!;BaJS zQC1PT(UaVoeURnq<1&6T9C;6i1CsD1<;f6#LJxoDfC8TztgO-_ODyemAV^S!ZV%E& z=ryK*WmeG_RdSee9d#!&(V`Y*?P$AKo~EWUGZ+@|$k}=DhIXhxj~38TK`0>WZ?RH3 z<--ybZx~ME&B5q2cRmstb;i*s&QmX5?{R$Q10D8H!k&q<%Pt_!pRg5U9?UV2c)8-4h5+|v3g$Rl!!G0q4!2~+Igbc7-s6K+ z6ci2AeG|9gr*B9I77ZyBTnjX*A1%oEEwLu6I$&6tko~bClA&!VV+=GhCM!`-DkPsK z_%m82w{;)}f^IcE@(bd7g(Q;+TqEOWp0B)|tNdh#F(eVA!65NKllxyRD8=pC6^T$@ zXsW&ev}i;WpILNjitLdq3tro{88!IdXB09WBq;n3E%72$i3@(ju0{!(=n#*7x>q7+ z(8cN%4H3OmqJJy=-lhlUg()hFh|(`n@Jg)EP}X8fb6s=(2D2N@+h&9@E987mLH(`- za4!F4s9T;R2JgHRP_CPg$9o|}%L8AQ=|Oiu!sDtH zBu~bgg7zpc1WpxsYct~WrN^7v{8=`Ku-HL?ZNZ@RsKk>ch{v6k88rAzOVMpc1}|U7 zC4Xd0QQH1f6hJ>*o=5MB+UAV@_`0}7OtJ~k8_}|nX$)_dz{Xn=Z`*&K z`VHj~ppu>)9sJDvq02oRR-`k^c(lpqM7vYHo+1^)k(649&*+&&(x^sEYHEdGRFG00 zg&?e+4M4pDe{~5YJ8jN7D)|o;XUf)!b4D&{8643XP4knhmBUXtQF5)FV)}?2sZ`+A zK@2e4S>86e^ecPH=l44Mx)1PQnx)*I(|B^00w=L@Ys*f-Nw#8?8YP6W?)SVXnUR0K zU+!q6#rs|yTem__QZT+F>lSUt39E}uRHZ=PU2PZ)KDD;;!Nb>rL2jQ?%C*m;=W9mo3aY{&3sw9@gE|mY_or6 zu9nEa)dp|g)NEbKwEbaZ{eH@})AFE1oQV#6_bC->*83~oOK5khYUAQa@j)I+A?m&$ z&sK()iT$1jJ*+mX@kI@GljzKwF(@z7JI&5Tp zH1OD&UUfZKy1UvYyl2n~(tBN4CV53?a;Hk|A+cA?mp?VCHq(%6t^^2xrg#!UK1aM9gK`ZFj_s?hB})-H^fDuzt_DG`9ES*I$Qw11SW^ z9#2B@D1l~&G{<{QdPC6A;=JZ}55c@c7xp=tVk3R=Q~XGK0%6FKf5!m^OGTJgO8JCl zHH!u2diCo(jbmoHH-lL1X-WKGJV383lw^gH!8hPA^$WIruOc-|{}I}F>~D=ZuDmIY zylh#v*(#9v*UopO$xxjXYmlH_QQ7DDQ$D^wU(i7~Rb$F1i>x~!TW8Im_-I4Zk!=~C zk^9WMcjYWB7fFn&{nh*FxMlL4Dap0zOu!%G@_|>MCg0@_V&E;pxA#A4gKNX~|4VOt zM6s3JT&eDicYQ_4vuxnXDY$@%_CB*z;S0s*$XRGAJ#owVv?#r;zULVeLDAaxJ6V`k zE#8_b9tjXuMDq$cq30E-E@R4`B6mTx-QuM3IjK<8#yQ!(=!~8)rx2t_kmMQv0N_h1 z(fjBhYPopbE)OgEAxB28CetZ50^7c`-S-m#Tb0lYtHfMwzkRU2FzVCHl};j`eZYtv z0fJMe-|8>^M%+m+ZO=%)`jiL%eH;hXaGo&%@1XfYR0>VJvnlWCHx%E08%_7~XS?|41L15sgU9zbL-q$HPNUN$;TrbJ6`Q!(;!lS)LfK^_jT{ zlX((9SBK5jT0XNDkTX}NiB{Dz#0(M4fF)-{1n8lgj}LlZtQ>d?|7y9cFoc6Nis6M%Krv%&!#>SlTBhZ` z8RvumAets_$MTtORi1wd(+JR91ODGt`O#l-B#r^N*uAWh`B&$78~!)I&&=3RLhvT1 zC(q3Ht8+r1ioIfcG%!vBsp~nT+K5PqRLzazNG~oZMJ|Uh#xGLzevX&f{JHq@O)r%B zCI`&TG=nfw9O_sv@$|}pw{Zw5gxwpU*_f+NV#uiey)M84TZ_MX)T^&n3ismF2+Te+ z-VE5b44Hy*@TN2o-mLLlRT$f(9$I;WwTouTpQ?&3$RFaoQ5Ze*;ScPg*&O|dnKh}o zjv%XQ{a&6wne7roKLu(#PA}H9nGA}lcUI2egln|FrUW;mvMGcd%8`?BVPlpbktaj> z|D#|*o&(LeY>0F)x%1|1VRS0hEj<6~RL>kED5Jd{Y6sQb=|H9=!6yBS~t}p@G3|MM;KE#c} zal@eLQp(omYXJm%uw-U9O}HsBBc9+RxYfwv1eMTE?fPQ+`L`da0FVR)!+cwJ5&oK` zYq_T!DA1QA3dXPoQdFi5et>|@R?QM@A5#Ju4I{@w>5iM-ntJmDVa`6WL| z;kYUQBR;1bj8M$h&6N%Mz&G1WGnA`>seILMfvaij_n=z+2;F$uWKd3zR}~g^+$RXk zmm0BdrxCEYfKd-TqadQYK`L&;QlyONeTY^+34JJWK!u zCm&!@oVT?1X+XKUmf@7K-mTSRGkWGf@4c)toqRPyH%y8%@2~{RCUB|`4G6C@d#5TZ zaG5Jb=9l4%iI~X)zz;nq)IifRyvHCT^m&K*#k-*;ghcO;L*r$r;||Wk9?t1SbIfSg;El%7!&G&ZtVU3*snK7Z*^;Yq^Kn@*O@V!mF zf;tP{wURtoHltMPoA3?dKe;5CMatux?|WMtcI`3tCX69;PCfGm)lg_EtuP*z=6>WE z`OU>~n4Z{eQtUarox=0sv_qHXy2ZP6{qLYe-lI*W%rpt~-+GjN*-vs$TQ<#y&n6Gj zNw;W%yC9_idK@wGAI!rbDJ4JGTOomPvxdHS-0uTbYvhnLzf}zl=lrLGP*OiK6VT@= zO@M&htEGKI;EhMqI)i@Mug=RPzepebef61J{e+!5F{h~fFT9ZMf#b~?wtKpD1Gi{r zr}mr13z|Rc7|CYM;|%~-8-f{y3Qu~ZzgK%ALBkANECd} zYmE?RrFy+XpVcke`w=glgil2P46g|z&dSLkvwA?D&1y9F#G%;pT~Z&SMx6eC7Hyqs zF#)c`#Co5C%tj&bgXW4bVgHUotodB(Uwy${p4f?-&E5ijcIk3Rhe~6?NuAm_0P)-^ zrYuT_hI#m3E31+uvoZ<^B%3k!S5m+72J=dY)+mtGa_so|Co@JQUn9XyGYN#SMD^Z+ zW&ijSdUjZDkIinPEl>&f&be@bpw-%<@5py_qyr^_Vkk(7teeF1bMb1pN8#z~4aV2r z)xEx_inz39OJi=;E_*6AdL8DQIP+a@zK{O8O~)| z$dEfDDs^;6*Cr$FtRwIWhVmbR5&0g_Qy$P*h<{K(ymYtuX0i=yjuhW(e`kIK17}IP zQ!esJUc1A8ygg3-$-CWI8#!(T=dA6DkKg=)%Io;lt99+UMm{!2poa`408amcT4-tbk!Y33^L9)V*5KFMlLtV^W_7)*-F1h`6dcuP>+p!@Oy{18sYWZ2AzRaCHKG8 zIhvj~8&0+R>dF>JxiURUFV8hm&s}u)Watkuj7Xt)w~U1*4`_fC8JV>2I{FU05F#e^ zYc2H@2MdM%RWH_lKBIzQYU3T@UhJUi+Gz&0#cnH^j)&Jm$6|lAi)n%rx9XHQ%27>* z62DU8ol8fwC*0T1;d0dzs&IB{#?Z>XEr<{Sd~E}vLApz3KBdke(vEvED&OYhfX7%z z8zX=7K!_@PE$GG0SeHYQPp|6Xg1Kc54Q=}Q`#H1t+t_a(ZNdQ^={-fnxjuq~oYQl0 zj!gP+J2IL9oGHwo;?u>vlssVWMrTA@75eT$QwwweN!Bg=!*Rwa{*JI^$Y`5{Zj|wB z@iP*I7}zgDVWb3p+8jq}?*t;rvp-4y3Mw^>V9?as{(WCS=~(nTT|U2g!@IZlHf+~> zI#_<=8gjYjwQ=e*As}$2P*ap_Kg`JOkXzS|ZAxZ*cTKG6gujCN#y#W`92ELIGoYdL z;mk76q3q#yrS|N$?aZY&bOOxx1bgLp`15yj7^Uy(n)42u<93lkgk~ZdQ z5wz>qUPa|GL*26+xz1qP`d{WO0ann z-0?TAHJWE@9ioBMOvG_Lm#&d}tsiy1Y@unBPIYzZTkH{O!}eY}wx6LRm9;JU_S3Sf z*>2JCXqF5kt82Tddaf858z#e#>%cU9*USFZN&d%-2gvC3S)mfU+mEh$MNmV_7pshZ zht4Gi#q)p*N$5tFqk79LOjtt;QL52Im6$6L{|+{H>tLSu%$oNVn}TD-VoB|5g3}Ux zM>oIHymg1x%gHs+)x>dbd`tUdMdcb@mpEI~*i7OvSw&RSQYemFR`^+ySq+EWgQlnW zudKmms<&wWu?xyf371kX{nInU4r?udBUy?{54zi_pcq0y+pBuGr>K@W9Se#mWc(mw z(Sy)qAK)Nzp%P8tH#v$M=!>xeq;P;Y3^GsQ_%AF6XAGAbB6})DNK{mKKmNh^-A&k+ z3#aypsI7#Pyt-gdSrKnXgwv^W=61;TRgY+h8+*NZw)~?1P6(rg#tf8j9k|or82hGo z?e21C4Mt+L@44)FoF5l6DfZIiP!FpvzPRTZiWA*_-PCVMibg@2=YQLhWi#;3pxUrw z6^+t=SLX`>UVDAp*6+zP$M*e~)%sc0tQj=7gEeStj(<~_zHiL@T1IJ(!C$=YmV>W{ zv0~3M3mv}vP$)s1*Kt)6p|O@+kjtD)fZM@WZwy(%`mfu7gI}%JNc=CLtCIV?_(Hp* z;zXMLjhO6QWOd(i%16o1G8OzD*a~qs_ujPt{}R=s6o< zlzwAXtq{dpyQWjzw&r5<$ACq@h2+Qb(mao@MaTk8(hJ{qz@I?Gvln(TaqYhD&9?JS zH+`qBBwz56)t^>c+^5PFux{7fDffiX;A!hsZ22YVdV0`tUfph3b0KL}8oHKpPn9nR z3H5X74B5Lyv_nNt-q$C(mR92e2pRp?x>x8`Bs=AR?>-li*RsGEoTyc0xv$7$z2IH} zhB2}8NX&brPzg5l3t!BF%CxwJ&t@@X!erKPV;MByNiEBN-=UB?6X(vzc8sO*mZ&LL z)Yn!roT$8M1E~9K1JK$FLcVYN4G^a9+we;xmzE4aUxTSV5AWC+yuu;NM>Pr0->O04 zmBEq%sf;Vt)!mF#Ru8`rX7^>2Hr#*W*N9B!uxJ2hahmmD;@|(-L4A4qP_8Z+UWmwQ zei8ibH2(?e@_SMY+p^fiwOK3<>&3&837wOV@K=qMD*Mz)>L3xYA&W@O)5Yv0I> zd*1O9L@Mf2Z@Wb?8jua)=Lg-Lob{&}r6N6Yr{IwaX;Dn1cOPu6e#4rmMx_>VN+fFc z9^vd_LN^&r5jxEA{d=VM+SGp_mK-aAPlR67bzZwYjt|}$8xUbi{bZ*R=OKJJ$n#B| zEclLk@3~^Z82=I1ev;;)m!j*X1Kf3UD_owQp61l~E(wIt{d`tiu3GT^7_JELKits5 z5xS=~ol3!7TveQrKJWUgA(Pl?q*k&33rHsm0r z7E12ZP3)E-{noCGc3eIPK&&7e>sbR8?UR^Nm3U0vR?@%^w{uDXoum?3>WmQ{=ETIf z?ZR@|B)F*L^*%@C@eje>sBSfs^pj9E11D>WT(%u@v9|I4T^XvX`}?ZkW8cCW_M!Oi z*1I)!VU*qt`>rT`;GBkZ%f>eF6z z7}}c09{x1}K=hQ@$dL$oxC1~Dyz5WBov`_pQ^rflu?u&1*iWVi> zq6r##{Gx06n;fs)mg_oZ!kTZ5Jwv=}EIgO45W zs`YJKQdU0=UsAq^X!MazdkLXn(`6MmSJv9=+<_`L0Rwej<~X`BDM zf}u(q{21GImb$9*dL#T8O`Cv-gjp2gp@zyi0|z(Bd{ObB@=>zNZ^0y|_)sA9wx#`A zJy%%wWb%gucPdxaWF=uG(V(yNa0GiLo0}^U?d|{*Otg z5`KA}(r4Pp^SDel$A7DQ&5u*&NO&b?`Qo!5w^@)yaUI8cfS2589^ zg%FQ{&VXpx&>JY(^Ip70K`AD9DKaswG}+IPNj9jtry8OHvHjekQ+`-0E?@0TY} zpqTQd)$ta-&5WAFHa=Tq8R~u>$ItaGu~rn4^RTn;wk|uT=WQ>NU-}<3n)$wT-!JC? zC%XLbB7QoKJSDfU#t?C?hJ~j{-@yz9VnUVbg~X$2xqkL+;aBl>@5vBac8+tJ%G41Z zF=7a}$h)dk3_oSy(O3R>8~sQQZJM#TY1#8vtaqDiqar*Zb!wK}mUDNhrd)6gV7Ac#ts~_I) zN!jg{>!xsp&{38>5b6Oqmjw(rJF?r=yI4utTe}vYr;45s5Aup`HpL|^p2Yih2`HZh zx0!AT{oW|KL^6b~%ZbW4oep!jcn|;ljm<8ao$yHMDcTDCr0QPCP0_Xd=8;~mxhNbB znGiK^(hpB$m-wqUwRXB$-w7@tvQG4R=*PM&{^l$|d_gH_%U_77sm-5ze0M_d>AX&( ziAhCM3(z^$KfV!BsnNAhZ8XLPovwAL@4m;aw$EzWP%a03`S_XE0rtKz#fi z26ef*1D{SNVGF?bBVFZAW9K`q_*o~`OlRH!)D*-DR5o5e_7Djh&(`*OL;8A=sqCAr zK312 zQojo+wP>FVgV4PZBrBiv|0h)A;X?KHNh0>MpnMujnfXh^x=VfmFf(~CsJQvgW~Wc6 zeKhEy(?I9z1eV%R(85(Z#q4Q$&*2($9jjz@>S0WZLctJ=?W1_xo>)uVTK((|#$%C! ztCsWIx+N6~6a_!V9|I0<3kIGEBKBNaWHP5`^9zP)qhol@2Zo^O9DfK#!6zL5!#H)2 z*w&F%!VZNrq2f~L=*Tk9FMC;@30Byb7L9yh;Wy6VALAYmU+k5jf8#QuybR*JZH|Ml`=rqEVdf7DEzRi4#vOA=J% z0SLed*)VH=I9#n%U6-3(Z*~sCA=S7`s}9g1mj1;h_hcOJ9{UVF=JGoY>U>nO<=6}J zmAZPLix0@QhE#C*w{N0T3Iz4U5tbCF=VJejl8ta}zV!5Yr8-=Dp7NM_56{uOXH10C zX_~s_L?3*#l%?>7+wmR6XX}5>dArHT$7tzJg}JsZe?Vj3=wr$_ma9!@_VnR@EQpb7 zYmEUVztfnSah#v)(FQF3@%geAd#qrR>RK6&KQE*+UNQD04qqnqUPuKGRsbhn-u1=ECNb#}nSsO?C zc{*_UeXx&SZo9<;__%t_M|@)}ufG3_0L!j@B-J!U-Ndq8D{Y)8HD}5%Z`rG)!y|`# zhCsM{XxF!@QfFCrjWTLJzKr7W--FY8n851qFg~ApgKb%YjY;vBLW7={Rh;C~ZURJlOKX0U9d&zyMsnKY zUG%qLlwUiyNQS48qG~fC8bQ{8fb+Yi1Xj z;KyEn09=Q;7Hg5T!M%N67S4=D4G50LgjF*QV(AQ0UX3=eT8IB~>)TiOto^H2vWG07 zz4G66ge(2WT8`o2F#mvTlB?k%#is#e#NOb8m|yMUbY@2G)3U$2{IcDPCdLi5giMen zYOWJP##~mga#qzXS4JR!d7Qazfhkkq(dQAw^2v^j9N$V@3`^rsd-|SwIG_ zKVrTmPYV8bXTrn*K6trwq7U}JTco{P_#CzdhP>$O96FEWaJxJ?pRQ+9Fb$3#Ggbl3 z^Qroui~upB_KRv>-7Qu=hJ=LNYUc_}odgLu0w3C6vqv-y)1UU=2xBd!sr_vAFT*Pe z9DIAb_5kpCN7m`Wd{@?s=TiBr&J;iDQicR7f7Uf=Si${!mA~nLBH)w3$b)Pea4aM3 zmTv5Zf0c0Gaj7`Xm5Dupav)%tnei1R|2jB#^F7oR zO}XcHl7G-hbGObbJ^8b^d;VAWtf-INZP$d$lewnj8I$$Hf$9MA$6UO~#fw{v5UIKP z>hX<{_#qN~bCU(7i&K70aVJ*3$n#x|%RGoC9Eu3q2sCY6hWArEapv4wtgBgo%u3ntsh(we=N7#TK8X=Vfmyk_V`uL=j|olatsZrzF1kUm`}O2wwoz z1%48gtdRou3!Y;ZI&eO|rKk(=+uxPX(`Tp}o-me7%j&86{B^fkxphNMICJ*9hJ3qVEu z1?_1cDc~)tYXQYuL1e~%WBQ7@1Cr|@QdA@XndtYp4QFV>ois)TX(xQZa{=dRksEDy zS5uNN@9?yT;Ig|=`WaXF9>wc$dT_KCXJ=DnejxT~;>l(Mc@3j9J2x?@TO4-9-m2w! zyW93cHK?2NXFoK{5$6(5d>_SYzQOx5tqs%98XxbAE%}gYyJ?DDCWTYKKSsS^8h00? zB~Hd{GM$0#hh66g*wTT~6xp2Kz+-QmxUD^C_-0S{E5W_m#{voI&O3`F7lOxh-d~}i zSpBF@S&p!*-34*{a^liEqO?;l1$G;05d|xOZb)QhwaWv!d${SQ$}UB>7#EJieI>mQ zUd{vtvQR&2WyObG<1i!fcMTJhhcw#X8H5S98(Pc&0>k8>#e{of%;X@VM`MuJ@ZNID z!e`cZSUs?X=F6RHSD_pu(2+$ao*02>2UL@93(W<;1w>-%o@d-FJ_oI2x8Hp0zw0o5 z!{^)_5{T1JWtje;@0g@dG)D|5OpCPdCvX)&RC$#>^(nS`a?6X(bZl7c&C+ydJdMmx zKqoFdF97~G?%XL)yY8W~_gIx=h||`wI}Mo4ZgrH_E}H z6m`9+a@hP61bJy#yCD~8ryF1E=`sr3E`LVms3k159svv(61uby8xcT)LGNz3C&lvE z7im`RNL2RqL4sdB^yJv71E}!a+v7Nx1|;*6@XlcRE-;Im;m;sukJjR=qxs6xmxh$a z8QsPr91Sx6_c%t#S96+^&49hm7CNyjYd5Ts>cmY}>x;d?>MocLQ;~+MhB7ndqsWTu z=!d;r->s0j>UzSm0>E_#y~Mh!E7op$dR+<7_BLTsoO{1k z{g~AjX~YN#FHa$;dmSF^54Smz0iEaY{yd2WH{@UbH|Lp-vQza4ZjvQ#3;HmH*t~PV zN9w;?nGylRs57Akh8hbsg=s>0=J1j3NBTN4Ir#Xe#CmxbHX-~*<@OS=8KWbzMLOYH zQQu(R1Amwlgo*|P`}|ZuySfn8ZDjhn$qOFAQY#kr{aNWl zB%UxjlJ!o!JV3}a<48wr!DKwLTQ8vTED+O z&*zSOTWR7Y&2ts{C)ja?+>2Q^^F2KopIdcGso|Oy_H}c7;L^_b!O>lf+_9suf72-d zb#j=4y3o=ybBGlM#vX2WA_>Wt%dEaH|Gb>sP@&DH_}*r&Hts$0nl`s4zDyWDm9RN@ zE1rJLq;W=d)%60>i%@oN zjitXWSeUPeM)^;%q8EsAArb}1d4m2thBIGBg4-vN_dCf0uHX2)tbKZs;q@=JfQmnU zSz}}&B9Bz$zR$oJgm=J%%mY-`g>}*X!K?JKtY&=QcdiqdmXACKl< z48(_QijzMFE7d1*XSAj@87RumD2tI9g&*+O!KOq~!VbaWf=6S`Y?s|G7xBwecZErX zg>S}sLiuU&a;!-^Hm@M+N?=RtP?ZDfoIZI!fu>72QU)UgHW5d_qRGM9E_M33A}kdj zN^F5R<9fzlUzq`QEHeu4I;0K>edt$nDb9nlKiLS1z<)CMb8);w@CuQoAQ*6+vDGVf z_^*M$(WDe#gvUv7dXr$9*+*$0RvUD7((v8^@-z@KJegk~o%R;{$3L9`B-AVq$(x$F zRDjRgdM{7)3jwiD0|bb@Xb%L_HBZ5%ijOsYk5`jM@lH% zHgzfQkuUbjzQHA!Ct=$cFL9{D><*to$~qQdGbuB* zwgaVPUqceac*hs576a9%R0Nd=HQ8MQ$A7j$X1Yb%drIM3tn_?<;Bsy$KttoT*9}3E z(OYJ7VrF?-$HuF^KHu0_11^X{xcv@V0pd?T&Bxl665ltzjtb4?8=3tTeb4jlUt(H` z)7vIDd1veyNwL>?VC1m2E1qFX>3_O(v;OwFgz5w^jA>aMF1=C|KX`C>GUq~UwW2kT z5}e{)4SQi&6UC@vCY2DwD#z_!)PEC|)-tAJ7y9-2U+{I4X~BOR`{E2@xFWxQg8fPd z$mxLjDbU|ih&Z{gJ(=8HY~8bcxWBLlOA?q$XBunm2*bamf3Tm9|vFNFU;$Zh22Bm?;lspQ+?x4_pX4Ri?4=DmtgpfH~vx`S!p=Hhzcd8xliNE~=ptniD zx0~RLkDtQ@-Ini_nQK%0G=PxiJO#Sb0C5;eMkFik$R9Y#J~_8C(||q#APppcVsTcOvx#YmJF}k3koepE{`GYag_JNSh7QJ z<4PcXc`ic>k{MJ7wc(|!|F_&HQ7j&Z)BhuJv+8o#1V1=LbA#+z0VAwxCDJKE-lEFh zBGc>_ck;orvF+uizkx1U5P;#U!b_UGbl&dh!6&4UC1=Hb^RgDSardwYc8Z`E$^qkD-nv?b8gh+?-Pf(q$y}KbUX)jEB%H zh}1X>Ki*a;_hW5?q(HYsOHzjWQ=?RqN271Ll@zoI=jJp|ujHuZ8<>{7`KS$>cGdd8 z9}E5=53Ic@u?LUBe1Lbi0Cf|V%fmT;06f;3rsRi2wG8Ma#!UJg2O@`$yaoPc03!IX zd;ZS-P8iPs1=qi~Y;mKlwN3+G;pXZjE^l5=HV3K9>Oub4_&~7%qks5{tS%%sU?QgC3v%Hr&RUpAY)BGRX|b?JM+L zIMNP_1JZzZE601ZRIYzN6U1IDfS$Ga75+@^| zoSm>pvV@1a`0fk!^`nx5xxR{mpV!~$c79YP{s_O&Qp3pmw~{*l@<^C@>m2xusfbaZ zbmjoSHNv+MFUInn5rm!7JgRjWqw-&?R<2vVBkYda&@iVlmNc8G>b-@>NqJ;?G|M&_5fB5V2 z8Mp`3J{2fDAWw)ZCK|gCxwinya_!#5l@J68 zl~8FEq(P99ZULoJTDrSSR6syVy1To(yFnC$t3bd7ww1CC5 z%=CKkSe7UufmthVlKq_IpDp|yIL4eX(LxTryX+?R^p2W?@0yz0(4C}$YNPRqzF?&^Ou zS?!3_g{!-Nay8}3G8bv@r*>H{%RZC34>Y9Q|Iv^}#GgO(tDgOlruzIy69pPe=}3Kf zUiOFp>{+nFfkp-QFO3RnV4KEswJ!Gi-vkeN=}du`Ocg=<-COcSl2w@ru+Hy@NF9IM z+i`~%^T*--mry0SQG`hLy?y9zu|Xgo+u+ZdSd8(}dw+>YIsQ+HNDDyS+W$;MdbzFn zli5-{VfNt{9>t`7X=+++Hi>6n{K-n2{w>Ltpfmv04h6`y&FGV6m}I3_qKjQN-sfWH z654sNPngpa;WwD|vfc<_ky$l3Miw6ZtlWJEWZYMjn(>wPDzz{DFfG2A-bmR7nzTAU zVW#|E>za{Koszzj8X*raTEx%Q1caa=V%w%9?mJ_y%`aIBU_og@UR0y@lKhwxv<}XK z_cu#rf{~Z_SZ1}nASbAaM$eo8R39gfEASCrJr^4>3 zs9qEMl@{4l<|ZL@K8ah>d7buvtDAWe`6}K%-F2~v{STYcrZq^BjiJMtF}yDvscNa; zs#Zhko*{lpZ*PCL)JpwiBtkr*5;6a2i#?`$rZ7H2`GPtXMF|yFtA`grY?vn*_?Fl4{gf4 zWtfd#4V8i!Be}Pt4ie`dh2lHF9=UAfU}9lidOHIvCnzBAuOBP(`Nf$q=M0Javu3xG zce@GHrT$`A+Mm~dLw*bj#1i1eS=B$FZa#t87dAbFz-|J^7Pfp}EdZe{(70IGNqT6K zm%5pDS9&ilee7kPOxI~geos|e08bapso#s`sXP`Q_i9@ls5?t%8BQ+8NZZPb)(129BSCq zXj=5LaTAwA6g`1sidu!xD(Vq0X%i~W_@h+u&a4OUmAN#fWo;8oo$&k)^IwcDG{)W5 zi-lnX66TA~jCsGpxtT}q?_dU>bE3EQZ+~T;NY*2LaPEY7^ws!n3e-Z!XTwkkVzkzV zWZ3aUi~s1WC01(h!gTnhZ`U4`bomUsZUHa%J7v-H%_Ujvdqb(oT|Zu&s-R)S$kfOZpfdzr38ZCy+pe zW{P1n>}HGR+_(7vRN|#=%Ic*KT?j)2T-@gjrwepym% z1K0MqmQq;4?jn{~OvcJ#HxZ6@6No7h{0I^Q>*|6XQd|L*v^}Ui+t8$G-^=u$f`>vH z`eO&F?v0^_QRO9!v*^Ey^XhBF#Xkv{`z&dr`d~b@|GPqO?|)FCmk-J2PyTZjz!=2> zxjY55cGcE9t@<0er>gNu)`&QlG{!PJeFkVdsJzX65P$Rb4?0d%4J4N@vS z(*31aQY5cOq~_8}kKup|LXLFc!KN3V?}~Z_5U?cc%mTFHu8_0-!jDCc0&U&jdWhP% z{=M`_*OOLcq66BEWHMt>gNl{GkIW^~50ev=gy-3g3HA{27lEc+l^LG&$;&YS_?fxq zaeNQdT3=5CRpyLOl&F(Q1Nqqt{rPZu%~Vx^>=x3E)f(rNij1ka;l9;3{mtToXjtsm zPbTQZxt=4HZM^#{G=Kjl6J6~RfTptVq!0D*H)c=i&^$c1;#BNi>)%8a?=skmb~8uQ zC;fd^!Qa1B?zvq+E~tJMBGBL^)m-j(@#x|F*C$}_4}Ddj1~R3UsH*3>(FKM-Cj|9D z)O7o?03Yr${j`{o9@f?9iWt(jaF6cu)uYq>=hxZ`M5~L_MgQq|o^O?{dd9)G--Di+ zP=RQL!{t)FIYNEaL|PH~-(N09RA3y_PaAolbLWZN+lUI13T$$zcbmbTCJ=87#A)X- zZ}+=MV{8JL=x!SU+Tvyb$9X|@7R!4j61i^@bhrKLh*penTl?nnQRX;#f1Tv;cNWE8(>H-?5WTf^>kWVoMRbMu?i&2PsLeweM_aqSX+9SXPrz^ zLgXyUqq$g7MC*e#H7ypRa&E`xgEOL4y9kk7LvHy3QrT*(qd|f7(Xf0~n3~Htbm zu8==!mZ)K53@jRZe~jz(nR_KruGA=*HNa1|5x8!}Izm{>z!tAM=w<7RaoX>2XDYFZ zwwcg}^z)&`Y<)Ia^kud=nYN9K0@n;58os+b4SH7tM400)7u%~?3hMVy!s;$tRBWfM zgHW!let%pAYQm`fZ=bg(P3uq!BXlB#k^4<+&q&)BP_k2bbPv}C3dX1_*Ah`F-G?WR zM%8vqsGtf5YJbKEtEd|_u@;;+XF?ZbWc&*7~B>w_oAD7)OpYKDi z)Mw06*PH8x+6h4>uG5Q+tUO$oby5?Bbv$~lA1hZ)Xk5)I=gX`47h6Srq=v)HQKCCD zHP(KdXG=kacCZMjLw2~fA`Q=O7kfI6+s3f?G|le^cA3w!T+VqKuhr z$F;*MJn{VVJPnMYaP?z>| zUYZ=M-JHwUe0OQXvnAu$%5R!_`};%Em2WTQ3M^&XGuVCm!vUdh;#;u(T-W6*NK~U> zdIQpoZ}&X+B4TkhS+wnpJtNyCY}}N`*L&``-rS{(BMN--nP%hISxRK264^$j<2oi_ zXIP5@wZ4*R%~5(ThCol;_2FyVjf}L)z?VWmk*}(>%(h(|&b}1Nt&*6Fz zlq_G++VJxmKn4%{otx)2d7%9Mx_z-zXIpnMy;B_DlYw;u3F4k&L7}APkI)ow7A8Xe5@I%^9odeiZvcl0%hSauO1#Okv zWwSHB&)zg@9HF}7(+DT+!qL#yo+h48+Ri#puSA*VhV{9gb*UM=L3JscwxRTq7e-%& z`C#r!0*)<_xNf0!neSyk_@FY<$_@c?UvH8XJa;mPu)8ng?FtKLhx_DmX15ZdJMw5z zw8}U2vYVl=T+@%0RND`MLgLkYLx6Ue%194GAN!}byndo>3L47JN5e<4f-e%s>J6UC zJ&;t+DPQo0{1T-Z2%%99ZPixHmXb9fYewa|G-Gah#GuFH@tLGH`1xw(1*B?I1l#*< zwrb*hY~4$Lb}64h!`uo|OoNP+Pzu~OS$UOZmy5~SN|4iUN&bCQpfkSOJZv6hx1hC7fE!3BZ31$NGS71FGJVqEl=;PWE^nJ0+ zG-cLQ2`Tt#spz!VQWDT+K-MyQSlCZFra^Q4M{UD9VboKL6%a5xKDc$xU8AQ zsnu+QEZwlDwz@wIgX7NUXi9JLW^W&6_T;Y~j(deUgDhrh1T=lb*Gt7lE(*qM*Qbl$ zL@~kidn0_Olbgapa-PD8sKNf_uECLIYfkf0%lZkmsZ;&q6VZhRe5SeD z-9}_3$@knp|1tL{#Qpn;yI{fo9>!B%X|DM~;Cv{lm|1@4&9s?ak z#HJDEMGHfaWZ4E@-65lJhNuC}+uL|qwb6P+QHWrK`%Psd66MeqQkdFmxF%{(lA;=^ za}x^lnskMo-M16E1vhjIRkCaj)qw+7cjE59Y7w?}VRVm_xT$gcI( z5jn4VU&IqZ>(trbt`lc(*9_6S_|pqWTNB4GvUb|?IhdNdCusfmm`_`#63i}lTyL@j zPP|^nrkm_7F`GQ(PVi^n&3_pC1-jHq>H_Tr2E2&TDbvfvpPop3)<#DmAl^OVI^F$p zK~P}u@I%h?e}H5C(#w9*8<`XjgHgNChm@oO8LE$j$i?_4)#=u?Ge-N8FG0JWYz?Zc~m^w39OL!-#nERdZAR9c)V% z@={%3UYy;{j;uNI_PftuxENvZ_7C^OIo|vH8#GEh2QZg%QM2YQys>%+mrb_y|It6D1Khr!1U)}7U-Ux}%HUm? z-U)7?@SI>19JcchE{zw3V*Rji9h~I5y$E?6PVC*dT{_v@!!&6W!mwRG1B}sRPW65# z-m?OWbcS9QP45m}fFm-av+q2+UdF3lTot32uIZ6Uob$1{>-9lg+OvE9EDMOXOMyaS zZ_ctSH!?xVZRWmR3e>Qx6m=q6a%#^vP^x$aYC-Fnj7&Z5 z+-Lb^KM|$7D0NA2l>JJ?biPK@6ezS@&VTo;Otr?LVm&oVOh94y1yF2kPL}0Oe)ge; zLTt>e4W>ipDKoyZZu+j|b3No6vbSjfYp$zno@_`RcWjYSKof$xa6zFx}MqY{B7e||`E?1js zX*5lXQnY3}(^X|kCKb!YERP-xJZSwYKNWzzg0c;Kk{KC|Kj-gP0_UOmTE1dFR91eUmWSZ{Jy5<>2|}WBOnK#jkI^iW#tGjF>TQ01WFB5XZ_$pu z7Qj|Xc1M$gm;5rnzxMETbi4~R4)MfnO7vhKFz|Kns@wICcDuj7K2uXw1;%v>c>JtM z5R7>7CH!z%x~YdnD+l;Vq5exLQI%Bshs=ZP(^WF@=tZN@ZA)>Fc4W}Q#)S_E&0 zN1Z|dDy1Kegk84na%Z-&Yq&j6^ z4wf#~;|nZnej-{9@Z;>f3l|ec6&WJbME4jDrAvNdDT+JQXxRx8=QNKpDtuTq^;DdF z9$N5QQU{`>VJ za$3_ICsRpTt_l%TAhclQbk=ZcKPzAK``uLnU`n7P7?QCZcLKS4byvqqw%|T&7`@Fq z=}CTrjnCal^;I1Jx zCo?91)r5de^iyT5^ww@$S-`ymd=i71mN|pdshP7L{7K`yl)hwEvOlQmh$BEsUyXBB zm8Z&3Tgv+tU20#EVCQsoG&=omGdoEfaWO-`*>4hzU~1JT^Ds(dENCw zNO=}=xX1B>EPlr6phtwp0Fs4)EVk?X#qI~3_>0B+SWfV$1RgwG-m+8QP6tJ4_1_>U zQlo@*?tMX39ruv!W?c#Aqovio_<2i6x#jwBUxvJu)|O;p6q^Rk$_djo7bl=0SoE3) z6xBwU%;I3qo!E^I?%fNwaq09&r;O2^eGA3*Qh@v;ughyA)*^cY^HRV!w_SNUipV4g;IbS zPr)O%^3(j&#L}q@@7}la0hCAdwJ(*t3oaxcLvc)_X3VW)1wQSPr%1a5yxZuC%rUJ(;mD4ZQnJ(31;+An#p;wA(EN zh(fgcj_2i@Jr%xVw~P9b8$IYC)wgE2fAyLJ+G__(lb1|)A=M){FJp zILQxh;TCk{DPHy`S=d@S0CTOt3@mz95!BP7ab5NUGEEc$NG2P=!|+*t{So%kkEIb* zVSj`G^2a2y&(CUTWMpMxa*~Z+OInk9tgh~2_!7DD8$oqlhD%SxpI-E@qdzMQFjUif zx%z;1Zt5pr(7f}Nn`K<})`V(xZ-u|w)-Dm$6S^>JgTVJwb^;+TfIMuw^{~@`4BWy} z;I7YKVFEIIU?5K_0K;yR$*<|I=vnzNXkz*iIDF&2bE-9R_mpcUt|y7{A7-=|!!2Uf zYP;^T%y9Pb(JIz?g`wlN*r?4sZBUpIp9kj}%fNQ~2*6*@`gP-+SNz^>q|=qAgnsIT zQb`ctJ}je*OCta&@3hv`2l*KJw!<0NTb|JAXMl^l0L08RU@(hRwArnoZiU%qYpk&L z6I?sO@l(L}3hn~v9UR@iDz1*&qYfHalW22_@1 zn2Jq=<645}6rfcWdO0tQJQom*0f#mu+6$mkX%*mJj^Jx#9a(xTkif^PbZnRL41=-vLP!+t;yniO17`88Csede;E{H*M!Pp;lf2 zB3Z8h_fe9s2xolhN~9r$OFimXv_&WRr|LxiQzB9e$2m8Ix2Q;ClzG)sZMMMMQ~~gT zwJ{<0NuQ(z_LL|Oc(W)K@^_z1?w@Y*p_La%jny%%;Mg80nN}dHIuWI4{`_ZMr#x2B zVH>Rt&qh?O2(m0^XE!2WRjrgH(w|*WL!~tiND#M*pR7F%?e#JiDvrx0Yo1fJBO=Gm z7xCcT0%t1xp4DjWoCgjk3(YqhP_N{;HsjjZ)UC%;mf`^H&9e4aKVVZ%6m&a`n!OZ6 zrdQ4sOz?C(fzsa8Eb3*w zfRU{M(VaKJSKG&V!Gd9Dm|ft(H&AwSePKwof)E2`&5di0Mk*;z0Ff(sA^vXtmF=Q$ zhhG8smZ4v9#~?95f71rRf+jA2`{jK1IV_gigMECNCfY$jV*;iz*OKg2?g;NX&QC@Y-MviHd_XW2|`?XQsJS{jvu!Qglv!l(F=V=Dn+I`yD?Q)g|MQoKj& zj}_w{<>xl=C$F)Nay}mC)A&8mv7dmX9NMzH1OlYCX@||6^L%(3yhRO+?B14bz~qKO zS^Q~$n-_8`R$hRJ?4W{(&!d^5(igM9e6h8n?sAzH0@j-aKpM4J79bKS$6^OuW|V5I zDhNMh2;7xv$+hN!0D@MvgvJ9^AfaGoZ44IVC@45?5_$Q;g#q0d_;{;0r+u&7`UIAy zjRlayo8m>LPQJ1O(aZH}FUN^;4?>m&;4q7#K*9lU!Rd0ZEfZljulC#^zrzIM?8^+` z9#gX7$Ycs&yhZoP&=((EqQizk24;TAa{PRp6$R!dTgl~YS=Mi%plpiZ$HL>R5`~oJ z<&!O^^up@t(-j;@c^kfYoa;@@BqS7Pn)z$Z9Y?*3$bM`(dc3uq>uh;;0}2CYTn8)Q zT~lL><=aWL?e)Jsa!!O*87pt=I}rOM>JYDIzZzrv2`8ogq7j)kn*&fe-ist!%3s0h z8OC4XADvKPRMV0Rzds%+wss7rZ85n13`|z z0O^M#b%JN&Gk29K37Nzf(6w zV?^2!en(TzN7Ux|?zkQRD~D`Ojv3~d#h7dIONxMt%y+>8girJP3>O;-c8JM=9$mBL z!7d=!*r*AIhTX%R*?C4YcJZFnkS4o+GuAU-=9zeJ$y-DdjtiIEZdm2yxOyo+Slq|Z zS&>pE#=bzQuUdWm`a&cUe^0tDh@sV>v>eVWPo;rGL79HqtO>>18qNKg_haS3=BHKV zVd&y5pW%_w0@WGO(FF6a*2P^1JsBlrK3k(@x$SK{KQeo62iT3F*RfFiS+@@$?mrv;OuvPQ(ZrY)Wq*=eM@X_AjIv)<0fMkm`cy++2r-;O} z!uoo?mw@K<#QFhP%5T_feJ(l35cuXM3;=*BzdLT1$V0F>lgOHXmu2OYM&`07s`HuK zgVbVjtH&_!k`0ut_14cULVX}ErRK1n;$TVyX=>sl2RYI7aaiw8U@Ly(L1T2_zO%;t zyvwzIre&G4;*QM^G8UwWabSd1qp>cbQ>V^jpWxAyj(yfXkn|V-eY*(V$dT`}KXsez zGh(-5YKccz0oi%}dH4y3 zqut>1y6=S@q){wWlPnb|XjyY?H!rB$&D4l%Y3XXSmBhu;Pt9owy|;AU&UuPnl(@Xs zV)z`1cDs1ip26kc8%+{IyAr8Ez1ZJy@bG|1mTKD2poS-MiZ2JA*1vOVUX^c3$P0mC zw~L~O4Dm7Ci7wM8^u}vo1@GyXx1f!@-Fih2=Muya+a6+ki>d1|JnmihCE3&)8%k?Q zjy<-UAC*0gaRtTrBF;ZFF(4gf02nr+h+gTwa|;k%fM_AD*S;wG zc6HXH>ruc^-^B7d?VzZR{9|MO8tVxncfH zgnFda$Bw%V5JX?I63_r|_)uX1iA8)90Ic%5QlEEDnpEY0IyGuT3r35j_fXU^nrtX? zxA$+qavRnjrUrh9RJR-yeqE8|_pZ|!kf)6HN&pT`ONml1gJAj~@-QmHYQ}!G0=$xA zWN9DxA8T>I6n(5r)wm>Ar0aXUVq|d;s}Xa0~?wiIkRS)pEWe%5qa=H6EL5)B7U^@Za^3$ zMXz~xHuXqhq`O7t1Bm#h2(EV#nDzPznO!dJY(dgjf8ZG4p8;Qdj)`p$NkBku*Sa4E z1T}y0yh;9?8@=x~shEw%=xqCW5Xfl&P_E9s1h&)#)BOA>&viQ=hI#QP!@J93WE}58 zwW|$zlagUMOMq^Cp-gZUKwXCdG3?%#t-d&`9%uxR2Wa?PAP$2f;#E1oMyvSpC)k2; zr3$!b)t-bKKrdE;OlWIP@u0|y9S~JnDb(;cH~@PK4lgO^wI27El`yWS)Z$o@{y1H)QGa>N9v`1f5)wrSVxnxxv$8Tpa)7m zg5RO}W0ewck`AVv8PsE7>5pSDPg_Ydx8Yjq1$teeKtX>Q=tX5pnCiQ%PpwTmAl6<~ zKLKG?)i*TGu`2-bP5L-atjOd*u6D6ewjba>As@vxeDmhV4sl5w%2E#s?!>Y20*D_T z3lPbjOZ?ElLjLA8cq{J)w55yL39++06Y zv{2T9rBn{h4Q#w7yFg(dG6(Re5=t(S(AHfRIs%Ex6jYDEZjmA`!7k8!3{X=!;3^e% zF!@as_Ntacm7waDDL|{GXz~$7jY9LuzCM_O%`O1PPXnYFoMu3OcVk8R)nk~Sm4ht9 zx!L>6dISP2p1|f~kR>vgCP&H#Q4|S@PXx#wkYuwgB%o%`Xplf6bYsn-F=K7~x8e5~ z>IxbBfhQmr)NF$5Es5o>P}8(*$h7pmYeEljo{w7`&+6A9c2y@i<6 zA7e)cwhU{5ij!!{3S?nRwQeNyVL3hL$p)U{4OFHjblz^0RTvUo=6V$R0{*ufoE zj06xg722TSy^~o7+4yxR)N?+C4YS|^biTSb%rAIKB+=^ZyOc(M1Ug9E@Mg!)?{7T7 zF^Wf1oztHNDb}<{x>KXftI?M4*aO@`Y6P$pn`6KX^PpwoaW@V9AJ<%ttM^F- z3QFerdRdXlh!KfJB={xksfcx2aNk_7Jo2FQW_9dg-IUFH*b>c<7_W}{@;LO_f*dMXx%ci66B8*p^1we`8-t zvvZFQHbh<)efWK-&HXS-GRdO=$!@6~ysm*BoR*11WWUF|wm|VcneRn;LJB%BA2AFu z;a@)zBM)T`X+Nk=VsZd(WU31GwOF)Xlu2bqL1x%!>-)bS_SYjt;{Dc;5I19<|9ccaQA9E z$9;153f)}X+eZ@aAigK>t|FCmyB$Ts6S!oImDmakjN4(Veo2@@w-d<^|K#@C)YAZ| zaL1KS$v@N04Y%+2CAhKL@CoMo+iMo$+#(u~2uh*bv1@08kwQAi6_z2qbB)7I>LyVb zl9ha?dE9XHME!{ZAVa}X>q?bE_w+N;%Rj4v5Jo$o9?)PY?&yIR<&T*1Ghf~QY5WP< zY^AkfmQtBX=^lpaZEN`BA3Y|UO_zL`C71IlB7G2~M)XyxY$xOMo`4xVTWF^6w=M7J9&Aw`i)j z4Fc1-Dr}k;AfGYz6(0GCdzNZVB_4=R5#8Aoi z1>NV+|d+G@^6A$U%^4em(Ob*`?C0E+pZS!?Kz7p8C=_VXPB?>`1I zfjbNMfMN3@B;!}W1BpJ-CU`%}G1`n{x6)Y`N9+^eh4b_3t|xlAZk${xma)1>>}NoJ z(~FKQX4d5*M+H!l5C!GRBtc(`d0-Jx@%bjmbf6k4k#y#DO-FNheP+`YlP7^Q%c9eHQ9Hg`?- zw)06{9FS~gNhI(l;$u?Ey$z;N?v+=u4#BhRe&bPS4Y1-QD%;jvBML4{f9bT1QW^%H z%jvvW0g}>+-Se0F=%#@00q+q8?tgD2Q#OmAv%?09`jZyaLLaRSyezn<3#hWf&JfB; zkm*v7NQP2MRp-hU9VCJdP}(y`xnj`a1Y)cp2u608v9LF}m-C7D;ka&`F~B=jqL#qP zpQIJvZgV%{$L$eP@7HIXPgI&dujRNy^?1%VS0}xnuxOlDdt;^-kb_%vW9?f7e?HUT z^LS*=BatZ)e}25ZiNT3cH;^PW86vN`jzJsO3y{R7a%=ZwmTXoia+TRUFJFV`>*j{x zX&{8e#z^t8cYP`3SNvhH1t|zT#i~tD&az~=1bW<^ zaCPArF4tOV^`_Ml+Pgcoy@U6u8re#3&FO-ViTf@fAGa~ssO%&ge7)np+egr8eC>(l znC$5?oGLFUYIg5SGD;-0Z%FjC9;`B8^(!$c>R~$IkuotC5|5y7sVRKGOx;P;`TCb| zWqF5u&Zm&~1foP(;v3!Uye>gP3WLL}xZo%nS8J{g&B0`0_Z4!FCVjPA`#?_;Ae|Dx z%K-pL>!x~moTn6n(O5oXr#G*M)y7aqPmE@r{lSWf2D}JnUVj9ur4A81G&4nW_ynT= zJN0poA5eE(053Y_l6pfn3fO5K1$z51qEO*%&-bM3p93Eu%Lyt^m3K;hH3j-xP1VnT zwMg#b&R!aXv)JHF-Bq}R=JA-B(cJ@BZ6zCE+6}aV=~w#pcsE>uI6};;IpwG0F;LyU zRpoYu>M0L^(3(OkMpKn=o@FLSiFwtJbG!I=bjq7ZwZhq%?Vm8Bs$>c8B?mcxAb0w* zWBqp;j_Yz)n3|^Khst-aDUs!1!w=RE{R?`7rU zbSvLom`&U<-sXsO)V+0sv^yb<3oVZbvyAkvyH((N7`!@d zKal5@4H#6zTZg6K`Y;%ovp_E2XuPmGM!Ni($=879c3r245^izQ)H$8)5B7Y_xCIDb z5SKXI_MOnePfmqzeJ(fLh#Jf%=j!Xyn%kS?$37)n;#BQZOh3O}9)vLW-AeWao(5w7 zE2pxgLFVanYcH0k$6@WabEI(!<-C1|X;Hx@) zAKu)4Grs$AfRw^YqzvXAIYpl}osx99=Ov0bqm zE8rc47FbB_>W+kI1Tv2KAf=e7(2^&Y6M3{VTc@B;R`^f<9ANCzFo5bP^tb_mKmnM| z-(j4>KP;9zNmwh7(e@%D0n@Lg9I!BwRKAx4-&1-Ca+dih&*QJenh+ame?#*&>Dm?oSKk!<3iSBDo&& zjXC|QmO=sm#{dD>?8~=eKz!ijHW?)=7U{^Ph=e3dzF;!aJK7i)b;xBwpp?(u3Lewa zS09*A|Az8N-hd)OS<)*DVLebcy1Av#gD-X2{}U`g9wt6m)P?ZKSlN=@T1P4`CA!_p zD_i(~lm9*Q?GNu`)IdeUa&P#FUUj!%t3?OVYex#JlDmV9TG`e0#SW9zn#2VNSBycF z@^J{Z#bBj7Vg$sl&2Jj(K-VBAU?2=YT;><$%B0cwy_vljsu72}DK)1(cc>3L26s`4 zVlsZB|7tQJ5IQ})X`EMw(v!|7)6;KO%4^OyvTUx;Rwsd|#uyOf$Mc>9FFFITskFIn zE{iA$po)(1IE%@rVk`|I+Q6>Rpm3Se$Z|LwZ>@LU08zp?IJSH{IY^8mIQAxx!le{< zFvJzGV{be6fyb?Im;u~^*cr!C0N<2(wtI2YT*Eh&0carw_Azi?_0;4DN_neK;yUAwBje~@8sMMQ7qqKB2IU>cKsWN~ zY`-Pz9O$RC0ahu_DVv!fX9;>OZezJd#tzvL!nbeE1z91enD3Fh30N>zNBcLVAeaJ# zc9`|f%rs5a7Ci{a52iA-jtFZ&qO*|(NVURd!1aY#n*3}*62(g+)?`X9tY)Yf4eFre z@iEv>rLZS_;Yo{hdMvi*LU@B$0Jq)HgDD_P+j;m!%i3o|t?qA=u5SGFeaf?yB3{Iz z1${x1_s&VaOctMxfbn0Fx&lJoO&8PZy;gxSstVaG3$-mhn1m=xnUot2%aqJO7}mq9bYh|JoZ7-{_O zPh9~agj*%up2!uVdk@yDqh56 z@3Tg5#15)uXD>*~1(7|`83uW4`YhQhul5F18nLNX4W5Wb?_m%nLXkqYbefpg@ABLN z7j0q%TuHSnzzxOZ2-Y7N34S@yg`v8SAdApo z)qHq`=Lto{67=hx{v=opPhMb!zk0fYI|H;59@dzYMAjd5mOo(0xBYEY)m2F5Fq;64 z*Ck6tpwg;QYM+3y&?!J(4WfnVOLjHxY&~d(TLthvVu9*nL`c0u8PX~Nz-yn!3zwB! zT^w&Kt>K0Wan1U+?n8Ra;sNea9L-gE>M1c&O}vEmc|3V4*^N#YZIiI`h2G)7Rt@4ekPAL#esTW#Z;!i4e?RY;6ha=ulA&`a zphN3Vm7>1lU>FWUp(obMd^bl+dZ>ScQXJ#dZ1X$VTWXh@vj~boc zsrr6}_aAN}Q=@vJQ6Na6@um~xm<@A2l~{Na>`a0<%pqZp%%a4w+16*xQh}$^6VsF7>9L zNR<2ue(go>Y17SO^7ezDa&3*6B)AhW>rD{Z|VE6|+geJs}jl{lK@&H~C&iVW98A{cv+ z-%X(7aOZvVdE3S~v~+5@%ZU%G%I-O*Q@Jz9FhdRAI8=qag5a@i@ZWeCKq;v|HTI^iZRZBgJefh*8$?C@5mP({-{udV4 zmuIDB^Nl}!8mpP*o(avCwQXwVKiU_;&Gsh~?ig%AYM<;h7kV88g54^6FAaA(T$)d4 z*}sNhZRw0d#S-@(#-VsxEx+biVJr-PEsKY^=)S*u?^pH~S#bFggEP0V9CMt1>EFOb zA{241K`T;4AMu?x^Ttw3bfXdF^EC8da)k%k_j=_Tk%gsNv;{0FcqBzq(4Qcb=|}T# zx~Xj{su_Cjs}fFL&4|GD?7)uo6hjEp-+*sCcow{~^x^VOU8BPt3*4 zDH(1tTDv%ufGJXup}2F69I!*IEjit{;ENDo-%2SKD0F1>yUeZ?A-To3XQJ_dH}Rke zK_)+PCw1yrgd)aw)*DNSoSlDw3Aj_|qx%?*yYd4fMW0iy1%d{tst2*|09UeiZoAkY zMW&sV4?k0+{y|?L5aG^+5>uG^%YT+Um;n0^fJwq<_VXsaZAJw#Ag1J#L;P>QyYvQh z#D0o}BJ$2PZ$L~b2g-5Y0?>dqI0_S>E-JNH7K!3;w86)g7Wmti|J6XUk1)&j(oEh7 z{CDO+{fcBxRaK=(=Ochu6X&8ljJGh{?I_@js%i~BQ)MRAZ3E5Wiu+&r`SsqS+h0jGa}Y_S1~-DUM-{_pR7QeQr5W$UMN5b>R29imX;#zL zqly3#DStUTSe*o5Y74l%VyG1erBeL$?sKRv5QTpRWJ)FAc__NV&mLDRS6IkNNT4f) ze`stbjSsprBHzChVI-!9q!t5*7s+f231-A-1T@(=Ntp6y;FmpS0#CVifuV?l8UVQ( zhoj(+fx_Xzf!4gYL-ZMOsoR;|`P`}sV6hl_yXMD2W4{O@i2s}gASr~WZM`v60Fpy9 z6tbBz=|8MDM@qon#D0f$HIOX)6w0+6nhnl=&jG>8pk)MZGCL;awWdr;b&0{S$cqqq zMnGSc8a1P&3Go8WxECYHed$MDtCMOtjje@RFoLoshq0^PpWdL?ELmJK-|6F*f&gl8tWbr**#ViY5vbC0#~1{8wrOagL0bR(SEz7$8_4Gi zfwZRk3WVWeE59#6R(uO6rbL!ndZDSZeIj2_@7M?un?Wq%4p7Dly#1#KG6i1fVt93N zV!OHnij#5Z)W1yLfRg5IHw66m%?E59jmcTCmr$D?hGu;QBJaP-uUQ9Gvu$+pz62b#exps6c@>Mfudqs7Ud za>Jyrm?ltA3Dyk`t3@(Y7$_FFIaN_q0_b_g{&q06$7Yg}OB5h78a&hoxYP7_NLri~ z87GvB6XC~lz+wI%5nzjLC;8AghEm013_yP}=1p2Fe^l83m}$Fub^@ZDNx)*REui7O z8`>DnGu@snW8ROT*Y-vhny3R(6=J_xVXOy~>Xq-}jnMGf$Cl69U;2*$O3$#y@x-P> zM@lEI?vK*}Om|d0*VE+@UJf>!tu>1)z%h4M1(@sNgOJf{kV+w+u%HjsM3;84)z`90 z;a-i@a^F003q_gT-e=c#pyMVWvY+8Ifl$nE5K!qynAkYv>p$Z;_0bI;revBT# zg_5c|E`~_t^+UO~+(!z?p!1Ha5Z~0NSKB#JE}MNxb_vRm!Y-PJCq-OgJ*7@(hpm%l z;a1jPD6z!ylx?>sh83}~X_VK3TYiyQG95`P{PAUOm#+d;VaXFLiWWlUGwJI5;a`w3 zWWU#$!9&6_k#-@#=_cV9ZW^=&y3VM$gH<1M5NUy!e0u6DTDGIYK_eCm0qQ6{5)oxA z8}MhrTo_DPKju~hz-F=yqqV#*`70bjL8f8bw<`R6?m=6Q(}<$3c64-^6@4FC3v^l&yd?aq6wk)LI2>-J^>nnu`bwhcv*g-`VBa`6>Rg-&1;uASo zh8;AG0YniD+RD~X?d*KYC0m~E_EuCYbIva1F?}E~OQv-aBnLYsp@0rvz;L-U#Dr!6 zlqEeD>m^ScRdvi!dbr&sMaJ#Hr6BCf7HaR1EzxRm3ZS0WGslvzKyT#ve0kB?Jq>75 zUzkfS{Bq}M5R*m7P6WoyW<(xqtp2ZR8GY=sZbJ+SgnwMoh$74~Oba>P5N5KI=k!GS z^G;ky@9wCW9SI7hZ^3bbnTCBdgRa=b!n}b1k-&3pR8<*;_3ki<<5^eNG%dkC-HWIe z3Mk*8@UqCaubMxYMUE})mtMGx^<2o?Y^D`PPO^bwOFO?5@w#&D<8?jCKi)InFj{l+L#gS3m$zOBvyXu)v*!-ykWt>;KcA%NHYe2biXix^#?Klhk;kxB!KD zif(at{>%6Zrk~Qp#gJEJNHB<9+bV6gKS_0vU|61eG zd(NwMH`kNUQbMJ2vjkC(N(>B)??F@rSD>y_5oGGO0eemx_!df|L6ywDxU7ZzgVMx* zW3T+$C*WjaL0;_lZG=(z>w^?t2lh!A0C^45RIU*#ffR7fekUc9{@^RVy^XPgBmjBp z$=yjL=YPG2bvFWaj!~E5q$%}r7lTe%;4WSevpM0Ld3P3 zb7L}J;CBSzKvNVHnzFIMjsYS9ABQIF^#O4Hr*q$o_ZC{9C625u=!gtD-TBayM5xg) zIsv;mzBOJX?U4i0l5Ayq1Br}{GT90Pz*ib|^yWNfVd)24ic-G1TMeiZgdBE9Fl+;r zw=i7G(H3TNbA$S86;6vx$epBLK` z!SGt&4)32e3G&{piBPyA@FC?{BKksLHQ+Ui`rx)G4>ey`cz-+rF` z{>L%L;fynL&vmc0&d+%+7FP%NK>$__hyfaFbD}&`8cl3K!Tdh$0p*DpY89AKm*YnM zEqds%5OwGoBoB2*#pC3t!RL|eR!k3s;vcV)=KQ@UG(;Fqtx6JrTwB{XJAgK5%3Zys zT`Q>`bPoT|dc{gYK}6dC=iR@cGTopmLl>+-8hw?v3Fcg>Ngu38Xz#fkR4f_+U{%u$ zZ3OWVQ>lZ*fW0bBF78Y0zQ9mQWXHDecXQ(B2Dn}c#ZADWxb|pYj;sWxG9|r5FrnwP zsV2V*wOAU#*Po6N?)(l!ve2(JWj&DvaVicZOP`eQ*rH=c&MJsNN z!0GiD6d5%jkcHnHwUztS0r0KI=s{f)Poijl4J0OA+Fdh9d7;DkDWKjSVAqt zIjv`=5iM1AQLOM&wx5Skg!LVVFgo0$`IZ-}zVEqNTE~P5>p@dBsku+RAPwOSL! zisu=0?v6icYEw5!j6 z%0G71iK250b@_Mzy(CLsC{U8;a#SB4mI~^7;*7zU&`%y3XZg_U4#djv8 zw+nr!T`H|Ao6Dg#5j<`&eBeoiQ}+Dsno(V+f+TWeH6I*~z7zH-xF0Bj?hS1NNnLfh zcUoqZ^%Y+e$_h{k<7GyTd8%gceKsRKv(VtPKx6t@w<-Jd$39>>-^dD#XwZ&l-!D~! z^6J3nbe`#;<^}2eXP)673r3U*&jyq+i8AyHMze9_Pr`M3*}n@usCh-BoF#)cOmx3C z$BJMno3TVT;&Q7e&(oNis#vyTxvXn%Ao&tOg8Ct&I!BX6;N!p z0J&L<_EUR2B<6EWvHPw%1@HuT)P0@1AT`IY@#jN+b~8=(gFUD{qi;STp@^o;-V^aP zlKilmCeA%zM0PDs(#iX907^JSFs+>>jpX~q-bI@GcBtEOG#M51nDXmrG=sR9*_HRT zY#ui4DkTZMNT)zX_MS&donIG8gd*Aw6?l8a)PbF5;sr(_+A$3}w!-@Qy|)t5MK~1%oM4)ky5u(0^ZoD)7;I(;ev-StDM*%^`$a z|0>R);_`n4M6+rAh10JpKqys&Waj*Hn1Mf*-=p~@%FOz>3klq|gy4UlbXgjVu+C;5 z^Jf0fWADL>GdN~v=F)@khlLor6dV@ zzINNvitXRGccy7t3JGR(Kx6}WD4z^%gX$>*&FbTxMPC@O(z$^OqZ?brQ*g8iOotr6 zT~#$j(>6_FJufa2lv*IqiUgfqmh{3|weSNL^2#+%L#z@lI=u$+KHvfEEN{a<28nFCi2dBOZx)hfNtVm6_~+rA6E-l*LqiL zm@GsnvCg>a)VzqGEry*D=;hDi&@>^+v>!A6^=^7Fd)$WpufB2q1$gCLO}PBCj6SjQ zc>MJz<5;L2FoZEs=2gX_Y6tkoOv}ynJ9PBbe^f%?MrmZcy$)KGK4Pzwf*H^pPk||b z2(d8)*3g7lhy*M(z`vO`(O3T@4Vv>jMpLe)h+}@Lp7dooO1Rw z=-)McMVt@O(S+i^RYl23E*uhveCoQGm z8#q84Dp^NcKHJZDOODv*5{(9Hxyz#N8=d~~?EVT4;)V`@`BWB{HSjXr&7JE#t#zD} z%|#Sp3{N1UkrSFzGC5K(1dBzg9KCXZ2X>(*yr~Fob!K!LIh4ZcH05^}ek+ zx;jFeDYpxx&5&5dCau1+` z7>}CwMMb4W&m{0jz5*X)CE6)Cn3G^vcQI8RsCguIZkU5%!Uld{&a<- zLG4EP0+5>p^`yGOU=&2TXGwQQ14*R5fAhQ_qwcGGw)pAkaa+8~OIacUA>)6BwQ-H~#xWg<~1*`E6Ai=jDGktL^Qo6%1Y(8#|R=^h)05{H3c=s9%ZXlMrk`&n{OaC>GJ)4iC6?JLjP z=LPC;aYCv&M#rY%gb71IG$a5GDGeR?{J^l|^LLE^uh3FGSyGc%R2p|kDu@)PLiRtc zkt;JTP>Hc5g@yx5-k?LL`s`l^0?N~|ItI0GQqcLPn(LwXk1s5F<1=@G zu#^S75<=tx^tqkAL~b7M1Bz%jn!0FiWb8YrYVkaOe$naRX>QT5dDpr#I-gw*sP(eU zQ;Xp(kPx?hPVrYo4u*pq@hh?u!G6Ch^YoJvS1i9knFZCJ)A?As@6dx@&;5nZ1ZuFE zXGa{12c=qNlIP^4#Fw59y>#a?J~vUuE{C^K3@sr`$rPD(RBEOLk7oexwi*!HL(@%q z&ORajm>!~*teY&fhV~G++qd0+!#rzWHv=;axkyv1!Mj;Jje}Le2PPtXA2r)h`prC- zL#@rM7lki(#K~leKno1wj14ggg=38;}m1!~NAX*Sar1 z@k{UP45Z@Dw%JkLuP)y4r`u=q9vx*@C3>4}i_DQdW<2`YIX`Dgl789@vw1@=>^{9` zRE#xZ<<#X8WWa%yXhE$-s7u~wy(WHF zD>yMAf~O0cGYW0wFmi(A!2_n~CdL5#>S#?40}U$SVB7mCa&dfKqyCI+^Iz7`}W;t{~BOXbtjlir>YSxcl>ctF-&JvFHpo zK|1jfH0wZ&O6&RTz7R}13DS^YvO$IoZ;LFH8M6>d*elrBOQ0{PK&KogEP<*n8b|Fi zgxu5WHFKv_=wr1^4sd@+UxVhE*8oQ7iSfEy@{J^@?+cqK6OUK@zo;b%hOiS7PJUv% z%`YkEvZ_8Ij^_um7$-qxmxid!njQ{?@cA9?fQfQ^R`4qP@;4Y||0M~IHjIJcK z^zqfn&GpVrq}?Y*?kigCh|QqQzW}nS0q8`H_T&Of(a8P;!ByDi3EH9#;}NYgZO1s# z4hpyd{1UZv~TYYI;f~Z?|gT;0}+l4 z{xJ^olc6)qS3JvtL9CdT%n%TY9ytft!OA6?h*Ti5KLGNIv@xtu2I~hm9f!^r`j{MK z4MD-ekCsCnq^NzEz{=y@Tf&+jNmsJxM52*VPMyp^06}LT;JYn|z=^FUPnI?|`vxTx ze|ul&4bX+>HPVi7r^K#w9Q;rO`xsqf!fU^=*!j$AxJmSHU0++ex%?KN++nvHzo4rQ zn7?BnSRsTxRXJqjFxukX&TW5GH~mmsLP#^n;Ql%-qtDVt=q`rCp(KpNU=>e7^EJBp@wuPN|@#l}@0O-_BE%ujYF~I7_ve54|rFa?~!4f@)C)#rc z?kexL!$>2tXmJ#%K4_*({YD3w;&`;4V!7|2bDB!{VN^tKr)@^?Y3ahbXh~E_Xvl(6 zpM{bzr#;>WZ7jlP+BIdlWAx!s=jWa%N_93RQ69N6_-6*X6Uj1aD~ z9qmqts#P?lXx5Pf%}UtWw`nY7A`UqqZ8@bg*NjHn(@hF+h6TcmXG&mOf}jRZXj{_!0^Z%>>)mG<2KNa=%M^@hzXo(}`_ z-{vY3evmgCh)Y^45wcKK$@2toRjYakP}7iN_%kvk zerBneSkH7I^}p#9cfwnr*IkilvZaY#Dl)?NYQ;8CM)nJNCZ(=7d;|tWcDUp1ncEMC zTVxywFI@ga>G=e@g-M31S!+}Yi?djo7>)fjfbji%ZR0vw>I>aFvq^o2kzYn7@`F{z z({q6<{9AAYqauWtC=H?oxd%7 zh|W6)#yF80f%un*re1V*mg`DbV%%KHa^M5p6&$M&f0|;_#e@tnH0!^56&+JI2*34; zNETuM`B-835s#3vx$;jxz`J=O#}2@U>yF|LBqX$IGBsPm9~p`{?CCJ{<7fo@W12KM zAzvcjBHL~oMezK%0$V3Vup(Wy_w;=ouq0>2yiMs0ovjAD&{4+m!ZBX`JTf>v|AjVC zGO&r(;XPBK$&=tq_-NL5c1Upir_hiJQ}TAN_rdQ|c~{K7K9brK=W$?i$|e=Lp-}09 z7=2zih^G~&w?YPMzKbQ5k(dRj)&xrjeDT;UC?9Y9P$7uXLd+SA`WmnM6!86~CCI5< zM%Kh9=))zfB3;4?RQX_c{1iI99{U0g&0YMj6W#Iip8`g5i?>Nncu1`wsesqAzrX%# zZow}4JuFq`(ws+W_z7I|n<>sFvAt>wlurlebO6&?2g~m6ged9#$UvE7(=JY*_%AP> z^XGV80j6o2EK#vcyujFU={J61QnVk}5`#}-sA`#u(6kU;NAj_}(NztNpKOEG-<-J) zyafj9H+$>n!V$tTvPu%&1ZCEA`%Vv&B-LLa3L3cEOF>W*vRH_3e|JIXhyZ(0ki{exMdI6V*jf=gjlZHa>!mXskg z8~h=zlVvTmALr9DWxv(8dea0EcUO|$snq35e7R(=$LLnzcNE(fm(Z>8u3RjBZ1V@7 z4#`#tGX^z%9({a8Lqf>CdGpt6Ts^@d9-FMR;^J#xcE>_I#K|v+v=O98RdQ8RC~)N@ z(qRorED@b__F#vxhrCN56nHK_yhyr2-S^`^n4IwNY0C1!*;*!d&cfF{LUB4Lka}VT zOfC3aL^23Uoey;VvBV>@R%#4+eXI;_MPx#ZY%(ZvaSI0`WB(@$W@KIGZBe1^G}yNm z!aid&5}EUBnwdsDC#Vbw#$HJkJE0LdHroys$>L|Pv4Ex9kYd@<9DF5n7&(+Hi9Ey& zN+o=brH!=3j6YV#O5#M32D-mkq*LrDI45@5_^=t>bjfQ=#d-QH>E`;RwJ4_?F4IrY z#}n{PkkJgDKuh&PPVe;0exdg90&+i9^;teONzAi=dFI{N{*AP+4(sRS4)SGucQidu zEjEo-WLpvTr=Kk@&B>VwuzWrU+oSbtxPF--#s>9s1Drx~01#QD1d*hR}yTqN*#uFHJUH9#(hh)277cGGm+UC%?_+72d_OvJXKcKK<8#zuk3^@DX^XB(-Q7xoGB-XK> zAYoea%J3bQ-rP?$;Tv=a6fwVyqNs`lWqVGD>FNpJvd6il87@Is(x2z|1fh-(kvW0V zk=!m=5ym<}+}80C6$!#JKWMmF94ZiwNO0vE66!e7#lur7x;QDD7er1y+eO}sv=jI8 zg%t7A$ZEdriyAjTW})q=|47B_rr_W8HL!ucAg^Ce;;0{ckh9KC@C$9 zvwKzqX0rJv{dDd^dHf5($?8@gRBK8*)~q9$Xqxbuw36-Tq`lyO&w6{%&mux#SFLS6V`p35>LI8#E6 zZU@7HB?7m=P4K0$YoN3L(J-O7!|GqN6@|0!PoN*#qmup8Wr^GL+xbm_@VwqpX@WgqbNRdKhfMC>m(jLlm-#fx zg;=VDoy(yD_ZJ01Q}PmciCiG$t5!LXIPQZ}5wm#Sfj{u`do-NvP z2aHyj5pv^CsK#>e5;@QD?3s)Ql6eK<{Ypk_NbMC1|EtfEBxkGw+O&TmTrph z`}gtH+aoJp|4ety>qEZ+0V`*rW&h9D%8WVvO63VQhRP`e7xoB4&!%=kZmjJ^#u>uG z;s*})8t<5`R`_(Iaok%Ay)W~oYUR#ki-%=Aw)C^e9BOtHYt4scQctGjGP2GMw_iuq z{+|2#z}{j@t6`!m<#Hj{U)TBDhR!!LHH#xrhu3vbPFn}5WHodx@{B%1tRC*vv0xiN z7@gA{HD-v>LCX?OtEI1wvzqXL=rcqT@_rV(oA~2x<~oLL5@U|{y>sPQz5w<~+Yik5 zKYNX^Zw@-x@42Kof!K~P%zA?g6Ott8^_LPI2M;#P(lfAv5 zm0$Q)R=WpfT@*%z0}|G#XU=P`5*crs%`eZ)%pOsfrfNkT&%e_2J1T<=o*l?;Ap{77 zE={{IQ(`vjh7ky(l!FNBxZ3b{euY{cj=8pHOTK*_gaibU^wLeKV+X8hq>LI7bMUJV z!FDjiL)$xVe<{ZbzvX7XL7r7<6JUS1@6onbCda~sim&%8;a6<8{;NQ4rW#qUf0yrM zw6II8X-MT}YNxP4$y=f|KT0Q~H`Dom45>i5#CZq*Ty%+Y+_q%L7Zt&$z05MMQ{s0` z+5*2^>V9A%+saw|!Zn^Ww_sO0E!^KylC!$3{ zC*0d{O~U@6^4oqMh9dQ}`;P{iKYH6p^GALVa;zzx>HXfC$s+Uf+V{lQ?uV`WrpK}9 zxa|+S?FSFkUa40Y0JnP#okW=pH7C-`h{FgKm?7k8^E zlC-urTcKoBQK3|?*tBw4ZDMkZju3@DgiZJ}xezG3?H~phG+;>-pm|>6 zkCu~9E2g-G`0px`k91Kam!#!In72i-y%~sF4J=MMAhH~b5Xm`J3O@9(*`=J%U7I*| zM1@=kPwp(zt>pAWPo20u6{V(Rif9SS-EY65$dnQ-Wt-aMH=4Yb%8S`0AfJ_aZzU#E z$8W??bWW~H5>93nZLF_o_FV$|z;ZfH_{eqXHm|{rmVSAxRE$;RCOuAk9FIiPk$g(994XImhB9J7(~w!AjXYj)#7RAT!TkFl zKRn{afQDCoH9LAlzkKBfX@ms-pJwPE%ke%tdL5>2Uj%jT<;_3(bc-@8WeZBal7AKrISERDH*~__L?{8yPNOivyK0? z@xN+Ko0rBdN1_9QPiZBx#^tuf`~RcQfc?^razKwMjLI|rA3esFa(<@9IPKXVxsxT* z*Z&|;z>}wajDOEBbpm^zfVb{Ec|vB_R^T7e7Ld|y=L+vniuY}GPHJ(DjR%qxPW!dW z@*e;EJl=3aeLPVtJ~i_D#@jXza&}<0O?;7VXOVfOzsMM=sGci2-9tWsA zDYa}DmQZf`RSc!B*WXIylOC3;UJ-|VDCy{$;x&b$ga+2)FCGDhLLy>r(so zNwRA2c2~q?$>yhf9=t@MUd!$wpOpqoHJGSQn47?9Q|`@F>{A^=xqJD;qIEd=W}D~fA}ia2mWcL zvSlx-TORyr3jOIYHFHX1{sN`@#nN@so&|b!&S2JDVNz<#NGNMJFWWLxVJBDoqS#+9 z{jFh{H#R6(XzgkSuJv-QdY;fNW`7R~t_W&357jMGZGNR$pUtGpP~#+z|J-_Xiy(hx z^r^nX;xlX1684Gbz)eE2mv)^^3Z8x`+^!hbSgHF3HnS3;81I}|4*SWyR9hk6-XBlt z`WkU0EAQDrqZ-N*Px&hPgu$AS<;$?Q=#=T&Z85*gIYXfdF zgBuRD3JXuq|3U0O)?tm04XS2;iE7xgPGxe_&H7b3E}tZntP7!~%8Fi-v7io!al$p} zcU$vqxuHruI4B(Vu8S+(tHs47P@X?fQsIbV?H3|VH>>tL&C-^-{4)6^!QOW@u9#YF zga9d&AvW03x-#GP_2;OVYk})R$tnIydfFd+Oq!*UCJ^ zi<%x+Zg!KKkm45{$u8?740f~5NoCU-wQ^%X?qG(v6B|uixfJ~R!q$k5dL{&tJv9Vx z^l)K|vJ|qHO#;)7H;&cMS!z(OsSTNiX%}f4SBsK%RYran7hCV%AQ{~eOOSwcL59~$ zCJkJsqap&_`>!Vt@RC?i8%TV)U&-$YSvYs@+d#Dkm5PV33C`>EPw-_;uakrZ8K$JR zynfXP-7rG@O{k?pkyFm))%Mc|f$0g=KUA*%k#xTsV&psUSYDfMDuiOG{?ZcL4^RKQ zsrK#{81WI@mZ#@~){iI|)fY$6jm^+#Fdy6TXR(Y|dRQ*8jM8oP?W$v8&gj6kE&#x=; zMY$U+5?()e3vX?(v2$;-nyY@Zw>p)?$|dg2#P< zg_4}4R1}=494(E#-FKCyTd}HS*x->d{cBCIjyhsZdc}TY?WU4}v4(K^ zN5Z9T-I3t@x=C}>WPq-=sc6d`hR&3ESPDc~RJqvZfLXWq)ve zI;kgMVlJxGvKoCuUlMEl&AjOOs)Tm=(ja}(yGykRNkyt+F59s=XYr(0I@ecM#-F#9XFR4bL6_Pq=v9m<sAnE0 zwPNHqcV$>Zll+=#5i4>vCM>cR#A>GOxm3~-h0z*mJkW14rr-GY45k11zRxt!-?yZk zkGa5kn!~r3jtR+r<5{D3@Oh2p)2@H2iCuRUQ}%R7_j_7gr)5S!Z~XXRmA7)}JojAY zHyrDWtFz9Q_`mK7+D%t>f+OQ;U7JPs_lasf->2en&KH+UwJR8k0-C6V;@u%z{>(X| zZy|6zk)KKHXyQ@kRkm%77gE*LvvQeelGrsw}6PDUu7Fim8GRbkV=?fXV=_Rt% z^Yzi2<@6Qf0phxvN&AN1o#Hzvk%EAKc&id(3{4J@b)cmxu!HRv1s@kM){D|5K3T_U z%t-$3xDt!)Ag{2rjoXk=UPAT?t`Z?z$cd~NJ^nS7mYQ<*9^!-W$aR^lh!?oLu(tc< zDpWZAqHOAJvYNyXo@j`SF_+&V;^*d%se1E{L@__vIQxPX&4k3~PM9YGWj1=G+>|AW zpiZODrQ+~wy?A6QU&;&#xq7qTqg6sKKwW6tVd6CYv$kh;gJ!8>1Tll}UY{^savmbN zeR#H{Tz?;q-J{a}RkwZJSy�y`?}`jb;|XA~UmrRNC6V-A~5<>$@iNXQG6-s<9o> z-;Dg+Cgdu~A-<2DdhS^fD+W#33{L1y&D`^-cjFK!@JjV?U%-|`ZJh%TE)E8s=H%j7YqL_Ghvk%1vr9>N>faxM!0@tV+$&E zsAS0)sB|%QgcQ=1-GPalzcx^31^7@S)9 zfFrZC?6R{PhtI1JyWESX>@J^sfvx%NJWakNLG?SF9&#Km+*cbm81@f>k{-v9=#kx+ zu#>1daHTz~Rrsj&4J2xczvJ>&DU@H6)#HTTrTWHe&IV&>8ho`{=g9R-96;n;TE-8R zMRjccr5a0gF}zpHU|E=~l9g1kVAf3hF9jy7{~GSQs~`t@4IDU{bz zN33F-Ea1Vuarmd!TlBMk6y= z)9E=v6-f&(1%mH!GNk{%`=NgmE7}po^16>;`M6!0{1eL#{l^OP6@}j-JPIeNJ&1|J z`Ikb=$AfpPkpF`c5&5Up$5^1q-hEYVHqBPtwIW@u0Lbc79qUp1K>afB^%FBZPhZq?E z^forl=->x7EzFcV<8-L-Vw{NF&+>mrC1w_F>j_pT5C=(r8NvSBc@F4Ce}Mx_NL7z! zpcvs(@H7~rl(!TuJ@^vv3?jjcH43OF3fv%3-})V(tpV~E=g0DpYHVv3NK~}|NPR-O zS%+6bi4`t7-l^+k52J;Jy3aMA>pH=$M(ZzGKT_uJ{=*8P9bMZl)>?8`HQyX&q$~(l z0v-2i?Ir}5cDwKqfb@~duW;X)rRN;|e~glO2b$(tIRo#{;ADHI6!7Le9er!$ zmygR-ce!u44V-#az^?mH<7=;$@&PJu=&n|gWlZ;xS|%kaAEa_E?Vwdl?VK{(9zMwy zuRdbb9R|E6qp!!lm;roEm4WLe>Eu_ti87UrjV$^lC!K^|Ui^D?Rn4wUa23<|GQ)sl%{1O*AESp`9vteg0OKTh$C5y#e@8H*|0k`ZzFERk?wC@|qb0SSlHG1au{f$pF`zsMkJ+jO+fw z!ei!VDqKX~pJgP1E^@R)<7C%6C|ZFd>bh`hL~wV=Qc51?$)j9T1|6~~M)=kPjk{j9 z0>oC{MJYC#6&=K?NY9HD3R$E-u%4mFA|@%75K&AbNa5 zmt?vJP4jYKHYkHCr zJjFBfZFSVN&lN21*@}@R@(H~VwDk&8lQZyO5o*dO{c#OD$XhEXJ~stBF^w)lT?x-c zS2{75%ehB5n9m6=>*s=|gdDt2j#j-ErOf`~sfk@lG6@x4-2M`fm4yvfIv5mP2|z2d z=E3jBkmCx_{WZ6K11~5c#s_SWoFxba8E@?%3v5O#`DT6*EsF*p++Yw|yg9Wo`VY+i zKwI6Y;Sz6J8(Y?Ie_1HGe0l$10W&H2!UqHma4nL%126m=0)UBJ52bU@fV>w@)IdEr zH3rksbkAAJ$5!A{VJ1x)UONFAVxv<$Hh}N+Da=BN=oeA*Ljnd>d8`#xT!W#bxiTP@ zLIBSpFU*;8cETE%GqC}~B7n#LNH7!qi(u7`C$!=2T$5**dQ%YU$iS4ku}AO zT6ql3xTy5pa_CDG;7gYE@d_YcA>_9ayg|q^4Mcch2{r`)Ftt5FNc$zFD*_!=QTgb$ z3RA>*6j(XHsQw%B;aI$^6dhc_YS0?A1H$oOGXc0_`XTfF$OM0{^G)G6HxKXwHuBzA zK&$xit_xmD-#Z*ZLKO|IFYxDYC*Ppa8Cq3tgNLRHFF|gLKKnyoIS^VGhJgEsp*|!0 z3p?@^FI;2=eZRICr5&?tofG-R(WryTyh~OE$kTINK5; zo-8QPFUY6hP35a41)z73$viA8=99KX%I)WpVxiLdI4*a<*3`F5R}Z!%wND=DxLNHj z9f8dK?HnY(gLfejjPZk3%rU`)bn5lr&G=#gDb@y7zZ)6$GTekci-FJRs#Vu z6X>9mZASovX)djKqZgYoCC{^0kuXdB0?}b~j2RNWL1#~GE<4eAOz$4I!I+7nX<*)N zV60z8PHdl9FWMe#(T};wEMu^s89|^W0s@57&5S$&LNXI4ti{Ox$jn*jDEBW&HnF?^ zdQn_U45|tlN*1HKEh@e9;Ie_Z4{+Bmnme7^Dt_igNoH4yHJ`Ia<~Kh-_BZ{?eDiY z?tARc|Jq%r`*@#b`H@Z~kmMTN@B}ZPoGqR^eSrqMVRH5aq`t@wRt4+8lB-K@fSVu~ zspMz*Y}SV9M%tjYQvmpcn62%zSw>qitOk`;-WdEPpCEzSC0m>V7frpa!dw@_z;n+R zco9beh);-?@4i{zrSwV1iO%U=V(Vq=eKCUTwJCZx7BEe9+8L0Pa#s-ToarKj?Bh!3>YZt?K%)*9>5*M z?{VuxGmTG%Z^X}_z0hMkhb9_G9KIca8J)f@$F*SnG6n5c%t#O0f3pBUjOG2ZPwIDJ zb@Ob}#lkxvZ^Z>1$DUcY7`BlW*pYA|HC7A!lVo6?oAcCx=r_nFuRmmk`A%|qK;XvI z`yGt|++o5$)Ea~tX3;jB@mlnz7tePPJ9szIe)l^kbY{1(u*cK7)q&JTx5E|HZ&8io z-b|>Thp|Qa({`0M@9Zi_6s8ex|%ch zLcBt0ztULs^St=TbdC0_i%;N;-yy8=Dtzyxuw2kFfd8@R?$mDH0f_rVgl#}FiC)8M z+DF5l?*spIf>UnD!7^9ZyfZkr7{gziaym2fFopQw$gV!j`El=*g9!EescC%;+D)H; zi?gi8fO!9V4YnJ!g+n9tZrZ!QO~eMGFcxW^ZebEEo(}|uBn$!~zqO*QbyNKlx;a&R5KLM#Us)nX!PU4I>?{f}S zsE{D;fL$0ex~B6;`_Izd)cx*!O4(46kSvdOZxEqSR@!0qOKXCpMjR7Ds-;f7U=}(D zvC)wyUtx-X{xOmHatZdGxcP{b`6-X89H*xCf{d@3D1XM=qz*Fb)GJ=*+2{=Hm2(T& zKJ2x(a@?uX-{FfP9kK4%;`YFFR6y|C|M zB)ZKSNekj6s;1W1depcq?jNxlb<=k{M-nJ9i+Z=vF=>CKqeQPQPZB(ti6w*8!G`?Z zduHrk_I7-*J!xq}equwt7S2@AL5F-a=j!ISZH9ya$h|0vHh z9}pJzrX!9<98U;lW5$Pv$rGsk0wKu`Q8SJ;aDr6R#*5#T!R^b8mU&0!hPiQk(`p7lz(i&F;xX5@KGV8ybTK+E4L^XDC5t6YWy zV;TBhEaf}PFYYEZo!qgM%c9`JjHgU@4sDSAFcOy6h#S35Z*n3&^P;lq-(7cVB__1*N;afG(nVuXzhb!~L|;sANXJ(>>tga42}7=Y=aRXD zPeceQxEv~JtT00ver9&ARCfFSkzU0~A z;&_!{!$e!IesR6aC$Zcs3uPHzEmY=ZdbJ3bkVj6N<-_lP@g%rv?M`dE;dtPU4o#R398Qy2&q%7{m! zw;^1=FbChQRCn0Zc9LTuFI-S6FqZzqJ7j`#GIS>u zU(F4Aju^Ib*Sl>@jxDEhBI#wBOI(bFU}vDWbhu;n*?lyvhB49me|49(ky)@A{nUYHA5|=C#Y}(5%h!QU56}~F;Lf(}TZv=>5!+pG% zZz8&<>U1BukcYSQNcY1r`QJgduS2xQWW(%I<-=_$5xC7iUR8WhPwrr%yr=ZqsVmaU_8mS;mGz(TCLFmLb!~K5@#h zaSo$2glwvy;M&q+;l>o_u<>vey!l=$bv|YtwdW|e&b6q0;ER>0)h=T4b_RvX>zFw_ z`|ji@#wfQXudJ;r1$sUDS>z*@rPWjiq9}{MB1BglLg%-Z_*Ai6e$%-l14JUIBPxS@ zzcUHXom`6PvhbrGw(+q-My9KszJ(?*qY?+`n9CyOhTQsh_vRI@=#!e21NPb0CqXl* zs-PVU!k{5>d&@Uo@gcEz6VykFKOxqOfp=CZhr-qH3*=ge~Y!M7v*Z ze>ya|LxIo=CgxhW|4ZvZiv`b#g>jXa^=M|U=jx@q6Xy5cjiol{`MWmv>X5?8fBx!K zU%^1|Z{rd3})<8&IuI(Fb&`Fo*)d(waU z!C1d)x7{1r8P1NPA-X5n{3zRX&)^D$GRw5S2WdR;f3_OECw)(#(V|4^VBvB+X{ell z!*qA-Ia$AgN|2SJ*}c>H`=_)JtukZTsNF;v2^Od+N^eGB?4evdr<#n0AcH(>WoS9ol&!$Rsh!74+*#r*6 z8K)kLi!6etLAP}GWQE8vE|`=b;y}&5AmctKG$Q<-k#801^EdbqgL8a^Wr91`Q}KeR zn8QrftA)h>L6?=n+({2|>g`JYz$NM-1p$9$Sqv^_vYODOP+;$An6KvP{CX`b9l{8T zqCs@e_==PT5!YezX*83GJ7rR1sVaw-fRS21;s%Rlpo!atjTiC%H%SFhtWO zN&M|3nY6R?uR|fDM;F62i;F!+TZ^^&HAC7Y=YkWBv}K-hf>t9q@Rg$@YevLN!f%eJ zuz4;QN0+RbSQXQH9=tn;Gp`N(WGhM#GmpaFZhb0o_+6u8O6_d%St~Q)>aYlSgopFB zUt0 zzj6{Lg^S8foFe5+5wP3VW>amONirPsdUDImzVL7zwFOa*CsGr0!ZtC1va)t!^ zBN7TEC!WSxeu!ZEQ-j~9CV{7A((kkDpfuPOOxh!`Ela}%n?=+(ecIiY>mTdq`2NC! zy`6onRJC`}ETEg6q0B;piH4;F6ME+v{vyYNfNj9;cHVIOH00{wppM7MX5XBbjx za4=&cEd;tgcgggAp~}~Krc_edPrXl=;PJWVaomQHiR>J>HnhxyJ9SglkyjL}?QMUd zoW+I`Eo+~@GG9LEV4AY@;Yp*pIAmGX&hHav7`h>u=wMN+)A(sU+I6t@yenpNs>@y| z)7_&lSzcPXCkoUPWlam-;f+JCf#R&Admk>J32W-nAaMu^stFe8O{IeBuqf`t;2+=x z;_g}wTU=yWrwlxNe_3&@T>fb>a=2x_i(ZtDl&QJl!+gZ+rHzkHtQ{r3tdWB zl^xY+;*N1}IOFA~5fCg$@EK&`HaAOOQnAQFNH;YDRDD{3)MWXW zaB|<13v8Mo7&S+(ck)!HX$KYd&5yQt>{Z3HiFcOqA2Gt(yq>aFEFwZpXn5e1uQrHO z@hu*G4b6UYFX8ZE*4=KKDEi+urt2Y!wpPry1byCOu7=`{D_z|4$i*$5IWPj#eZ>+C z|4q@kz9zlCo*>;-Uem3|sjFegp-O{d?B}bMSO`fcI8Iqd=W{YG$Fn>#k%h@t!q{Ls zW41i=|G+-}hgcrap17qRnkJAV1cd~>zeEhz<;;pXcTxR2)%YF}cnH8!p zQ;}SC+I!o@M|8Ic|K5DXYeGA=@{OmjoVA#x>#UR8`c(OOE{f&!I~SRQPR0|)G;;D@ z##8?;xnsrdT(BJxi!xmQ0JeX$MWs;hz?QDB;yDdkW1DcPtRu&hbz)0jI@<~>hIR2m ziP!yxicv#jwKX$UFIPVc|3^{;39~XciC* zWO@aDm*Nnin8bgj7C;xZVmTl0ZyeE9{+kI7o<%E5CNs%AAn*VG1VFGmLF`2rR`Gh9 z^PTrLEsiH7Y2lcpKpaHEPQ3{}I9jLz$cM9SZ3}JbKKgd+)r3(u}}vCEuh;Gq4g|HkU{4P>Ko}+ z(N4DcW6p>_x;hc4E~OLF?RvbK4?+dD8`H1dRqPFDh9;zQkP5dm$X`(!>XUstR?B zGE<&Gi`+Phweyu4EY*&@r*a)4bp;S&_Vq)WrVJGBV%?hZtLmp zEA}edh$26|8Vgv6ImpkDP|=;WNsDHdZfNNvoIR`Mw>n1FDfbGUE2l@itdr=LHEGQ! z22)v~2@^A_fjB3)j7KjoOvjB8#3sjSys=FMiwdUE8)kblYP=f@Uo+B?b9-%_?$)r8 z@_YN|yc+l%25zhq5<&-HGg_V8a!Vj{vmTzZCz9smNf5u=;_3g#+FOQ2wZ38B3J62V z&?zY;NGdrr5~74bNP~b#gTTUwHMj=@9mqcB<>#;%~IE=mTt$IA`Q?ps2gy|;gd<;ez0(ZPP)F0*>wY?5S zbm}~k5;=_fkz)7T)La$eqt~hPku2!bTwtZ$4<&lNrh@&F75XxsXA|ctua$uXQeq)r z{`HJkZJL|A1;*(lL8#&5?2uP=G4KO%J@9LM&12k9tl6rk_2C>>lBD=>zOLM(E~XgF z%#4?M2v3}qir+67jc&VS;I=vXv^tu+{ri^~r^zTLnJI`%ZT0%s;DEGI;*(6+lp-Cme?T#p(f2v}-{xyof$^$|s8Y}cu%}P76yp<(*mSr3 zAbA_lLXmWCesnPEH#;ONlyN2d?F?e8nS8yqi}P75z?$@i5YsR2y*;Oy@Yw(P_8c|k z@my8wncYr6UXmQ?AnG|+k}y}8Or}|nUSyI>GXrT$q&SbITnhtS@M3n@MXfFsO^T}# zuSgHj6hpdzGbdavQ2qIrSB90l<=0Q2KB+$hI~0e9sQDPlXrdF4O>~Y{WVI%}-({|- z7$oy)5fg61k9*D7t)q?qJ$!Gm4b4p-NERcK&3JM_N=C7+Pl=v(fII<=H~-Q&-;BMZ z$ZiLuSrb6;OC*hMqsbIcsQ5XoI}@;FBbgf45NCY4G*pm6{|z07WM&Wha+K_J@N!XRvT31F=ZY2W8-~2lNa(;;;H#=N*{yF-tYl~_!foq z*cN2w+n1xtinHqmI051E9x7eRa9Y-Ti-%hk;XQh)A++TQ1q6WG=a+h5J(Ng3--GLG zkiPmd!ztTvZfp@uB|FOSkzgxg4N${LX_l=O>@RcYZYZ1>aSclkQ;2#)=}Pv|BPj9Z zFFR64Jfg7%xit5gmqd5ol^I`Vma3C&zRCEAPZ}gYJDVBK6_QjyftTt;$2L1a1pf@R zVi5&i8+D-vmoYvm(`pOu_dmRqVNCaZs!_Q2g6IC2yKR|f)k-JE7UVs=`W6k^A?}=0 z-#Ld}p_%Tl8x07cSY(`f(`|oNQzcT|WEs9Z%2dAi z#mD@<-zmpD@%Y4Ice-_kQ7UO!a_`Mq6RTvQ?zEfTxb9O@E>qz{KPx5e7P{TP+oq(G z>UWM66-_^VJMA}EQMp5$B6Ub|eCB`2$LX6}TAE|oA;#dFL0ZBm^Ndt*a^9x}*1&z# z3IC)m!h@musISNRj-RY2--Z{8Kj>1+^L`?6usM;f`x zyV)Dc=_EFk)G;?3(tEMk{q_}z;HZ?`@YoEK`u_| zq}XU`Y`I{;JcwhfUcc2*0!z5suZrSleLXRV z?J)93M0dh&)Cj=j4p>rU&JN5IIk8}TWs?vO3UN=`3s(xq^pS_pqw^QYtlBO$g=bsB z9x$B>a~KAMsE?d_2o8U^&&i4OVYDtXodiAg_;xWq5*&uPRp9Xwu~&y@W$oj(sv-ID{aS8jv97J&5aWam^nd2>Nzk_;KR!Dn{}u5`pMI)(H|DE2 zo080D6yM|!{>L}v38hElSAKT9U}iYn-rXBzzq{6{E@{=Vsr`X>)wVK_Jjh(eQ-n<@ zWlTqO%|?xP*La70jX_$NYV_n}r)5u7>1Q;3ZSCdk=Z@qyRQ?I-Wd-_ny(EN+Fdw!E`Bc&9!>M4UzsVMI) zQ{)|i8hj94`oaq}iF~!i-fDORGW;gft_5BLv=AJ0R5i#*oJ`ySUG@0*TOZa`JrZsw z6<wz*Eq(aP4v)EMQ=hIk^g5=`wj*Bt&%zSar7Kfb9l zx0T5klvw3KBVZz{jbvqo-w30pVO4>*2B&#l zm-}1F*?q!9!XD}RD(XW-F9$!?k*bop)~iI=E^BjgN+xV$%8@i$AAR!h4(ATpuS~IT zhZIn=Oyn>>hbJwR-;n;9{E2UNceZmAjPpxuaGl1|OLnN9s|wkV9r$_yD}jMwRnZbEhR)=Iw!40FphEz}~sw&?Ts z=$?UHwbps1%IyqmKk%46MW3>r9%vZ5g%*?yY*mcaW#_K9vgfh~A12(CG7>sl>^T`< zInz)7Q0KcEWH=e6Z-p~*Ro`M&%FezuE)4CkJJj~o!|gLijNn57@nU=nUec0;NW01@!J}QxeEM}Ze){B-QHlEhbjZJD zV9&J2qR%AM@ChB&@J4Q$-spk@l2<_9ohFdk__l8$b9U^-z-pUXT~cOi<5pxs4Z!Y7 zNM;Z?(6B~|gNbu$h=6yT^Px!9<2@tAFU1`X1#i=R|HGNW8T$ z$}0Hgj;%B>4CPl-FTo4urU#*@vWwVaM(m2DMhG07g&bQN#{6F2vqLMi9Kn}&-#BZ2 zMtLOBVA~IR8^dT>42E^z{#roQ#x`%}>(k9| z(oD^lHvktC+UQW((3-@-8UZf}lhZ|-_guy0SL9rDXhkxiUlMj|_^~~UN_;6Do=Zen zq@ZZ>BxkEJbS*wKMb5#S^T2Zg#GR}wJ*R&M#9oQ4N`dTN^y+w$%hA~@4a3IOro=>g z3J0XR;0jX(P)2!XT^$CcetwUN-alD<%AcXE``#ORCLRt_J9s?Vaj%UVGWPTN;$d2Z z5!hHd9{ZHPEOMHLPwtf>zE!uOR&i5$ucGUub}~&RS^9{R^Z8TWmo@cy`OD{YN5-d# zXI~JXl(Y#@sIKxR^8Hh6P76f1=;(X2+n3~dzwgN9F-buX8M1oMRolf(aHqbTU?tw| zat4Wf|9E6c)5Q3ra-HI_U21P*jkEQ-4ZMu?eRoD@7aDwi{%J=-ohf{On%!mRZ#^`U zh?K`?HR8!0Ax96(cw?`fy>C=B6}xcQaDo5wRA8{6{9~AWjrhsdrBIzLgc7Y@roTWl zPxB0_Ch;XLYB*cuJUD~LScYc)X3OWe&dpb|dU83zA^W&}uZp;c#yd&0k(yh3N>O$v zC2(8~csh=l8PQzV8m9Gxjp@Z>lP#>!(;&vTRG0IRWdc;J2k>$Y4I|h>ukgbDE#rC6 zzhS?DYZgT36Ydc+wOQFez2rWTU!PpelGV70X6=0R#US$Z(Bqc3<9yRZi;j^;%NlDw zbz$k%~V~cK+_P#~=bcwe9thqt8)QU?1hL9OCg14?mc^Ok2!OJa?Jqh0?UD zQsj?qL3_tdp?PlQ$X>qLL(F%!J8`XOP-1lJC2k(UFxQ*C6x|+q9lC^AT9W;SH;m|)pK{f0KIbiVY^NF1eo4vL$oB=kPg1((LNrF>@n5em z8XdN+nBmNZQ5Oj>ekpm9JxWZ-02@_I7QHEyeuH;g z9JNll^Xw|IhHpLH6}N)Z&6)3}ZX)b#Lg!yF;XT+Z!KRaU-8aAGM!J|{N_6dxk@6kZMPR{s#Hc$nYSHrp7+Pm~ zx)EZyw2D}m*%>UhMad)UOQzDQ3&aDPQ>cb!F9zqI!mZ}vutf5)c~XTzdcG4=ivoQ* z^F-6l7?)Y6cO-(D1B+#HPplCLYD8++w7KVnT{$i-YWXDIdMI_6yuM7Wuwo1SUMktV zgY|XZ>>rCYe#mpfqNM3yI@VJbEqE`Eb3k4P#1jyV%nqt~MYhW8}X%7SEjY-J6p zyo>TqI^9=3%W!n@Oi*=)qQVF}f@kUh-}A~UyCSnk=j&Vzv@timON_2{HxnUcqtMZ_ zSsu()J=Hr^2Tg3P&lztner0J-f-&Bt`XmsFuc0Q5dbhL7%Cx>yRJBM4rQnv%1=V0cveHB%*!-CsYI(R)w&_gLI zwDj|ZwxU8yY1)9{FLrb!jXm)(w}^dRc^DOHs;7=yUw9%0U}R(dALw~ARlX`MyTBey zClt56{W<<>`@V9amUe)786ehAE>49G$B)A?F=r9fG28^cFD|WppSOc~9mA13n$4GI ztsJ_d)aK*BCyVEy_k6{n&+I2~o<`e_ui_q-C8C-#IEUTyHQH>XcU$7R-AP0z;$M59 zqz%@KzUQnmOc{LI5v}dqJh-o8d|tWHTD_irjPwZU zDlqTEaXH)!DWMaK>s{l_k=*(;RWWvcW%*26++lujXFYSN?oQgbQ(>PM`U{;O7o#-% zy6jqyMVxw5wBHWB(ynV^88QiFn{Zo{nQM3^5`&=nDeTf%q%@My)|X$SH-6{9H9=ph z%Z!wFje$=tHu0F@Bt_EYsijs|c&k|)#~SqnFKeXox2l=b%8UHEP1t;{S!>Dj%DBp{ zb*Em<%=+b{waIDcg0TBT)9bSI`_@W7EqOCm1rgdC5izpCh*umR#=5?krqdX2BGFW> zn-Vi}CiO-%^IDWdtlmz7ZF!ol7ox3NLpQd?Nf-qbnJ`h;Ix7d-^r?xDNg3zVrb3ZT z+oaLCzmUfBW3(vZDNP)UTBSU)zS*YLGqIzKShFCSc_+opc%XycOIBPoA=h3ta{h(v zqo8Yqwn+o32UA`NDj5-YZNf1mS(G>Ze8}!`+=)IgGrC5Y_KIjj)JI~h2w#DXU@L#_ z?ybl1@4ksM`Gk2y!w_{e5o-o&I~kR=EoV2t8~q@$?1bPOf5Er8p0#m-jExb4-InpT zXCy}o466pi;=?>1iVkJoRW3f9+q#sDK8*+|Cg;KQ z*&I5C*}Qwv_@bbqhXX5u~ z`MkR(t;Qzn>cQYJ1@*-sIwQB5&l9tx(dLCi+3=vj_*>dRC*Ehs-4jIT3yy|4%5z&a z`7yE}%1JDV={ecw#zvz8CZfu|5_ zaV6PIRS=Iwy1Cflcj;RZb z4lI9@m-1?Ho{BsjRLL$rs;K+B^${(5m+?x2g1^PzU6mh+vUhzg>d%`W?zUw9O=92W z(EOy4Aw6m=mp8@r!0XSP3H1Fzcx{JNL_VM^{YVV>`)B+}tq(9JD_Vp<_zRJcw~lj_ z;FA}L=IMWPfQrltve}xG{{!9s^WSVupa0MA)8%*nVqL-gi7p1m{2=}_FD(nc>yJO2 z4e$p)xgRk7H?RtR(GJ%<(EWe^L-J##ivgZ#94LJb7u3(yoQ(&Ay#eZ@ZU*%h5w*EwGjY>dJ4a; zT$2Q7PTQ#(kD(%CnPY&-iEQu+XHQV!GbH_$`M`CpZxU7xMs-^#H-d!cngKLE)M#>$5ynDeK2@v)hfPv2~C*QQu4)14S z)-iBuAsf$TN-%UAeITg!N&C72O*1WHCm?T+BBO+(^-Y5*ORM3*fHmQ3dsJ4l$;c?aFSi>Xf|#6~SYA6zrVywZLZ$b3 zc>+E(UDw;zswXHhHFNYpZnE{09mp$ z`gY|%4Bl+bw?Qe&meYRc-;Pgu6-dOj0#pD(A^9%gbQ}5XK3pBhjTr~#X|yDYH2Vdd zH_NCe6uG5za0Fs&1#kxe?*$t2vxn$PY>*;TP(Ji%T{s<-R*iV`AVJW5-4;T- zdS%2@8|6bFUHk#nIF3;e@w~RJavTa%)ZAI091qu2M1kneWT~G3p~}r(%mt$QC)dE$ z|0U7L^ShDXscWJyh${XJb*UiaMj)H*3;|Eao%XX&MYYZrIr*?GkVH&}J^w)rfcBQA=sxe%a^Z6 zx;-0U(l&H`IP9=gmjkxuc2F`=rSk-EB115bYH$ZkSlLI5B`jYA_WmsP*TH9w0Qgfy zfAvVn)N_p5UuC)@chWgP6YHM`Aq>`B-5G#Gc${!(H3FCtiAj{$t6&AIH6iYku8HLE zuos_uBt8WVL#g*Jrf6vV;_U3$f%8lrMU?&qMDUM~pbUg?otm+Ko9gALSo*RQ@aS(m ziA_PbZFOmw(-X?U@gOV_c_x5$mOupCD>05H&y{byTL?ao04<`l_Lcs35#+EOlm#pg zEV}`WW@yj@Fg3Xo?4bj!4uZs;M$rg1hCfoYsSV+L zFc(-=^#)3D1_8Hrh28D&tR#o(*B>t2{@>l9xrR?+Kyjogz{>J{^To_JD3m%)2mj*G zXDbb~HH6+R4K2A4xCTsl)!YSJM*7`Aup=Lh(^F7;*fnQWKo4%W;72*Z+ffE0B}- zY0#jaE}2y=(JU(i!W^5?@~EuR<11*6hXG6iY)C9HI$LA-pXC1k8$5G+3-U%uz= z$^s(QdSQD?tFnm{#hL!}J*DFSG@b}tbfNoq5+CM-STqc%9kWS);5-e}8GFyWw&=0) z@E=;yHjC!wW{Gif$4s~mXU`@D>$S~Nq(YMEY(RuS*>a1Dm=&?VFaZ9!r%rEB(+f}& z`FnzZ-mj-8Y}c`+xW#rr|=7Nqe#n@{Znv0Ol7x7uls2$CLME zPIt)pd*Jr4r~Fd`BT-%XmaUDLW4E2P$jY3%$>=a;>bEh)OsJ0=qV~$sep&kx1qKsb za59d;PGX`d4MgCMK5st%La^(Ig9}B|+T~7pzsD?8jlyN675d2hsef_cd{~#LgNZi7 zZZ@lDbft>asAwtNfMY$@uw7V_TtuHDq?py9>-H0bJui`o96>9aHjLESx6zkqo9K+j z^!_40D-EeiTf&Yx`R%D%tVX&st_i~yuJB!@ocu-{1P{hF{w1KSObh* z8jQNKBiQYg#m`yZ!)3ylsfd}yd@5;TNEn52cOZfizFg3?)(hw0^KTsVcx}Lj$C;%Q z>@qy(9-7V8Lq_2sY#uGr2TmbqLfovx*k?f%TaQ=+;FpXTplRb2Vgqna?L|>!q#f7H zR~kQRYxm#Vy9xoQCd_bGBKvT=Y$Aj^l#iV*D+*)A+EdJR!^lIV#Tkp3BiNCA?Adg) ztq25fRGM1sc8m*K>zq6q=A~`G79F=J^Zs$#BR)ng#c8S8Aaa9mAB;01hf^@c@SrtB z0^VEWlKwF-UT$jCy&z#DLn;Zu&>IDI@OY#y>CnWDUh0$V?9wv*M?U2m|2$o?ah^VI zG)&&It|i>M@i2Z?Rlzo~1*S!&&95e&%pakxkW@xA`JuRh%Qr@CNF| zN$LYW`^QNG%06>P91O`W$%93v7eC(+q$%LUN?P?tyls1(JDGjB`tM+S@xh>S(Y5CL zG_1-h&EfbN91>Mi4#p3P!oFRVYZrfoyIbj;yFi)ftM!A0JEN39emk68O!e@1&HBWf z&S)UWlCD}H=7n#@13pE)32$9vX|eYucg$xoVYRI9<8*V=c;3WF|ND@U#(BCJr(*rz zo*Ddt$^olf_pqaeGa1H9fQ6Vz2^JUnw?gv&cq>!M(&DE#Os$MeuQPfzzSX`;Kz$yfL;P1^bhZBN(1|=bNKx}{HAvFj`141yI3Mx+BioAK zdl{MR<`q8W{sSLjg)3)T0)qU6!+7i3pvb>#0rivrid-fK30<@_%@@xHhUNbO!9aPm zi9>L<`IHft_wVwNOXi}l)2H6|p*qxWinssb^Insc%`PUm@#+QFpC22{!K|hGq_M;J zpDPBOnvWMQhBsd8Rk@b^Cq{tc&2vT5EPNinKkLfFf@(rpwhv){RZdT^$by48@=yL8 zawY*hmMb4CDu?yYIh4)9SD~Q#zge8-XHcP5fISrD5rXjb`L$2^W@@}8ito@RnwP>D zx==p`0nxs+e3MBBYz@H%w3}))9O|X$gMrJ=wCP4h<8^YKKx~`AJgvcU1K-01yJU+5 z?iHbrj?kht(_n*RKKqkvU}&0aMZ$e$@O!%F3imO1`gR!h+h=SzIb8Oq)A3E*}(99r&ud_D>MWY(SJ}48%2Q*~tpw#5!)j<%x z&*+h3S2-IZV$nh21)jXWz_coAbp%k$#z42i9z3AS!I1MP*jZXY81}Yp2f$A&gR&Xm z-$1kdGoZ}30?x?#oaOV`%DU+Ze z)2jyPPQ$NDb%0jW#G#(LB+_I9h_JsVvbnW=IxT;?NV%&7&d%Zco^JC+- zqS~VlE?pK|Eg~_eq2`FSarLrhdv7gMvo{@CYR5 zBZX5(JRS&$uZ8gJT@D&F(?_Ay&Kw3n9z zKrjR`aYD@2sUgg~=5c=W(`ZJiuW&S%dpb{kvHS;f+FzZle4Ve_?x|g0sL^<* z{$}*K6>gE*E7BtZdTws1V$dR6Aj=ZMmB*1AzLb*+lSG&bK+;3$=mfY()CarLB!}^N5H`pe0fZ@% zVvFbujNbR8geDT{gB|&mL`Er3G1F+3M)O*5-F`i_ZksDm^p*m}(H^*)Mge^9SF{Cv z2>N9TZ}T-Y$F!h%>j)@3(0gpR59j4i!0uxW6amE`0;AFgX$a=BeNzNX1IQbZ{HE&X zScT}xIqs*-3K^f>+shKJM6T_DhfLvlkyYx4<5K>L55EBcb1=_%pM@m zV-1ww&Mm(Sx=VnVXJ{ssq+ z0>HAXV{@36xbUVZc@YV4W^92eD)}U*W#M#PNq{b%%#5O50`;bgpL9k~nq;*kL8BiO zjd2CsV?&_O_#L|f@FI>_c%%W;;YZ9k#Sno;UHb4bX96h}!vIL{sGbQRobYW-1PXQy zVU;s)`tMQUwRt9Yp}{MpN+WTb93juI{t8zyWpbAYqmVXsYN6!o?iQW?r5HitGhkU` zY#w#fK+0N@>2izt9zc}5+1F)ZXy0fi*LY^CBc!-9JNYze%Y@Yz{uns|mxC2Z`y5qb z8mZ;T>8#}m4Y#b1wI5ltbB@SwU`9&J(-Nak`lH{af{A!nqLZpE0y6bhc(4|Vo7EQB zzqY=AE`mv)c~3`AogiN_xTo2JYhFG6$o%`D#wEF(_q209P+kp)dU?n0a@>E!;%XtY zc3YBc@x2degYKB?T_#x^w)Ed3X>ldo#J~g4r3@*;B(nJBF#(&vp?NK?iKpwW=7!BJ zygKfA6REO+kC|84UFtQTZ2Gh{NJ|iGEWf|A0(U%6>@!k=X#?c+@eYlXAh9}Tt zJ}313#N3ymvD-@nW^vC~VgxG6YIM8mvhP@>xh7OM$@a2b0AO~_Mm1%wcyGR%O?gfu zxb0NlF%ZcYS9%O;AQj?A%n_>qX(6%>j)uuCZVP6Rzn{3Mr6l7Qg;<_(MAJ0yfyY_~ zW$;<5by1iNjv!{5i^M`HNqFNW{JK3-@td(Njt#;~OX2FQMN|rZxCyBCeI~pWqnqA_ z(Y#%Q*M?oKdF8Z5E|pN_EA743`CRwi6@k5ZIijf3EKSL!%)kk&=&TJEq4~&XVUJ}< z`g5~tfJ^cVq1>_EVGY?E%Pdk07Q`Cb`2CfXL6Q%g!z5GM&0L~*K^Xj57h%~tLyBS7 zN26uGx2P{(%DxidwUmRky+LNZXsA8eRuc^q5M(OCp1U9jzllRDh&-nb$v@iP*k~q@ zr)~&szb%BTN5$Lgh!ln6Bh`zu6hb9Pa4{Q>8$$JL_i7pM;wbwN(Y0S3-jD`#R0n0*2_ZV#|OwDL<4#sZI`C?QyG#sa+3~ zYQ7Q>Zimb}@$fn8(^153Gl{2-;c7$MFszPeY%}YUKA}lhI4}geiJp}9i7N1k{q2D+ zlWF(`WrPwsZ}#*j67lpbnz$rDInE;((sIcyk@w%Gw^y2z%JjJeiR%2?w_BnumLan2 z7r6Xy7C?dUeyne(6`X`kUgieQAq0Xi#Fj^Ez18R?NjBD{Qs^Qpe18`9nCYoTC))?6 zIcpA_JJW%9jR~t*Fy&G{Boay0_ITeCZ;}_SL&xg<452^Cv+wRg537v1F!KCAt)T@7Yb9SZ0E=VtA>{+r=Im zM);FxXjss($TLPYO-SPU)hMW<)QRxQ<>P4`E@wSjyyY=_>)=LNuUb5h6xK4(y`4;@ z5i(sE^D~pw;6RylO4e@Xbc_lX*g@S0A-U#Ea|CYZ(bMfCj^ZPXhCRY3J&$Bo5@ z9&*i@m0*)(c}e|NV#Gyi(z#%!Kvpc~@6PWyyOe!{*h!O-@!Nt)nz%uXmYiesSfAx-6FJZJhUAR5_Eu>AAMzu+zbc_Y2V3eaP;c;QbaI{Awt8r z8f&!PGEONdx4#j0FoKsn53ySJFm2yL!npAosY|*L%8{Pi=qP$(s+!`Jp$)=)7tAcN zlcMjLc(8R_X;^q$Wv_;yBK%XrDJ#i)K7sCr(4Sj-B3;NY-?*9ySDISNuDfo=TzxK@ zE>iQx$zstPrqax{do2RU_u7R;jzq(>yr&*<+p4RfVYlrKlB(5w9cSJL2uMY~SpKU1 zF{1*iFJ*7RWr?V&nS&Ir|`Bnp`3h;uS# z?^=GnDXW379M_7n=nN3J@6}Y$o!F>pyBZ0kj^rs zEM$(r1Qw|q!+srbJ<wi7D6SYiM zMN?C(!KaM@;NOqe@M+yid}eKmKYIzX*Mf z2yPjmy7pJB_X9d0Lflp;BL6SXG)#62C<$QWs;LT{?*Dyd_QSwgqw`GD{r44F9PsTF zO}EN|Z~rgj{P&7pD}g2U_pUbow?I}+1UQn2-0Khi!!Cm*-T?Z=>kuQ(zfik#EI=Va z*k!%??@Wftz9R*i#*H0`lD`xZ60qPoYr@xmjU`@-0xNl!H5~qz9D)Q3rqa;Q`@7$4 z4OYTK{7wDe{vudpi$bP$Rz&HYwReO*1PlCsPHeS(H~hsx zhF;n%h&%|^q_kRSQ@!>zVy)!Zes7b_&<>=RJyrX&8aO~RX}&f3=9%2m7Od}MkZycK z4sL{$!}Hqzw*Lz>95wQ%K=1Ieo%__NeN}lxkSQzs+JBdG-U<%EZUhIUp3vQVDu%cv znj0G{$k=EQ5Zlj2qpIu~9rq`-9uSseWc#?e7vb9iYhom`KKoPmKPKd`NwqE5Kytwq z&!sH4K`rIcF`FX8xnij0e_7_l5kl43F26fJo?HYuJUC8_y`nhX1A&}cI@~{HH$KsEXSANTWEw?k56j>r}Cn?ZapM(7x;cyOu>c<|4 zJmdYC-;s*SlnX4q&%V7ZV{P5P$4>bx$vnCQ#Fz14+^z1t%AT(6xc2F>0{a5@FCZQz zp7L>_wt^8H!dKd#u>F2=#fQrFh0ANxe$KnvGlvNNecH`k&^x~*r zG}Ifs;t57!M}Tf)0hU_1=4^1#dL7Vr>CyZpDZDxgpgP$Iz(FrihXqq@#_M z+bE0X`YtA2H8*)dmwj`MCIjsG6GfYy1~(%^rg|OEg*ti3eR;Rit6)Euqr4kW^k6$c zt~#0=9o|g()fyzKYVf@Y6|5pQc=x)AKWMuGFNQ$S{lQH-<*2g5G6#d|TNnZvXAM#b>y*;s#0n+f7KB?)b@55UU#P)7&MOOR)2RGI}|gQCLWxX2wQx(6!YP`RXv-A zd^b>;J1pxO33x`p(A%8!eAEg!QnzWa06U@pwlm5P#B?``JNc@+AV*2lF=aW}kf7iR zVetUqJPu|fXXJf=K=kgIyWydZ;ScGvKuBUrQ_UM*QD)UXdS+Hl7~iUUUb!w)8USurPJ{wK5mB(ONSYaEd#;kGh{V~V&81~bHZ2k5gg0WlIvkqO^_!D zQc&%{q^gKj4}*IbI9{vYt{+Z@=bP(b!VnSJJHf7sId5N=%zVUW+L| zqD@-4pthZR3aJBZMZHTTPCk5hTmbP|!gLqPQFCV11dG3?$EOHL)`wj3D zZyg7^keLHeHCmgdhwUUlEgxwUQT&X|eWVYi64+pMx=Sw^V=-2xQVrhz5y%WtNZmm* zbx;Nvmay-bBpwd^`Ah6C>OGdc7*?wcpY`1;JZ&GKpO`Cjknh?_k=!tt(Rx=<^)jdK z<^GpxUz^`CdeZWKv#s$*PK6LHwVZz#X~3lMHRQnlIjD(_(2&GbE9$Mc`gjA*d)t-37yaT`H z)LH_j0?P)O_*boS(-s%?SzF;AS__Y;nP^2SR|jE&w-=ob`vo6dkRXV9iRmid}hhz=-t;PXk-7RYA#ppCNJfTGe z~m z>_m3*Gc$&!Q>=k}*f?fAi4ax2XQ25)O!3wHj?9SR*oy=&+nKJ(UgXmdj!K8A}E1gGwYmeOHFl7%j? zjgdCoeFEf|8)k%ZVLss5&yGkl9ov+u`5+ts2g)oeRhg6fBrH=_cgFf^YN)8t(|FSi zpFO83O)49tNEn_B9;K5aQ`4({Wz25IKu0g|zzlOf;S8e57AP|ScP)KwXdEi#tUXF2 zMZ~UD6?oS$BivM{$NTQ5@${GNrI#dkrc0es>0B6&WrH36m&uNz0q`cmbqBGk**Q9p zGVyIyIjlZ5sz{&gOW7*D1Ggwa*u%ofgzNeVm2!=pFJ=XUt{AEQ0w)c{XQH+`A2CPg z%sD6n?Ui8QPRorozBb!1m|=-K=Uw2ss=ep7;l6!$9oJD_a^f!e-nNno4Jn#Bc9T(V zW(`zU&a=Xew#fa>p?9=_SNyz}L#aZ((q?W_usnrFO?9@-bejYn!udB8iU>X8(w~Ki z<35&ukmGl>oU3Zo2+x?>j2GZyaEFbaFb8E|BsGods)_1Du4fh4e1W10KZ?Ma9rzV;~jq8mN^ZNntb}YIS z3I!3Fkf~psL84?npk8wHWT(#15m@oJzHRI8DWgu;I+D(M%=?CE9I;=v$7r zpuMxEIu>EAPltyi%Vl$QD&rR*U;O@uU-Q~banU;ri7}+>&4jJq^tF)+O@aPjaLol8 zQUcw%Z#1?M;Zhqo%2kRVr}NwJA0cBnpI3R(1mxURstlGVXOSD-5+Ze5q(KnDaB40x z#5nF?33_GE5C`I65o}PIqJdjadgOQ-01U%SiE!D387X=~_euHjjR<6v~k6SP*0`7fCUG}B7!^6yU8ke z$V75AZYVjfMNqo=Hcf=MxoXIgDXUB%eV}orE(Zd)=?g7WJ4nG$$oGoLOK|WV$)>N=B3- zjB}eL>g0)E9T$tv(WdTH)Aa(P3EN)IB|lzy`F6#Ee4ps~<<1>)vDpPnt;@qo(yce! zx`{*$r_>T!(`xUYFL-7fT6cOA@w2$#EMtdBFJ`f>FJl{wBL|likWWb04V0y8Qc1ug zC*@53e1;>+CO$(|r< zgk}Qu#lyy(WiDzXS|(OANwcWxgS_^gHbrAnl6bZq1H}PF8Q~@J%h7~3!p5CN7dR-C zB>1DfrIGZbt(p4CA)@?5amS_2PTjc$PY98$aQGSM%6blwchqathP}xvA#pG@pf66_ z+a;aV_7m4m99CV+>AfZ>+n4Wn(e>tbm>zBkm+fX>+K5T|P8y8rAs zuZ>uOoQQg4&TAeC9bRs`d$qF@pK{E3%)C;Wf7Un|V5~pgc$vb{KbqKlO=2b_fHhoe zCn@Kduh;JbiSm~*_Bdi_ODIqErMrZYJDpGWrbXVJXZ%*!INv{&`y(SI@2Z=1TQEAp zXc{Ru{K)3I=U^myLM+aDjacWc3@fB?rb>8 zWfIk2KYz$OIz|(0->Emdsol@tb(hSA`ml+d4js%l;FBL2Ln*$gQPl}-p}g7{I&Za-rY;&tmf~gO~n)Cr!41;kX?9NE1?WU639Rw$os2 z9Skmj$nU4X(DDUp+=Fa_$G{&_4y^6GbHdmDWuLU)%~S$-gGH;ZmEocc$?|(ZYKu>) z2krz5C|ua?>+<_Okd8YF^a5Lu&l?u!JRaT@_U_Hly3;k)kIx0{8cdjiNP`gh6VMVzFd$wuv&2nYDW>1`U{ZP;a3f& z=X2j%0AI z`0Sw%8tMceAWk-$MnmvdAq&sfQ>Yqn6Q0TeafpHwR|O@zE6BI54=Im z>QcLdLesYK`zLDr-^wJa1L;TU4ow#0Y&5NUyY6SB2;zq!e? zl@9?5CV{8h4Q$TzTO%@()FuCs?o71ExiYZ06u6;5`(rTIF=-5>FuPz%?RH;F#@Ai8 zpf&Ot5~Kk9#Ts~U47slYbt!)w$`P{Ip9GmAJs8C$KP?*5PB0TN@CXnc3V$US`+AH+ zk>Oys=I%28bhywCw48s-Fdhp!VEV&rG1)3Lk+%q&HQdq1S}E)tl$JiSe)pQlR3qiq_$OcVggy#@H|$6wwVK~|do)81PLW!1IeqllmnB@faesfcuUNeU{VbR!}q4bmV; zH!6|>58d6}NJvOYcZhU1zqO6mZ|3}S{yAsn%$#rD`DWhzvY);8+H0?M-`9Oz*Uj!H ztG?Y>HIM08F(1f4J0c)>Lw~Hs!Ls59NU=*GX3!Mu9OP_cFP|t=zD9Sj^lyfm$lSmt zU*|E@os&CRr65yP0f^j8{A{p5XtlM)49_aA_cHMDkAF%oBpjlAm^6ze7RPGLM2>K3<61Rb1GgOl)cqeDo9$>n)_eyh3Yf_83==x9Kaf!LUg@mB!8^NUQS| zf8PWi;8?j`>AT$BRt`>|w5IAg0kcl?>hfX(Q0Lgis)?EjEg;*Ivz~^-jfoobq3a2d z6F-EX9F3{!x-Cu9ys)gCy8)$RT6OAVO@U*sSU7%2hhmA<_FL0aXsM9QfGIO6&jjNX zNlKmWCosE;x#&4DF&a04a@UFO-Cr~oCuf#qBL|>>s^m^%bmZc@pW`tneM}k==F@S% zNR|OR=ustcUnthInu502{USx`tynZjuq1 z?E>(&_4kjN*_MYR+`&3fTC-I@;gjqD$U{6w+kn7S0g>9m?lC_85_zHp>wrW!Fnt3E zV%y^fU695Pr(G=X?-$>cnx?!3h=&U*q0;~m-6UHKSbBa-@R!>i71I0GYffy^;ZCNG zN^HxWcdQz(S>(s-5-+~FdoIpN>_e3?{Ae4`m%CPgR{{kdNF|P%{UPfzv=w?L;9E3X(-O27)o zuLYO$r>!+}PoSn!LkZwnwq%QfXnI9pksC69RoDT@ITVDn60L>vdtfN-CKfIgM?zyF zQyfd+souOe9BRT#jSGOf6~#9Y#a$3$IOj+WCX`+jPcoY28bd+tyzt8D5fHxdD*C8m z^_c)XW%srMU?A922cP2@41EZv-?(n8(mvnWY5+(&ZZf{cP@|&HZ>3%iluxT!c~{_D z(l9vhP`;<=`>~fG)+;$?@>oL9u*KFdX_3#}Q7FZB+x61WtqpV!+{vcq++l1<g`4Jk#xm~+WGw27wk69Zh!V+&sx~suuZ2sI+qfTJ8-Zc>_`SYf%^#-Fzw%% zyVf$%F5xDUfr)*`q`8pgthtQ{P(Jc!GZj-No#2E-31Si$bI+E8cZi#A8!;w`7CMo# zz3|q=`8rwY+T}_2Xb*ns>SRbxJ}oTP;3$tK#4*ZBA>QwlBIn4{T`V?2(Vb&SAz=q> z)JkgL*x?v|}*F z+$-DtLG! zjKKE#0)b~?;oIRdcLb6bmbDiTL!wPMvo2`+M~%1B>a+m%Ht*_mSK73vjs7KbtEnMJ z5(u|HZk1K`#Qpf|cvuTAOH>AfHMjdjlpmIzsqH57^L$qAedidhc`H z-C8sN^M&wio7h`wxfAi=yI8yNTY(JuRzG4Ee3N#mrhJ&BOO0V4ASR_}p!ECjjb;*r zLUXG3h&oo!xzw`0udgaT4c4?u=);ecU%(f=Qu_uLsnz#s-)a7ifiTsyEWs!xG8!?> z6ZJ_icKw;!=*L<*B_81(4xlglPzyT$N*WHf&NHyV-z)m6nus4-g|t!^q`sl8!Q z)TarqyPeMtDEMqrHSHQpR@LyD(1#)FPF}4WB&MK|F8}@_s8&c8v)bEq$9qS!)W80& zC0+uHSYvMdQjDj!o%*5|4&d0^e9Sg}b=`JyyOQmur6O^T$xh*X+5+NRJPl`a6)fsH zwjk@-F+r_s&lXLo+-DpvSdD9lpY_54R7;pz@G{N-B=FHG%-OpA`M`5{OX(TF<5yG^ zO7vDiIj0;D4f(cMZKa(uZS!yP(~!l)tz9+uJJc`rMly#g=NoL_o;QCLX-nsu(oyt2 zK8_}xYc?E8lo13Gl?M8~dyY^m>{n6fv_?Yz%xFQUXz^zaZ zWor?16p_8B4W6ef_bsW7P)qntriFW$9`@IzdaimLEDSVuc8KPg46J41=OQH?efzjl zJqlN;@gl4{RQAhxRZI+%$S0BJ5F?fp84=%ve}%%Hfz*vkOcZVO=!$BJ_$+-+Vv1@B z3--Y<3AEW5+ElsdXi^dsb`0HUgMbzO&2*YrzrVVP#$-x(V6&{IaWNfzvYao4?_Os zHH^z3Jt5q?p>puA98F09PCZe0M)L0hYCYf{nU}G0*ZwD;6b7g6AyPU2U7#=;H=ItZ zhg5C&pT@!y5QXrT#by3SQKH9opHi_hnx`=P{=c;G1fCfwPn`zku;3IV)L>|RgFk`tn55|2t zKyM*K*@jcYq{GO~GZLJ$p<3J_z^BGQr?E+jt+96DZ{eQ|UP9xhFo5z2q)Zb~V=y~{ z^eRUv>w+RtQ0+2+l;EHjmX3RrdELQ?1*E`FoPr9}mF6I}G=4KNzb%z)Wo;}u{1Amc z0Ld2S_N5X8`f`}G_^8&8NP_={3;2>rl0gI`fHePevZ^Ex5Cz5tZ}Yo>;A;}=C+PKP z4mRvE5EYC8t;F&-#)k79#F5#_>pwlALP_&18 z%2GEb-wqCEOJ<3R@GXrN;T~lh|HZ!u@@hn9$kl^H_){SMx>pQI$00FDfTV;Dh=pE{ z_0Grw|!wtYMC=I~9KMsh)MxY|i6rrkorSQWR9Z06lrOB)%R6JGfljs{@O47(Z zR6V1Qy3d+A%7f zFy4ngHdz{Q0XH;8Am11%)Nk>+rnfV(|6|Y`P%_4BpTu%@LSoW&qiHJ+NC`0x;_z72 zm#M8y%8TM@LCy5dGHw6i18SIP@Y{gx^(G*8?(<6@8)Nr08!A*&$NDZu-QxC01uw98 zj)8Pj8O>jd?r?&L3-nqS zph=x3aD&|dTMk=#JyWO(7{0T^4YTu3Fh{e=uU$2oCWuG(iQ$?)(tva^6_*!hHAknv zKEFswJXws;UgO(mlGR|QzfJ*dv2#9Pi=1z>KmGh5QXR)4(@tI= zhg~rO{O<1EM)^_wo-+Ld9cCIV6E=_6`39xGv{C3XvtZhC5JYKNHKA_0V9Ha5<^BLP zgRDXJqis;AB?WaIuxPDWU&fP)AAp&9wyNjD>k@k zNs`F6;d#EG4d3dURP+PV8Celq)t->7;s8-DTA(AkPAcMF`!$ZIU|a#_ai*4BIL=u^ zPmk<#Z+3++{WB`I{%F4T;Co|mZHTytdK;isqTl%=VM_OrA&^|D*)kC1bTi7IS;%ND zIs6o;7S@JHl~f@@F$6}@m%i`%A214)79J?Z#QA{rwc-EplawK-OxgHelMnoz}{acagSixG3d|l;#fJ6m+2;TBhc}DyX-b!i&i!=tN+Sva9 zSIQ7@g|8uF_}|2Ydm@8{n~_!T|Mi=rm&AjJ(B;66ld#gX?ehi5YhQyHFN|UF2I*@( zfZM#ZCR9u7j%x&>15<(ZLh$WofwMh_}xU0L0~T+ zrJJ?(28cZ-^!@IXKguqmtEB8zQPW4Q0R*HH9&$i@@I4ErYyNNul5V5!AYKw-*Gg^B z0TgmeF&i%h4ylgAtcRnSiY-%LUSgjfeT5K6kgeC8m4cm=WQADZP3{y}(g#v8E>57V zd1(SLi5@T6&wAdhHVbry3@ji|k)OcS0a(~~r)^Bv)wl=&y&u#-xdPHa=T|MqK07h~ z1yO;|4v60!8;IcP@`)CZFbL$!-?xPYfp+i>NNT?4s+Sbk?$EMEQ-_};waV27_@myS z*=;n#srYInF%B@o-gz-usIl%u1%MH&7WIX_t)W`|1c3J3RK`>V)u8nJj%^QHn&4fs z(L#N3o^5hKNxN6eEPTDU1js7KsQk#Db&<9U-+0~=<=14^%~Zj8qkjzAIr-X+}S(SVSsgT@^owz$gRFZ3PeXvAhg>!{yxGg|IMPI z;QGYa4VLD4eT~C`UaPyVI-pbI(Tb4bPD@?;J;ac^5EO*itO?Ma}^5w+=RXyXksJx3G~SRQEN8 zqXTZB*vEX*WC*-_gtP2d`s4w?~X-;0Ty&2CN{PB5K~y zDWDs&1VCvGE@#3ZaCeVA*rv>6shmYtC6wa@6qP8E>z6kvT=k%$ItJhE!}d%HDnS%B zgcz?d09oapJd+YzAk?;tr~{Kh&O5vh2r=(qkKeYo0gG5oE#PZ${?bFT-p58sl<(#v zT1=60A{v5x(+%03_l?aa%JUw_s*r#G5E_SDYht~UwVbQvAJ$cDET;RyE|1;KU8Zs&jaE6L5k)+^-=e%a7v~hqbVo z!URrV#amCkm^7Be!e($ACLQImU-S(lg`M6@FtPj?6Q}fxg{WkNjCzE(YZ%5>aKCdq zi3$c$bOb$XL4rK1K0axsrnG1Op-Y_vUGjD>n$l)P8ou_=aw; zPgJ-dkzE~*Sc`ZkWv}4fZ&b)TNtCdi_;v6?MrxgGK`rqpo=zOkv1QMy1&gP$mXGBf zH92?iwHTnk2-v21N$70oNZ6mMN2^OgcqeBWg51n;tV@pHRJlMa}yxyIT9|L541$S3>LG z9(&>#)&c@>JP&7Y%le+IseFUTX5?Xj4V8BOskc*;1&BJ+(oJR$#QjCUE}4JJzn?ed z;s=F4okSgv1zx4h-07|xv9<3bS~5~DH79H{JBmb{W_iIKiL3^>&-`uo`l>>>(`I~p zw^#U+EF~maO>oi>+gM@4F26RETPWzv=oj=0oGescArDueLJc5&4Py1f9@M8g0D$(j z0Gz&A(|@!{vohr2XE?Fg3S=ANgi}r<82F_XZUJ#^6KV^eIPOL-_#>@>Y=d-_D7=oh zHRT;o;Q0=-qRyey0xz%ungRLMYM0z9e&5~eI64@$cOC1_DSw8AxZK0OLEnl#kIxq1 zq~d4{vM4>A81Op0rNefC1K%6i3t?R6d){4?edi1}Y`WfdaO=`6%DC|n2zzIkutYBu zCHFY)!>i&cSf0gG@zdcNN}?>7EU5Y~q+PrHOE zp6gCg@PYQJjrkW&m<2j(E2Z0)K6#9@caP!ck4^h1WppfUcS2r?ednVOeI)MTg+1B@ z?3&i|gSORB1g6WtY#VhNfja}^BD4>c25?2Fgf!91B~5DmSpq9;8!+(5r{XVylS0oH zXp;ro9d3?l3{?tOsRubrUKm}G2+1Wv5gRCp(m5VSH<4HpB$TA~tqlxuMc6e7 z3>^s&PaMX1{Jim8m=YGl;d^qroxo0|@hx~bc^c;99QGg=&NBDb52@RbDg)|PKR{xW zEbK^j5zk-yd3yeaQv=KCTHQmC0$ZpshTmZG;Wcod0Pv5IcS04~ELU&P6J1_F!@`os znX_s3|H$*;<)<1Ee6VS1&g8$ZqxpYy?;9CZ?1Hu(r|bUg1k=eOLRjeWSNC_m0(nrbBSoD`zSt}wL?k{bD& z6C~Uv);X%&W*dF+Ekyj8&g%@JbFPI!Dm@eFQ>jwMR7v=9HM5(JGd=_(v_*(EN$Mv) zc@)i}xqN4iFNos?yP%9eU9EJqhcCsu5T5#;6atF{p?8EBSo4^c6k$uEui?Vb@;s#{ zrLn9t7uf>CHE`rP7i8~hX?4N0#G3ek68Lz3OGJY0phL6MRXP}MWT$Y-n)#)}LT4L%z`vf?z@9#c zXS?_IF192BThfMX;)G^MW@7GIt1Xd{w@e*Q*Zs-v+6vdM=P}b%%jC`o>5~{1BKB?j z-HF|-FcDu>6t`7#>?0hpHUj}rTfILNGrESQ_~oe*b4yCkr0}?P*T)C`UC+yqV}}FX^Bm@ zv+mt28gMpsM(ePv0F83=Wtw)w%1m z`z`!wdfG^Xxfj?s$VL3ZY!`%H6m^10y4FfKC%H^UE$o`$jbclDo#i?Q18Iz3J6UP5 zw?WFu!a!4s7%?2zySR>rJ(l|i2RU`(&39ZcKJyu}zN4}?3Qx~voDM)|YbBu4i>4-T zxxj44yoBMXynTP)cGMUbQ@_L7n;J)A8lLSPgf=#)y6xYLb}gh(3du!(F$_&tJ?*i} zhXho5qP^?3k@tI2*rx|Ow)N)p7pnD|l#Lac4fC-3Ru*0{bXEq4eIJl>pHq4^Ub*4@ z?E&t_QbACq-Ot>T&^`M04Fl3qw@Fm4E3ZpdN6gNl{cUIbqAa7@N2{uqaU zStuW?D)SrR1J!@9qI&vA`H7$qUx(X9tLd+`2nb??T#L7&u$?(BZijJE-+*I8dtlA@ zO~53n8)2#(YCEpnmfla)wf%1f1)MsbIMNmHER(T&1U`&;NHWk>wWGC5-baqzVmK=r z%(fNI=TBq^>z>=&FA)4fWQjKqYr2hf<(ojC80K-{bgU|v;XQ}*8ip=NE=b4wpl)VU zx6{w4W3KEa+pTx^86t>z-5$kC!v?rNMoMxI6pEPpB&9Iv2S@`?>LhdGw=qYL1ZGY4 z$@K);$)qa@5sfav4@rtb2_qDQ5w8VRy?IWaXexZVYcPFpg+}NC))Y$F{H+VOSiURH z`J_66)|JBHN%VBaLBYU246nHm#-5MyO_4THxXWi@a1HnC732cIgCsL8_RTiM^HD~k zOP?cW;;NCZaCu3gA8A0ocFm_p%Y(D((ymS{+5r-%r=k?gMFNwYP)8iV3(l!J$-A`c z@ks64b89^4+z(Bal)5CUgBT@^^Kwa2QkjPJH7lgaS=7P(*M zh23EfV~wYg)Nzc3^|2LvSqa;wJoh2epT@;7aP?<)b{AChPKuURCQq@ztkxJ-c#&l0hs~K)@UMuc9RaT37;+6I@6x)U` z9Z@%%WIR&b;4X*(o^7~`6wK;01AD{I&n)w8ee z^55@|y4U5iaWZmqKuVy*F_k?yj)fq?N?c*Ums0Zlp7x+)S4;)bXDP*jrJ)mm#7HC+ z!{8CSw$&KI8LS#IwJ*=_sVjZCm-cufv@)psCN?JG%iV5e?uJInAIDk~3tb<7Mz6Y~ zkl3OFTTZ7GK^PZ`RbtQN*J2W{;1%?C^>N`u;nDc6Ra0dTGOx)Xn})9qQj&__^fm|m z3LN=y{AE5VC8X}-R|##zCXp#JzwWLO?m@I^O6fL{>O}YwNd%ut-&S6%;v+RpZm$;^Gr_0k4@L(3s!O!mK5s$pwtS(Fz>kqQx^0bjfJe*KL-I2TbK#? z%ZjN`uw;8NQ2fJL;GWHpk`7aK>3=YIzdl>0XPB#6zXFf;YppQozC}>W-D^{+!Ag~q2Chwpf^+D#zaNUEGNhqyL);V z**CwXJ=T2l=FQ}|8NkLtRY9}d<;Y@`X0+JwF)1i5Y6GL#U!@9=bpu4@Rh5{&Vxqxn z8?bVl=C@ya33*u>zI^HJPUcu3;thOG#{q5N_2W;BMIZ&{-zH`Ow*uWy(o3{fLxfD3 z>F_$F?Sbr(P|G0`O#&93*N}ZjB$Fm*737*{K~xH05y>|L>=wRq|$Swe!jh@swpWq)Lo90gg5L>$pSo=Zc5Sq>R zx^Wt?WU5t=5Bp9vJWG>>m_Dv9V#BssKZnKM_b2O-=V#_SbCy}AvXWYelv6=>NBLHC z7q*z_7&yz%W%*P*tHh|Umj`aTIR%WHHXs*R;HBA<7d*_}(32=K^EC`ohiTvWI`Q|`5ho&rwOp)XjPW;+VTXrs?F`LlBn78{Qr{+WsI zAHeq%R{dB}V;`jf;sLkGx4WZf87Gb;uSNu007vng=}0c|_dJKKX^Zbp;CnB(U6HE+ zGjS@A*ryX>>J3A@pvg(ex`B9pp-^B}Yf)71c2+$%1W15v{RujlB%EKc5~qRDbo!pc z1budfPRVeNYK=K0V4NU759_s`YxYT(t~GA3viJJwXZF0q6^JjNAS#Ud5}@`^r2KZ7 zTmYg0&%_1*2m`>hYhco@<}4JN46+hRm|OF)c+Qe_nD%Qjf!EcE=kG+f|4zysgB)sx z<11J;`!L7B=xeMY#4gt?%jrlzmJFi01ZKSx4ObVIgpEVx%eU!20+g~xdC<}=pK6sM zzq0ut-+a8Zvn!He#Tk=ph1HRK0aUQHu#=tFh8~X=8Auay4*U9gAEa!U(v%v1=MoZ{ z0^(1+BDrnhx$#e+GZO0 zP27C&#r64!##@=@=^uJBk+3m!!*?`6?t;-?>D>kaYP)8YMS`U<{#mjqh7Tw~dX7*CT;02RS1 z?vewAE4c1OTVQ|oW?h5-y>&k-(%}!8ZFHwMa1Z#Uo-3)I;SFIBKh+%iI~C`_RGc&L z2?6$FnoC`gPcisP3xSg|`C-k>MepTKm`A1l0tEYfr5ZxUz#=pwPa;sEV>RLBi%r(c zO4e6BDE?8Yp=ASDQWJn+xv`9)17v)jZ4U-Yv#1R6m{s0lR#@GQ2I7>skrWstj`M9{aRF(6DZ)tnxf^Wr*?)Nf9sbGWpf*f5^0H_*& zi9%}ynQq1KS||VHn=CO3Bjr}X9tI@ISEmU&OuSC#I~`NWA3#ZZ(P=~{RWl7?i24Pr z7J}Q4qOBL&tE$`~mn+StoVL3U9z2MM{ZgWRO}5Ss)V;DzA|5M#?B_01SONx~woQAW zL}{~<-HXv_g&z!?5B7?qEJI{uzKQ$j=+OT5%3OxxaQ~uIHl46Ldyl_ zfe$qu1OB{o*xdrrW^XY!pKf5Lw(btqZa~o``e18jMizLCJb+Yu+`z0a=UG}8+f*4? z@V?)1n2g8?0h?*n5tzx`KoR}?bAR$eFDOC)e!{D)tgOSRhVwSa3PH2hu?Q%d*(~Jm zVGFTL)BM?9*HJ-uMSk!Q{W;H?R2bsL-0e{|vdkoH{A8f6XCUQx6?`_o63L)CPH}ZC zO<0(W2WqOgUAh8m|Q1sq(<> zm75OJ{{-Gk=985Jy?}lGtB{qaIocjZk!VE0{de=Qfzfqf60@f^1}WYdewzB86`??f z{tVq05M7G_Ii#ys`0Vxbtw|93?nQsLYDQDZ*Q(N*0M6)opnzWz92~4Qp}eKmz*7j! zJyq&mPef~gA2|@~`6j$`=E-;(x4f3su?Oh&75n>kU~7+i3MdJ{>7LB^9pRr>94TzAuI8>rM%UD@+Uw z3<&sbemvPVFmLSku5;~cfVvn6c>|%EBIM4=!%cl$h%0^&Y2FXT+mVM|_h;IOwrGhOCu>E#rU4Vz_9%mMLH?-S8Z5Kv=MrRyh zYYgX7Eu8GF)I|0)?IFL*CLivGs?y-%MsBD3MUX#!ERUrH$E%(%!=F^>L_meX!dOEn z`0px3M~v*w>Ix(cQ>gdn!>zYJKU!7Dlx{F~JO*`H+ZFI$Uk!%&2BbV7$yeT*6vq}? z%r>J_Z>gjQtQ`Am&pY}}Uqk}wY_-+Ow8Tl6GX~TWc+~a;aCyE1Zpn2Q@O5-0xE-n= z^+3wX(Ep%Cqh(N?v=K<8zU`a>JCj3W#|v1|e?`;IpwS(z~Y_q^ePB9AR_@ zsxuZErO9u^{6axf%eitBom3UYGZ7~e{p){LuDf|}oAscPzOO_#)TN8nL4OXd-$G^z zK|JTmwcSxH6;Roe+-+a60%fET_7KN+F71ZQw5DTE_wSs02(Q91@l6~E;_A^9c zkabdbK`l^ILnHR5%#eZ6d2(SU!UXHF2gqeEUma<-bOMQtO<2kRjmdc;khQR?3QhkR z4`A-C=+T5S`akPOdkmr0fH<5r z(R&8(0X(~0%&^o`;b3?|`p@@w4dwk)N`!CN2dt={{~ZbpLP8W)PCn?x{S!^X{SinA zR^Lj_zuP&~R7gzUy6Sm6XwCckf#Blc2&T0O$=`G6n8C&|Zo;aK!reYj@M$)^|QtrzgL+-1lIJCjcTjfozN*?B^WOe|y(A7#*Dcaa&63IqB1R zYaFuce=j5o_N=BEvWDV~W59MLxj(i)< zYtD~6QC=#!O)W~tNW~M|EdiptKrvBItAi?R_>CasJ>Ck0XhVjM3*ly*yE{wMHt$TB z1g~IAM}QR}?@e%(q-a>CU-)#>3!9Q;!KYL5Q>y65-eQ+Gel48I1pVP`_fAkM3pyR- z!C^tcL1S@&hpPB9Xf;oxL8indEoXO9QXXQ!co z32*ikgvhqG-I?tA=9ktVlvjE;lt05EJN-}Qj{ae zjGO^wtQ)@~r0CAj(-{WfJT~2!@#8ZXytiB`T2Te4v9wAvsbzW6P7t| zUTUnQWgu)Kes*Ps-{nu}q*K!M#%5~w zu$f0-qj8QVU*~8!l8NKY->u$RS5E|MbY$B%apkhM@0Rm{sVcKsRDxF9u#3YE`9wX- zZpyQ+D5j<1JX60q?isa8f&S$Hu5R^(wFlgm8}5TYZAny%K?(ayPZu9{!;2N#$n|w8 zeDg#30U@6JXR;a9r@ZKQbGxxYUJi)S5ltnmg-ex4BcJuEF2+KQb&JYY&%S);OS@O^0uPsB-FokCUt@vOrGhmwTu({*SI!dO*!-1GJO^EK^t0kr zde2N`j^_mbXsPt+P+74K>occnUALN}cT9(~9h&CL;j}7E7&c54>aV&hDA;{6EGiX1?U)7BZW1vTj=kJc#hvd!3C?nfrJ9!rTTvR?}VS&|K z8D8f}%`sRNR8Ck^?aOyI)TR{dB^{d_=gD0aF1cczaG?`evG?{UILJRZVUR2Qm_$Bs zt8Wpj?M`j@NmF6EO4@nB;5kQad5uubc)fPROhSEX{l#WHiD$|>f4HH^!l&qEBj(CH zAFiF|TwP)Hs?lnfQTd0n#whL=7=p2NV>f>-z@Eq`!u!r&gb$ko*=M@1lYCEi(aD1Z9S^gL#ynS8*S)>Qe}d@)>OZT-MO`G`S%B5Ochx(X*F{{2c}F{vPUvC0;hI7;wDe%Q%x29 zyC`~dmXHz7tXpSRK|%tS8OqFrCnAW0oR4Kur#x1k3fB(!x6Y?`oY^W~EpD{MbCm{Q zcvq*-4f0ebG%FQsRNUmT=^!6K_&P-yZ`A3G)U)Ij1?a9@P0Uo5M__mtE5Fq$VHm7- zs`Isf{-LBH1N5_3sd=+`wnRC9-UsMTP@-ooZ0dT8@Pvh5xMSsadVfz;w9WzDd`;IQs|~ixAw_#So!#%?nWW$M8;T|kx~hb4$|(suNSuE)%+Smh~317{pbgiSX<%C z$cw&0ej9j}-_={bkZf+OVqN$$T<9e+F=p$Xjo4wJo>&eVcvL^ zYp=-}pTqtIBZWTf+&1nhz~1LncEi2m`8>N6I$Nvvrgsj6D^~?ekEL2> zR~+m7`7dL1ct3eAzsQbK98E3XY)A^_q1X%mDXC1VUE4-aKjRXl=z_b13Tc>*x7jX{7}2qdLL`y1rw{vA1503R6vIqrh}aWpv(-<1P(`b*AGXs5N} zL=JN=pW5GBY&&==57$tmlpV^7(@h>Qzx}~W z&nJCSHjP(3xHQTk2F+<41 zWp!&%{tMdYoOjL1&NPGAuc`{a_ha~u)>ii(6w)6bShyuWJrkb7GCRYtx7heN7?)L! zPs`Q7N-PUl!jPw)&Nho7P<9r9{$rg7cp#(gr&mU&IR7wL9k#*+M-R&o?vMhjgzVe5 z&KKoQK_bN;Pit*zcm&T9S5m7GI;hn-CSHuPs(BApk9S264&Q|>cq>2*%# z1{Hk_7&gmynP>7ls+6oT82O)SEU99j)r1jd;=#Bia>)6Zh)!H3N(${;%;OuVGI-=L zH0pO$6mn3O34O;N+Ttpge!1JQ`*}29Hb{4wZLMq>J^M)En!HfZsncHll!frJ?KGLB z(DX|F5qI3O1U*s0>emXb9|x4Q&mU-(=UyBXc-VVdfew72omOV!(vOF|V`B|BE&&Zc z7I_gc*USO8G(Taj6jTYIng$>Zm1H?sp6o7_Tg^3#3OFO2A@i}%&TCoaFFS9?@mT82 zUYmld6Od#XV5-{KOLHxLwi|K5mN1`AN7X9YUl9fhXHQOoU|!~wJ>`VZg#6yY5KXy- zL$8N1gL95uhql55^C5v33QJGF;rYpjymUUFm2>u?v-_4ffnAFp8>KUen%%u7N}a;{ zz-dB}vnzlo0d0dk)icvkcIg&m+sOmVHG=HRF zv0vU$W?aEE=KRGbzxlQ9t8z_~+{g|Mq`moy8ZG(kWz!L7%KKU?{0eL?gOI-wX{G`p z8O)y8ol=P?Uf8WVk|lAFK>!qM&H!iV^=dx?0Qy`mpo!J%nR>VMx9a&?a$!&jX9F;p zZAsAGpsXvuAUqE65+RWZ)ZFbSkk*Sb0r0rmVawv?twrzS#+x#SUVJ_>Z^Wy}pkW1) z6K8!X#1=bv3*j6&HFMLS&q|(Lli6FkygZQO33ib`8c+8jKV__3#o>5dzAU)tR79A} z?b6aozZP;8 zqsLAz1LV&w5zV>2)d$BULav(kDyLb7r;@R;&ghTJcYUMFoYlnp(K8y#W1rUqRUm5w zNpK3f5005^PUzHYl+1j6sD$;oL~MP*?PrW{N#ZU+tg&Ow)lgZlz`F`T+$>qk|XXoGmzD*-46&Bi~%YXy6 zg**pb?;wu^s09bq(Hv?o!VCvs_b}83%^3}!{U#<6d`Ij3(sgI_4@+`3cc9qP3*Gou zy!-eNvt=256q`?Lo{Ptwz`^8@iRr0TWbIVTW8N`MHnM!d6CWPwfwKX!`mfVn)3a~H ze^%_(3rUKP>EsuO=xk|O4WyWeJqCS6ZOp%|8KVmLu!^KEXTx{f8!S+jQKhfOybvAF zD$RGJ5+@6bmg(qUo6ZI3c3oLVaP2o2_iBe7?T9>A6aG=qJ>~8k97?YHfiVrVSKEAc zxE1F5bnO5tsA#L14W@^XnvR@{A8daGttB=JsC0d#A|a&9rybzF-?6yN?v*&-fr=na z;w_eS2cL2pUXQUu4i4tu!v}v(0qA@b(UNjij=yFyl2n8r-D;0q@9be5= zSoEzZhuUMgYza2$;A_%L{)B56GFSS?yfT)IQ9i#iI#FxFenF4g`h{hLPwL0)VK{g?hR${)g>8XE)LI6Cn_u*SP~uQ(ICDpvPkDC5y@5c z!PzM9om#*W`IAXDTHo#u)cv{xwas3K1WS(Av~{pqS?4@79T;(;-^$ass%bwLF%ZdH52<8 zCd0SqYM7QJ8Nt7wqZnfn$ifamN0SLao7M^gUIL$2kP%|zVA0$F7?53%bZM*_!#mZy zt{B2>C$BX_Or}fjtE1Vbcjn!nm@LwY2_i}X3NQv54= zdPhm?H=QOm3_B$vVc`S^(8~1>B;<+gtb=fG#?KQI{&YLboJkbT?fXFySqATV_qMfl z<>Po7a1-H_$Z5r)mSHkptaqnOtaJgQO+Ji@gB}(b@);yjp6?%oSKB}AtlhrbV7h7c zerA@AB}{?n9^UY;EUU+q$!%&?SvJ1`<^PgFW)_;Iab2A2@OA=NMCJA8XJ~rizr)`J<|OHQV1Kb(nJ6tI za4m*nW8n?DnIs)!F3|zb?;ik-Wfp*CM0e2ZdnSX$;0WRjD49IS(YQKPfPblJc4umB-D$vm5DOPW@}ZRv#={ zLOIRvT7Dz0-f^_Z^}qj$1f6>Q@2R3PgmlW;c!4pGx|Lpp#LsP_rF(voA5W?|8+Zp*L=fHIDX$4^!t=}yFCAM zs{H@l8WM5>QcK4<@xLp<(NBKA&tK;sr)4PUdOi6M@`BT;BR3I;&n>*S->TC1`?!?3 zpqch7BO|}OMybEYkUST0VPD<7n-^<3v~Zx01iEj9B+*ud=$IJw*wGs#sMzZ*yrv}fQd-~Dh;e!Zhz?k;1BL1M2!zIyv4MC0x4 z|1PcPX%*AwBi1m%s^v3#xM9@?dkSX7m+4W1=eXDV2EJ2y?MI5 zQVYefzh?!V4szF6d`l2X*S$0(Ja^S~T87gUEg?<)lSt$lzOgLXej zx_4Dtzkh3e&*SWV+=?4OjoqJm>EzAoWq$UWIio59h4oMG-_N^)l9c%=;@^B})fL@C zJxH0;zgUfvJ?~jge42o?YJckUNmL$j<@pBvmRHl(#SMGb12QT4rZeemTgC~4smr2x z(lhGcb5gXEV0-UbF5a%$%vH}-o-NG%&SsrQf|D3Rmyg%+B*bT8ndjq-k^AR!_nTXZ zx-Ox@}rlY246W(HYhertR`F&3lRzwrWyT7%RG41Qw3W*B&Pye7ybP1 z6NrhhLpRP2!SV7j>_6wqri&`BL>l8^xb!?apfNw{qk2K7gt|e-tcA)$vT563edS=~ z=ApT|Wju~Hb-&?gfQ+Nz;so>WmHo>^6_+Bt#viQOY8%xf6!3>0xqCA0Q=SYfyF|zu z3m^_c%cuI^I6^4UuK{a)oRs8hTe_kc{U#+(Q;&{uz0;SxGP7CBxF(`ECo_~V%dPy+ mnvaA$u>3yxePGAVB}L(;duA!KRxJ|vpNyoOM824w*Z%=kfrC;2 literal 0 HcmV?d00001 diff --git a/search/search_index.json b/search/search_index.json index 029d35e..216c2a7 100755 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Introduction","text":"

This repository contains the source and instructions guide for a workshop on building a RAG-based chat app using Azure AI Foundry. The workshop is derived from the official 3-part tutorial and adapted to provide additional resources and suggestions for self-guided learners.

"},{"location":"#pre-requisites","title":"Pre-Requisites","text":"

To get the most from this lab, you will need the following:

  1. An Azure subscription - Get one for free
  2. A GitHub account - Get one for free
  3. Familiarity with VS Code, Github & Azure.
  4. Familiarity with Python and Jupyter Notebooks.

Verify: Your Azure subscription has sufficient quota to deploy these models:

  1. Chat: gpt-4o-mini
  2. Embeddings: text-embedding-ada-002

Verify: Your Azure account is authorized to make role assignments for Azure AI resources. This may involve having a Privileged role like Owner, User Access Admin or RBAC Admin.

"},{"location":"#getting-started","title":"Getting Started","text":"

This repository is instrumented with a devcontainer.json to give you a development environment with all required dependencies pre-installed. To get started:

  1. Fork the repository to your personal profile. Visit your fork in the browser.

  2. Launch Codespaces on that fork. Setup will take a few minutes.

  3. Type this command into the VS Code terminal when ready. A dialog will pop up.

    mkdocs serve\n

  4. Select the \"View in browser\" option in the pop-up dialog. You should see this guide.

  5. Open a second VS Code terminal pane. Use that for all further instructions.

"},{"location":"0-Workshop/1-Overview/00/","title":"1.1 Learning Objectives","text":"

This lab takes you through the \"Build a custom chat app with the Azure AI Foundry SDK with a devcontainer configuration for fast setup. Fork the repo and launch GitHub Codespaces to work through lab exercises.

The lab teaches you to build a RAG-based copilot using the Azure AI Foundry platform By completing this lab, you will learn to do the following tasks:

  1. Use the Azure AI Foundry Portal to
    • start a new Azure AI project
    • discover and deploy AI models to the project
    • add an Azure AI Search resource to the project
    • setup an Application Insights resource for tracing
  2. Use the Azure AI Foundry SDK to
    • create a new search index with your data
    • extract customer intent with an intent mapping prompt template
    • retrieve relevant knowledge using the search index
    • create a chat agent with a grounded prompt template
    • process customer queries returning responses grounded in my data.
  3. Use the Azure AI Evaluation SDK to assess app quality by
    • creating a custom evaluation dataset.
    • executing a custom evaluation workflow.
    • viewing evaluation results, in your local environment.
    • viewing evaluation results, in Azure AI Foundry portal.

This sets you up with a \"sandbox\" project that you can use to explore other features or tooling in Azure AI Foundry. For example - try bringing your own application data, or adding secondary data sources for knowledge retrieval as part of the RAG flow.

BONUS: Use what you learn here to explore Contoso Chat next! (click to expand)

Contoso Chat is an open-source reference sample that implements the retail RAG-based copilot (using the same data) in a code-first approach from initial provisioning to final production deployment.

  • The current version (v3) provisions the Azure AI Foundry resources using the Azure Developer CLI (azd provision) and deploys the final application to Azure Container Apps for a hosted API endpoint.
  • The next version (v4) will incorporate more elements from the Azure AI Foundry SDKs that we highlight in this RAG tutorial, including use of the Azure AI Model Inference API and the Azure AI Evaluation SDK.
"},{"location":"0-Workshop/1-Overview/01/","title":"1.2 Application Scenario","text":"

In this tutorial, we build a retail chat AI (copilot) that uses Retrieval Augmented Generation (RAG) to ground the chat responses in the retailer's own data. Let's briefly review what this means.

"},{"location":"0-Workshop/1-Overview/01/#1-rag-chat-app-tutorial","title":"1. RAG Chat App (tutorial)","text":"

This RAG Chat tutorial provides a quickstart for building and evaluating a basic RAG-based copilot using the Azure AI Foundry portal and SDK. The tutorial is grounded in the Contoso Outdoors retailer data and combines both low-code (Portal) and code-first (SDK) steps to teach the latest Azure AI Foundry tools and features. Think of this as a sandbox for open exploration

The figure explains the RAG pattern visually:

  1. The user question (prompt) is received at the copilot hosted endpoint
  2. The question is used to retrieve related knowledge from relevant sources.
  3. The prompt is then augmented with knowledge as context, and sent to the model.
  4. The model now generates a response that is \"grounded\" in this knowledge context.

"},{"location":"0-Workshop/1-Overview/01/#2-contoso-outdoor-chat-ui","title":"2. Contoso Outdoor (chat UI)","text":"

Contoso Outdoor is a fictitious enterprise retailer specializing in hiking and camping gear for outdoor enthusiasts. Their website (chat UI) provides customers with a catalog of their products, with product pages offering detailed information for user review. We'll look at the retailer data sources in the next section.

"},{"location":"0-Workshop/1-Overview/01/#3-contoso-chat-chat-ai","title":"3. Contoso Chat (chat AI)","text":"

The chat UI shown is not used in THIS workshop - but code is open-source if useful.

Contoso Chat is the open-source reference implementation of a custom RAG-based retail copilot based on the Contoso Outdoor retail scenario. It is implemented as an AI App Template that can be provisioned and deployed to Azure Container Apps to provide a hosted API endpoint. Customer requests on the chat UI (website) are now directed to this chat AI (endpoint) for processing, allowing for the user experience shown below.

"},{"location":"0-Workshop/1-Overview/02/","title":"1.3 Application Data","text":"

The Retrieval Augmented Generation (RAG) design pattern allows us to customize the AI by enhancing the user prompt with dynamically retrieved knowledge that grounds the responses in the provided context. Let's understand the shape of the data available to us - and think proactively about how you could bring your data into this mix.

"},{"location":"0-Workshop/1-Overview/02/#1-customer-info","title":"1. Customer Info","text":"

This record represents a single customer, providing their profile information (\"id\", name, contact info) and their purchase history (\"orders\"). This JSON data may be stored in a noSQL datbase like Azure CosmosDB and retrieved dynamically by the chat AI.

SAMPLE DATA (JSON) - click to expand 
{\n    \"id\": \"1\",\n    \"firstName\": \"John\",\n    \"lastName\": \"Smith\",\n    \"age\": 35,\n    \"email\": \"johnsmith@example.com\",\n    \"phone\": \"555-123-4567\",\n    \"address\": \"123 Main St,  Anytown USA, 12345\",\n    \"membership\": \"Base\",\n\n    \"orders\": [\n    {\n        \"id\": 29,\n        \"productId\": 8,\n        \"quantity\": 2,\n        \"total\": 700.0,\n        \"date\": \"2/10/2023\",\n        \"name\": \"Alpine Explorer Tent\",\n        \"unitprice\": 350.0,\n        \"category\": \"Tents\",\n        \"brand\": \"AlpineGear\",\n        \"description\": \"Welcome to the joy of camping with the Alpine Explorer Tent! This robust, 8-person, 3-season marvel is from the responsible hands of the AlpineGear brand. Promising an enviable setup that is as straightforward as counting sheep, your camping experience is transformed into a breezy pastime. Looking for privacy? The detachable divider provides separate spaces at a moment's notice. Love a tent that breathes? The numerous mesh windows and adjustable vents fend off any condensation dragon trying to dampen your adventure fun. The waterproof assurance keeps you worry-free during unexpected rain dances. With a built-in gear loft to stash away your outdoor essentials, the Alpine Explorer Tent emerges as a smooth balance of privacy, comfort, and convenience. Simply put, this tent isn't just a shelter - it's your second home in the heart of nature! Whether you're a seasoned camper or a nature-loving novice, this tent makes exploring the outdoors a joyous journey.\"\n    },\n    {\n        \"id\": 1,\n        \"productId\": 1,\n        \"quantity\": 2,\n        \"total\": 500.0,\n        \"date\": \"1/5/2023\",\n        \"name\": \"TrailMaster X4 Tent\",\n        \"unitprice\": 250.0,\n        \"category\": \"Tents\",\n        \"brand\": \"OutdoorLiving\",\n        \"description\": \"Unveiling the TrailMaster X4 Tent from OutdoorLiving, your home away from home for your next camping adventure. Crafted from durable polyester, this tent boasts a spacious interior perfect for four occupants. It ensures your dryness under drizzly skies thanks to its water-resistant construction, and the accompanying rainfly adds an extra layer of weather protection. It offers refreshing airflow and bug defence, courtesy of its mesh panels. Accessibility is not an issue with its multiple doors and interior pockets that keep small items tidy. Reflective guy lines grant better visibility at night, and the freestanding design simplifies setup and relocation. With the included carry bag, transporting this convenient abode becomes a breeze. Be it an overnight getaway or a week-long nature escapade, the TrailMaster X4 Tent provides comfort, convenience, and concord with the great outdoors. Comes with a two-year limited warranty to ensure customer satisfaction.\"\n    },\n    {\n        \"id\": 19,\n        \"productId\": 5,\n        \"quantity\": 1,\n        \"total\": 60.0,\n        \"date\": \"1/25/2023\",\n        \"name\": \"BaseCamp Folding Table\",\n        \"unitprice\": 60.0,\n        \"category\": \"Camping Tables\",\n        \"brand\": \"CampBuddy\",\n        \"description\": \"CampBuddy's BaseCamp Folding Table is an adventurer's best friend. Lightweight yet powerful, the table is a testament to fun-meets-function and will elevate any outing to new heights. Crafted from resilient, rust-resistant aluminum, the table boasts a generously sized 48 x 24 inches tabletop, perfect for meal times, games and more. The foldable design is a godsend for on-the-go explorers. Adjustable legs rise to the occasion to conquer uneven terrains and offer height versatility, while the built-in handle simplifies transportation. Additional features like non-slip feet, integrated cup holders and mesh pockets add a pinch of finesse. Quick to set up without the need for extra tools, this table is a silent yet indispensable sidekick during camping, picnics, and other outdoor events. Don't miss out on the opportunity to take your outdoor experiences to a new level with the BaseCamp Folding Table. Get yours today and embark on new adventures tomorrow!\"\n    }]\n}\n
"},{"location":"0-Workshop/1-Overview/02/#2-product-manual-info","title":"2. Product Manual Info","text":"

This record represents a single product in the retailer's catalog with extensive text (formatted as Markdown) covering information like brand, category, features, technical specs, user guide, cautions, warranty information, return policy, reviews, FAQ. This information may be used for building the Contoso Web UI, and potentially for grounding responses related to richer QA later.

The product info has been rendered as a Markmap for visual clarity. Simply zoom in/out or pan in/out to explore the content. You can click on any node (circle) to expand/collapse its sub-tree. You may need to refresh or reload page to re-render the tree.

SAMPLE RECORD (Markdown) - click to expand # Information about product item_number: 1 TrailMaster X4 Tent, price $250, ## Brand OutdoorLiving ## Category Tents ## Features - Polyester material for durability - Spacious interior to accommodate multiple people - Easy setup with included instructions - Water-resistant construction to withstand light rain - Mesh panels for ventilation and insect protection - Rainfly included for added weather protection - Multiple doors for convenient entry and exit - Interior pockets for organizing small items - Reflective guy lines for improved visibility at night - Freestanding design for easy setup and relocation - Carry bag included for convenient storage and transportation ## Technical Specs **Best Use**: Camping **Capacity**: 4-person **Season Rating**: 3-season **Setup**: Freestanding **Material**: Polyester **Waterproof**: Yes **Floor Area**: 80 square feet **Peak Height**: 6 feet **Number of Doors**: 2 **Color**: Green **Rainfly**: Included **Rainfly Waterproof Rating**: 2000mm **Tent Poles**: Aluminum **Pole Diameter**: 9mm **Ventilation**: Mesh panels and adjustable vents **Interior Pockets**: Yes (4 pockets) **Gear Loft**: Included **Footprint**: Sold separately **Guy Lines**: Reflective **Stakes**: Aluminum **Carry Bag**: Included **Dimensions**: 10ft x 8ft x 6ft (length x width x peak height) **Packed Size**: 24 inches x 8 inches **Weight**: 12 lbs ## User Guide ### Introduction Thank you for choosing the TrailMaster X4 Tent. This user guide provides instructions on how to set up, use, and maintain your tent effectively. Please read this guide thoroughly before using the tent. ### Package Contents Ensure that the package includes the following components: - TrailMaster X4 Tent body - Tent poles - Rainfly (if applicable) - Stakes and guy lines - Carry bag - User Guide If any components are missing or damaged, please contact our customer support immediately. ### Tent Setup #### Step 1: Selecting a Suitable Location - Find a level and clear area for pitching the tent. - Remove any sharp objects or debris that could damage the tent floor. #### Step 2: Unpacking and Organizing Components - Lay out all the tent components on the ground. - Familiarize yourself with each part, including the tent body, poles, rainfly, stakes, and guy lines. #### Step 3: Assembling the Tent Poles - Connect the tent poles according to their designated color codes or numbering. - Slide the connected poles through the pole sleeves or attach them to the tent body clips. #### Step 4: Setting up the Tent Body - Begin at one end and raise the tent body by pushing up the poles. - Ensure that the tent body is evenly stretched and centered. - Secure the tent body to the ground using stakes and guy lines as needed. #### Step 5: Attaching the Rainfly (if applicable) - If your tent includes a rainfly, spread it over the tent body. - Attach the rainfly to the tent corners and secure it with the provided buckles or clips. - Adjust the tension of the rainfly to ensure proper airflow and weather protection. #### Step 6: Securing the Tent - Stake down the tent corners and guy out the guy lines for additional stability. - Adjust the tension of the guy lines to provide optimal stability and wind resistance. ### Tent Takedown and Storage #### Step 1: Removing Stakes and Guy Lines - Remove all stakes from the ground. - Untie or disconnect the guy lines from the tent and store them separately. #### Step 2: Taking Down the Tent Body - Start by collapsing the tent poles carefully. - Remove the poles from the pole sleeves or clips. - Collapse the tent body and fold it neatly. #### Step 3: Disassembling the Tent Poles - Disconnect and separate the individual pole sections. - Store the poles in their designated bag or sleeve. #### Step 4: Packing the Tent - Fold the tent body tightly and place it in the carry bag. - Ensure that the rainfly and any other components are also packed securely. ### Tent Care and Maintenance - Clean the tent regularly with mild soap and water. - Avoid using harsh chemicals or abrasive cleaners. - Allow the tent to dry completely before storing it. - Store the tent in a cool, dry place away from direct sunlight. ## Cautions 1. **Avoid Uneven or Rocky Surfaces**: Do not place the tent on uneven or rocky surfaces. 2. **Stay Clear of Hazardous Areas**: Avoid setting up the tent near hazardous areas. 3. **No Open Flames or Heat Sources**: Do not use open flames, candles, or any other flammable heat sources near the tent. 4. **Avoid Overloading**: Do not exceed the maximum weight capacity or overload the tent with excessive gear or equipment. 5. **Don't Leave Unattended**: Do not leave the tent unattended while open or occupied. 6. **Avoid Sharp Objects**: Keep sharp objects away from the tent to prevent damage to the fabric or punctures. 7. **Avoid Using Harsh Chemicals**: Do not use harsh chemicals or abrasive cleaners on the tent, as they may damage the material. 8. **Don't Store Wet**: Do not store the tent when it is wet or damp, as it can lead to mold, mildew, or fabric deterioration. 9. **Avoid Direct Sunlight**: Avoid prolonged exposure of the tent to direct sunlight, as it can cause fading or weakening of the fabric. 10. **Don't Neglect Maintenance**: Regularly clean and maintain the tent according to the provided instructions to ensure its longevity and performance. ## Warranty Information Thank you for purchasing the TrailMaster X4 Tent. We are confident in the quality and durability of our product. This warranty provides coverage for any manufacturing defects or issues that may arise during normal use of the tent. Please read the terms and conditions of the warranty below: 1. **Warranty Coverage**: The TrailMaster X4 Tent is covered by a **2-year limited warranty** from the date of purchase. This warranty covers manufacturing defects in materials and workmanship. 2. **What is Covered**: - Seam or fabric tears that occur under normal use and are not a result of misuse or abuse. - Issues with the tent poles, zippers, buckles, or other hardware components that affect the functionality of the tent. - Problems with the rainfly or other included accessories that impact the performance of the tent. 3. **What is Not Covered**: - Damage caused by misuse, abuse, or improper care of the tent. - Normal wear and tear or cosmetic damage that does not affect the functionality of the tent. - Damage caused by extreme weather conditions, natural disasters, or acts of nature. - Any modifications or alterations made to the tent by the user. 4. **Claim Process**: - In the event of a warranty claim, please contact our customer support (contact details provided in the user guide) to initiate the process. - Provide proof of purchase, including the date and place of purchase, along with a detailed description and supporting evidence of the issue. 5. **Resolution Options**: - Upon receipt of the warranty claim, our customer support team will assess the issue and determine the appropriate resolution. - Options may include repair, replacement of the defective parts, or, if necessary, replacement of the entire tent. 6. **Limitations and Exclusions**: - Our warranty is non-transferable and applies only to the original purchaser of the TrailMaster X4 Tent. - The warranty does not cover any incidental or consequential damages resulting from the use or inability to use the tent. - Any unauthorized repairs or alterations void the warranty. ### Contact Information If you have any questions or need further assistance, please contact our customer support: - Customer Support Phone: +1-800-123-4567 - Customer Support Email: support@example.com ## Return Policy - **If Membership status \"None \":** Returns are accepted within 30 days of purchase, provided the tent is unused, undamaged and in its original packaging. Customer is responsible for the cost of return shipping. Once the returned item is received, a refund will be issued for the cost of the item minus a 10% restocking fee. If the item was damaged during shipping or if there is a defect, the customer should contact customer service within 7 days of receiving the item. - **If Membership status \"Gold\":** Returns are accepted within 60 days of purchase, provided the tent is unused, undamaged and in its original packaging. Free return shipping is provided. Once the returned item is received, a full refund will be issued. If the item was damaged during shipping or if there is a defect, the customer should contact customer service within 7 days of receiving the item. - **If Membership status \"Platinum\":** Returns are accepted within 90 days of purchase, provided the tent is unused, undamaged and in its original packaging. Free return shipping is provided, and a full refund will be issued. If the item was damaged during shipping or if there is a defect, the customer should contact customer service within 7 days of receiving the item. ## Reviews 1) **Rating:** 5 **Review:** I am extremely happy with my TrailMaster X4 Tent! It's spacious, easy to set up, and kept me dry during a storm. The UV protection is a great addition too. Highly recommend it to anyone who loves camping! 2) **Rating:** 3 **Review:** I bought the TrailMaster X4 Tent, and while it's waterproof and has a spacious interior, I found it a bit difficult to set up. It's a decent tent, but I wish it were easier to assemble. 3) **Rating:** 5 **Review:** The TrailMaster X4 Tent is a fantastic investment for any serious camper. The easy setup and spacious interior make it perfect for extended trips, and the waterproof design kept us dry in heavy rain. 4) **Rating:** 4 **Review:** I like the TrailMaster X4 Tent, but I wish it came in more colors. It's comfortable and has many useful features, but the green color just isn't my favorite. Overall, it's a good tent. 5) **Rating:** 5 **Review:** This tent is perfect for my family camping trips. The spacious interior and convenient storage pocket make it easy to stay organized. It's also super easy to set up, making it a great addition to our gear. ## FAQ 1) Can the TrailMaster X4 Tent be used in winter conditions? The TrailMaster X4 Tent is designed for 3-season use and may not be suitable for extreme winter conditions with heavy snow and freezing temperatures. 2) How many people can comfortably sleep in the TrailMaster X4 Tent? The TrailMaster X4 Tent can comfortably accommodate up to 4 people with room for their gear. 3) Is there a warranty on the TrailMaster X4 Tent? Yes, the TrailMaster X4 Tent comes with a 2-year limited warranty against manufacturing defects. 4) Are there any additional accessories included with the TrailMaster X4 Tent? The TrailMaster X4 Tent includes a rainfly, tent stakes, guy lines, and a carry bag for easy transport. 5) Can the TrailMaster X4 Tent be easily carried during hikes? Yes, the TrailMaster X4 Tent weighs just 12lbs, and when packed in its carry bag, it can be comfortably carried during hikes."},{"location":"0-Workshop/1-Overview/02/#3-product-catalog-info","title":"3. Product Catalog Info","text":"

This record represents a single product item in the product catalog database, with a unique product ID. The products.csv file contains a collection of these records, representing the entire Contoso Outdoors product catalog at a high level.

Each product ID has a corresponding \"product manual\" record that provides more extensive detail (e.g, in website pages). The product catalog entry itself contains just the {id, name, price, category, brand, description} information required for creating product indexes and searching for matching results (for later retrieval) based on a customer query.

The catalog record below corresponds to the product manual record above.

SAMPLE RECORD (CSV) - click to expand 
id = 1,\nname = TrailMaster X4 Tent,\nprice = 250.0,\ncategory = Tents,\nbrand = OutdoorLiving,\ndescription = \"Unveiling the TrailMaster X4 Tent from \\\n    OutdoorLiving, your home away from home for your next \\\n    camping adventure. Crafted from durable polyester, \\\n    this tent boasts a spacious interior perfect for four \\\n    occupants. It ensures your dryness under drizzly skies \\\n    thanks to its water-resistant construction, and the \\\n    accompanying rainfly adds an extra layer of weather \\\n    protection. It offers refreshing airflow and bug defence, \\\n    courtesy of its mesh panels. Accessibility is not an issue \\\n    with its multiple doors and interior pockets that keep \\\n    small items tidy. Reflective guy lines grant better \\\n    visibility at night, and the freestanding design \\\n    simplifies setup and relocation. With the included \\\n    carry bag, transporting this convenient abode becomes \\\n    a breeze. Be it an overnight getaway or a week-long nature \\\n    escapade, the TrailMaster X4 Tent provides comfort, \\\n    convenience, and concord with the great outdoors. Comes with \\\n    a two-year limited warranty to ensure customer satisfaction.\"\n
"},{"location":"0-Workshop/1-Overview/03/","title":"1.4 RAG Chat Tutorial","text":"

This markmap provides the big picture for navigating this RAG Chat tutorial. Learn to build a minimal RAG-based copilot experience using Azure AI Foundry Portal (for setup) and Azure AI Foundry SDK (for ideation and evaluation).

RAG Chat Tutorial Markmap

# RAG Copilot ## 1. Overview ### 1.0 Pre-Requisites - Azure Subscription (Roles) - GitHub Account (Codespaces) - AI Models (Chat, Embedding) - Application Data (RAG) ### 1.1 Concepts - Generative AI Ops (GenAIOps) - Custom Copilot (Chat AI) - Prompt Template (Asset Format) - Retrieval Augmented Generation (RAG) - AI-Assisted Evaluation (LLM-As-Judge) - [Azure OpenAI Deployment types](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/deployment-types#global-standard) - [Data Residency in Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/data-residency/) ### 1.2 Technologies - Azure AI Foundry Portal - Azure AI Project Resource - Azure AI Hub Resource - Azure AI Search Service - Azure OpenAI Service ### 1.3 Dev Tools - GitHub Codespaces (Dev Env) - Visual Studio Code (Dev IDE) - Azure CLI (Configure) --- ## 2. [Setup](https://learn.microsoft.com/en-us/azure/ai-studio/tutorials/copilot-sdk-create-resources) ### 2.1 Create AI Project ### 2.2 Deploy AI Models ### 2.3 Add Azure AI Search ### 2.4 Setup Local Environment ### 2.5 Configure Env Variables ### 2.6 Validate Env Setup ### 2.7 Connect The Dots - What did we do? - Why did we do it? - How can we improve? --- ## 3. [Ideate](https://learn.microsoft.com/en-us/azure/ai-studio/tutorials/copilot-sdk-build-rag) ### 3.1 Add Application Data ### 3.2 Create Search Index ### 3.3 Retrieve Related Products ### 3.4 Extract Customer Intent ### 3.5 Retrieve Related Knowledge ### 3.6 Design Grounded Prompt ### 3.7 Validate the Prototype ### 3.8 Connect The Dots - What did we do? - Why did we do it? - How can we improve? ## 4. [Evaluate](https://learn.microsoft.com/en-us/azure/ai-studio/tutorials/copilot-sdk-evaluate) ### 4.1 Create Evaluation Dataset ### 4.2 Create Evaluation Script ### 4.3 Configure Evaluation Model ### 4.4 Run Evaluation Script ### 4.5 View Results Locally ### 4.6 View Results in Portal ### 4.7 Validate Evaluation ### 4.8 Connect The Dots ## 5. Evolve ### 5.1 Recap: Build a Copilot ### 5.2 Refactor: Make it Better ### 5.3 Resources: Learn More"},{"location":"0-Workshop/1-Overview/04/","title":"1.5 Contoso Chat Sample","text":"

Once you've familiarized yourself with the basic setup, ideation and evaluation capabilities, you can explore the Contoso Chat sample for more advanced insights, on your own.

This sample uses the same retail dataset used in this RAG Chat workshop, but streamlines the end-to-end development workflow with core developer tools like Azure Developer CLI, to automate provisioning and deployment. The sample also teaches you how to package up your application prototype and deploy it into production using Azure Container Apps.

Contoso Chat is currently in v3. An updated v4 (Jan 2025) will reflect recent Azure AI SDK updates

"},{"location":"0-Workshop/2-Setup/01/","title":"2.1 Create AI Project","text":"

This is Part 1 of the tutorial. This stage is completed USING THE AZURE AI FOUNDRY PORTAL

At the end of this section, you should have provisioned an Azure AI Hub and Azure AI project resource, setup an Azure AI Search resource and deployed two Azure OpenAI models for implementing the RAG-based copilot. You should also have launched GitHub Codespaces and configured your development environment to work with your provisioned Azure infrastructure.

"},{"location":"0-Workshop/2-Setup/01/#1-log-into-azure-ai-foundry","title":"1. Log Into Azure AI Foundry","text":"
  1. Open a private browser and navigate to https://ai.azure.com.
  2. Log in with an active Azure subscription. Note the tenant ID if multi-tenant.
  3. You should see a landing page with a blue \"+ Create project\" button as shown below.

Note: If you had previously created projects, those will be listed as shown above. Don't worry if you don't see any listed for your profile. We are going to create a new project next.

"},{"location":"0-Workshop/2-Setup/01/#2-create-a-new-project","title":"2. Create a new project","text":"

  1. Click the \"+ Create project\" button. You should see a dialog popup like this. Your default project name and hub information will be different and reflect your prior activity.

  2. Change the default project name to something memorable - I used ninarasi-ragchat-v1.

  3. We also want to create a new hub for our new AI project - let's fix that next in the dialog.
"},{"location":"0-Workshop/2-Setup/01/#3-create-new-hub","title":"3. Create new hub","text":"
  1. Click the Create new hub (blue lettering) in dialog above.
  2. Pick a memorable name that reflects the project - I used ninarasi-ragchat-hub
  3. Click \"Next\". It returns you to the previous dialog, with an enhanced view as shown
"},{"location":"0-Workshop/2-Setup/01/#4-customize-the-hub","title":"4. Customize the hub","text":"
  1. Click the \"Customize\" button in that dialog. This lets you customize the defaults as shown.
  2. First, select a relevant Location with relevant model quota - I used East US2
  3. Next,customize the resource group name to be memorable - I used ninarasi-ragchat-rg
  4. Next, click \"Create new AI Search\" (blue lettering) in the dialog to trigger a new pop-up
"},{"location":"0-Workshop/2-Setup/01/#5-create-new-ai-search","title":"5. Create new AI Search","text":"
  1. Customize resource name - I used ninarasi-ragchat-aisearch - then hit Next.
  2. You return to the Create a project wizard - hit Next to get to review.
  3. Review the details one last time - hit Create to confirm AI project creation.
  4. Creation takes a few minutes - all elements will show green on success.
"},{"location":"0-Workshop/2-Setup/01/#6-review-created-ai-project","title":"6. Review Created AI Project","text":"
  1. You should automatically be taken to the AI Project overview page as shown. Note the Project connection string under Project details - we'll revisit it later.
  2. Click on the Open in management center link (highlighted in red) - it takes you to this Management Center view. Clicking Go to project will take you back to the AI project.
  3. However, for now click on the Connected Resources option in the sidebar. This lets us see which resources can be accessed via the Project connection we noted earlier. Verify that Azure AI Search is one of the listed resources.

CONGRATULATIONS! You created your Azure AI Hub & Project resources

"},{"location":"0-Workshop/2-Setup/02/","title":"2.2 Deploy AI Models","text":"

Let's revisit our Retrieval Augmented Generation design pattern. Note that we make use of two models to implement this design architecture.

  1. A Large Language Model (Embedding) for vectorizing the user query.
  2. A Large Language Model (Chat) for generating the response returned to user.

Let's find the right models to use and deploy them to our Azure AI project so we can use them in our RAG-based copilot implementation. Start by navigating to the Azure AI Project overview page. Then select the Models + endpoints link under My assets.

"},{"location":"0-Workshop/2-Setup/02/#1-deploy-chat-model","title":"1. Deploy Chat Model","text":""},{"location":"0-Workshop/2-Setup/02/#11-select-a-chat-model","title":"1.1. Select A Chat Model","text":"

  1. Click the blue \"Deploy model\" button and pick Deploy base model from the dropdown options.

  2. If you know the specific model to use, you can search for it here. In our case, let's look at what our options are. First, click the Collections filter and select Azure OpenAI. Next, click the Inference tasks filter and select Chat completion. We can see that this reduces our choices from 1800+ models in the Azure AI model catalog to 9 matching models.

"},{"location":"0-Workshop/2-Setup/02/#12-deploy-the-chat-model","title":"1.2. Deploy the Chat Model","text":"
  1. We can pick any of those options as shown above, to see a Model Card with more details. Let's pick gpt-4o-mini and click Confirm to get this deployment wizard. Note that it selects a Global Standard deployment type by default. The deployment has a default capacity of 10K tokens per minute (TPM) which can be useful when we begin the evaluation phase.
  1. Click on the dropdown to see other Deployment type options. Read the documentation to learn more about what each provides. Global Standard is the recommended starting place for customers so let's go with that.
"},{"location":"0-Workshop/2-Setup/02/#13-verify-deployment","title":"1.3. Verify Deployment","text":"

On successful deployment, you will be taken to the model deployment page where you can review the details and retrieve relevant information like the Endpoint and Key information, for use with code-first clients.

You can Open in playground and use the Azure AI Portal as an ideation tool to explore different prompt templates, model configurations and multi-turn conversational approaches to determine if this model is in fact suitable for your app scenario.

Task: Ask the model to Tell me about hiking boots for my trip to Spain - is response grounded?

Homework: Complete the Deploy an enterprise chat app tutorial with your data. Is response grounded now?

"},{"location":"0-Workshop/2-Setup/02/#14-view-metrics","title":"1.4. View Metrics","text":"

You can also select the Metrics tab of deployed models within an Azure AI project to get metrics about the cost (tokens) and performance (requests) of that model in a given time frame. The screenshot shows the data taken for this model after completing the entire project. The spike reflects the requests made during the evaluation stage of the workflow.

You can also click the Open in Azure Monitor link to open up the Azure Monitor dashboard in the Azure Portal as shown below, allowing you to drill down into various metrics or establish dashboards to monitor trends of interest.

"},{"location":"0-Workshop/2-Setup/02/#2-deploy-embedding-model","title":"2. Deploy Embedding Model","text":"

The previous steps focused on the chat completion model which has many choices for us to select from. Now, let's look at embeddings.

"},{"location":"0-Workshop/2-Setup/02/#21-select-deploy-model","title":"2.1. Select & Deploy Model","text":"

Start by setting the Inference Task to Embeddings.

You'll find there are only about 11 models that match this filter - setting the Collection to Azure OpenAI reduces this further to 5. As before, let's select a model and review the card to see if it matches our needs.

Then Confirm to get the Deployment wizard dialog.

And Deploy to complete the workflow.

"},{"location":"0-Workshop/2-Setup/02/#23-verify-deployment","title":"2.3. Verify Deployment","text":"

Similarly, we can use the deployment card to view deployment details and explore metrics. But note that we don't have a Playground for embeddings.

"},{"location":"0-Workshop/2-Setup/02/#3-deployment-complete","title":"3. Deployment Complete","text":"

The Azure AI Project overview page will now list both models under the Models + endpoints tab for easy lookup later (if you want to explore metrics or engage in Playground).

CONGRATULATIONS! You deployed both the AI models needed for RAG

"},{"location":"0-Workshop/2-Setup/03/","title":"2.3 Add Azure AI Search","text":"

The official tutorial recommends setting up an Azure AI Search resource at this stage. This was done under the assumption that the default Azure AI project setup did not create (new) or reuse (existing) Azure AI Search resources.

However, since we opted to add Azure AI Search during project setup, we have nothing further to do at this stage! Note that the search resource is not yet populated with our data (search indexes). We'll get there in the Ideate section.

CONGRATULATIONS! You completed setup of Azure AI Search indexes for your data

"},{"location":"0-Workshop/2-Setup/04/","title":"2.4 Add App Insights","text":""},{"location":"0-Workshop/2-Setup/04/#tracing-observability","title":"Tracing & Observability","text":"

While not covered explicitly in the tutorial, we will be working with code in the Ideate section that is instrumented for observability (tracing).

  • See: How to Trace your application with Azure AI Inference SDK

In order to visualize those traces in the Azure AI Foundry Portal, we need to attach an Application Insights resource to our Azure AI project ahead of time.

  • See: View your traces in Azure AI Foundry Portal
"},{"location":"0-Workshop/2-Setup/04/#create-new-app-insights","title":"Create New App Insights","text":"

Let's follow those steps (as illustrated in the animated gif below):

  1. Navigate to your Azure AI Project resource in the Azure AI Foundry portal
  2. Select the Tracing option in the menu sidebar
  3. Select Create New to attach a new Application Insights resource to the project
  4. Provide a name and select Create.

CONGRATULATIONS! You activated App Insights for tracing your Azure AI project

"},{"location":"0-Workshop/2-Setup/05/","title":"2.5 Setup Local Environment","text":"

The previous steps completed the setup of our Azure AI infrastructure (resources). Now it's time to setup our development environment to talk to our Azure backend.

"},{"location":"0-Workshop/2-Setup/05/#1-launch-github-codespaces","title":"1. Launch GitHub Codespaces","text":"

If you had not previously done so, complete the Getting Started steps now.

  1. Fork this repository to your personal GitHub profile.
  2. Open the fork in a new browser tab.
  3. Click on the blue \"Code\" button and select Codespaces
  4. Click on the Create Codespaces on Main button

You should see GitHub Codespaces launch in a new browser tab.

  • It will take a few minutes to complete loading.
  • You should see a Visual Studio Code IDE in the browser
  • When ready, you should see a VS Code terminal with active prompt.
"},{"location":"0-Workshop/2-Setup/05/#2-verify-azure-cli-installed","title":"2. Verify Azure CLI Installed","text":"

The repository is configured with a devcontainer that has all necessary dependencies pre-installed. Let's verify the az (Azure Developer CLI) was installed.

az version\n
"},{"location":"0-Workshop/2-Setup/05/#3-authenticate-with-azure-cli","title":"3. Authenticate with Azure CLI","text":"

Log into your Azure subscription from the VS Code terminal in GitHub Codespaces using the following command, and follow the prompts to complete the workflow.

az login --use-device-code\n

If you have a multi-tenancy account, you can set the default tenant when logging in as follows, where <TENANTID> is replaced with the relevant identifier.

az login --use-device-code --tenant <TENANTID>\n
"},{"location":"0-Workshop/2-Setup/05/#4-verify-python-packages","title":"4. Verify Python Packages","text":"

The codebase is set up with a requirements.txt file that has all the necesary Python package dependencies listed. These are auto-installed into the devcontainer at launch. Use pip list | grep <KEYWORD> to verify if specific packages were installed.

For instance use this command to list azure packages installed and verify they match the ones listed in requirements (e.g., look for azure-ai-projects, azure-ai-inference, azure-ai-identity, azure-search-documents, azure-core, azure-ai-evaluation)

pip list | grep azure\n
"},{"location":"0-Workshop/2-Setup/06/","title":"2.6 Setup Project Structure","text":"

This repository contains the following structure to start with. The *.sample folders or files are there for reference only, so you can check your work.

data/            # Contains application data (initial)\ndocs/            # Contains docs and guides (content)\nsrc.sample/      # Sample src/ folder\n.env.sample       # Sample .env file\n
"},{"location":"0-Workshop/2-Setup/06/#1-define-src-folder-for-code","title":"1. Define src/ folder for code","text":"

In this workshop, start by creating a new src/ folder and populating it from scratch to get a sense for the development workflow. Start by creating this folder structure:

mkdir src/\nmkdir src/api\nmkdir src/api/assets\n

Your directory structure should now look like this:

data/\ndocs/\nsrc.sample/\n.env.sample\nsrc/\nsrc/api\nsrc/api/assets\n
"},{"location":"0-Workshop/2-Setup/06/#2-add-srcconfigpy-helper-script","title":"2. Add src/config.py helper script","text":"

For convenience, let's copy this from the sample location - then review the code to see what it does. Use this command at the root of the repo:

cp src.sample/api/config.py src/api/.\n

Expand the code below to get a sense of what this helper does.

  1. Sets the ASSET_PATH to the assets/ folder in the same directory.
  2. Configures the app logger and enables telemetry logging (traces) for app.
Click to expand and view the helper script src/api/config.py
# ruff: noqa: ANN201, ANN001\nimport os\nimport sys\nimport pathlib\nimport logging\nfrom azure.identity import DefaultAzureCredential\nfrom azure.ai.projects import AIProjectClient\nfrom azure.ai.inference.tracing import AIInferenceInstrumentor\n\n# load environment variables from the .env file\nfrom dotenv import load_dotenv\n\nload_dotenv()\n\n# Set \"./assets\" as the path where assets are stored, resolving the absolute path:\nASSET_PATH = pathlib.Path(__file__).parent.resolve() / \"assets\"\n\n# Configure an root app logger that prints info level logs to stdout\nlogger = logging.getLogger(\"app\")\nlogger.setLevel(logging.INFO)\nlogger.addHandler(logging.StreamHandler(stream=sys.stdout))\n\n\n# Returns a module-specific logger, inheriting from the root app logger\ndef get_logger(module_name):\n    return logging.getLogger(f\"app.{module_name}\")\n\n\n# Enable instrumentation and logging of telemetry to the project\ndef enable_telemetry(log_to_project: bool = False):\n    AIInferenceInstrumentor().instrument()\n\n    # enable logging message contents\n    os.environ[\"AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED\"] = \"true\"\n\n    if log_to_project:\n        from azure.monitor.opentelemetry import configure_azure_monitor\n\n        project = AIProjectClient.from_connection_string(\n            conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n        )\n        tracing_link = f\"https://ai.azure.com/tracing?wsid=/subscriptions/{project.scope['subscription_id']}/resourceGroups/{project.scope['resource_group_name']}/providers/Microsoft.MachineLearningServices/workspaces/{project.scope['project_name']}\"\n        application_insights_connection_string = project.telemetry.get_connection_string()\n        if not application_insights_connection_string:\n            logger.warning(\n                \"No application insights configured, telemetry will not be logged to project. Add application insights at:\"\n            )\n            logger.warning(tracing_link)\n\n            return\n\n        configure_azure_monitor(connection_string=application_insights_connection_string)\n        logger.info(\"Enabled telemetry logging to project, view traces at:\")\n        logger.info(tracing_link)\n

CONGRATULATIONS! Your development environment is ready to use.

"},{"location":"0-Workshop/2-Setup/07/","title":"2.7 Configure Env Variables","text":"

We are now ready to start coding the chat AI application in our local development environment. But to do this, we need to configure a few environment variables.

"},{"location":"0-Workshop/2-Setup/07/#1-create-env-file","title":"1. Create .env file","text":"
  1. Start by copying the .env.sample file to .env
cp .env.sample .env\n
  1. Let's review what this contains
cat .env\n

You will see something like this:

AIPROJECT_CONNECTION_STRING=<your-connection-string>\nAISEARCH_INDEX_NAME=\"contoso-products\"\nEMBEDDINGS_MODEL=\"text-embedding-ada-002\"\nINTENT_MAPPING_MODEL=\"gpt-4o-mini\"\nCHAT_MODEL=\"gpt-4o-mini\"\nEVALUATION_MODEL=\"gpt-4o-mini\"\n
"},{"location":"0-Workshop/2-Setup/07/#2-update-connection-string","title":"2. Update Connection String","text":"

Note that defaults are provided for everything except the AIPROJECT_CONNECTION_STRING - let's fix that now!

  1. Open the Azure AI Project overview page. It should look like this:

  2. Look for the Project connection string under the Project details tab.

  3. Copy that into .env as the AIPROJECT_CONNECTION_STRING value.
  4. Save the changes to .env
"},{"location":"0-Workshop/2-Setup/07/#3-review-environment-variables","title":"3. Review Environment Variables","text":"

Let's review our environment variables:

  1. AIPROJECT_CONNECTION_STRING - is a single connection URI that allows access to all the Connected Resources in the Azure AI project (including Azure AI Search).
  2. AISEARCH_INDEX_NAME - is set to contoso-products and represents the index name that we will create and populate with product catalog data.
  3. EMBEDDINGS_MODEL - the deployed model we'll use for vectorizing queries (see: Ideate 3.2)
  4. INTENT_MAPPING_MODEL - the deployed model we'll use for intent mapping (see: Ideate 3.4)
  5. CHAT_MODEL - the deployed model we'll use for final chat response (see: Ideate 3.6)
  6. EVALUATION_MODEL - the deployed model we'll use for quality evaluation (see: Evaluate 4.3)

CONGRATULATIONS! Your local environment is configured for code!

"},{"location":"0-Workshop/3-Ideate/01/","title":"3.1 Add Application Data","text":"

This is Part 2 of the tutorial. This stage is completed USING THE AZURE AI FOUNDRY SDK.

At the end of this section, you should have created an Azure AI Search index based on the retailer's product data, written and tested scripts to retrieve relevant product documents based on a user query, and implemented a chat AI that uses this knowledge as grounding context for a chat completion model. You will also get your first look at observability support with tracing.

"},{"location":"0-Workshop/3-Ideate/01/#1-add-productscsv-to-assets","title":"1. Add products.csv to assets/","text":"

We want to ground chat AI responses in our application data. In this case, we want to respond to customer queries about our products by grounding the responses in items found in our catalog.

HOMEWORK: Think of other retail data sources that might be useful (and how to add them)

Recall that the src/config.py script identifies the assets/ folder as the source for all static assets. We already created the src/api/assets earlier. Let's copy in the product catalog data into this folder now.

cp src.sample/api/assets/products.csv  src/api/assets/\n
"},{"location":"0-Workshop/3-Ideate/01/#2-inspect-productscsv-data","title":"2. Inspect products.csv data","text":"

Let's take a quick look at what this file contains in terms of product catalog data.

  1. We have 20 product records listed in CSV format.
  2. Each product has an id, name, price, category, brand, and text description
  3. The product description provides the most content about the product
  4. The product name is a unique title for that product.
Click to expand and view the Product Catalog data products.csv
id,name,price,category,brand,description\n1,TrailMaster X4 Tent,250.0,Tents,OutdoorLiving,\"Unveiling the TrailMaster X4 Tent from OutdoorLiving, your home away from home for your next camping adventure. Crafted from durable polyester, this tent boasts a spacious interior perfect for four occupants. It ensures your dryness under drizzly skies thanks to its water-resistant construction, and the accompanying rainfly adds an extra layer of weather protection. It offers refreshing airflow and bug defence, courtesy of its mesh panels. Accessibility is not an issue with its multiple doors and interior pockets that keep small items tidy. Reflective guy lines grant better visibility at night, and the freestanding design simplifies setup and relocation. With the included carry bag, transporting this convenient abode becomes a breeze. Be it an overnight getaway or a week-long nature escapade, the TrailMaster X4 Tent provides comfort, convenience, and concord with the great outdoors. Comes with a two-year limited warranty to ensure customer satisfaction.\"\n2,Adventurer Pro Backpack,90.0,Backpacks,HikeMate,\"Venture into the wilderness with the HikeMate's Adventurer Pro Backpack! Uniquely designed with ergonomic comfort in mind, this backpack ensures a steadfast journey no matter the mileage. It boasts a generous 40L capacity wrapped up in durable nylon fabric ensuring its long-lasting performance on even the most rugged pursuits. It's meticulously fashioned with multiple compartments and pockets for organized storage, hydration system compatibility, and adjustable padded shoulder straps all in a lightweight construction. The added features of a sternum strap and hip belt enhance stability without compromising on comfort. The Adventurer Pro Backpack also prioritizes your safety with its reflective accents for when night falls. This buoyant beauty does more than carry your essentials; it carries the promise of a stress-free adventure!\"\n3,Summit Breeze Jacket,120.0,Hiking Clothing,MountainStyle,\"Discover the joy of hiking with MountainStyle's Summit Breeze Jacket. This lightweight jacket is your perfect companion for outdoor adventures. Sporting a trail-ready, windproof design and a water-resistant fabric, it's ready to withstand any weather. The breathable polyester material and adjustable cuffs keep you comfortable, whether you're ascending a mountain or strolling through a park. And its sleek black color adds style to function. The jacket features a full-zip front closure, adjustable hood, and secure zippered pockets. Experience the comfort of its inner lining and the convenience of its packable design. Crafted for night trekkers too, the jacket has reflective accents for enhanced visibility. Rugged yet chic, the Summit Breeze Jacket is more than a hiking essential, it's the gear that inspires you to reach new heights. Choose adventure, choose the Summit Breeze Jacket.\"\n4,TrekReady Hiking Boots,140.0,Hiking Footwear,TrekReady,\"Introducing the TrekReady Hiking Boots - stepping up your hiking game, one footprint at a time! Crafted from leather, these stylistic Trailmates are made to last. TrekReady infuses durability with its reinforced stitching and toe protection, making sure your journey is never stopped short. Comfort? They have that covered too! The boots are a haven with their breathable materials, cushioned insole, with padded collar and tongue; all nestled neatly within their lightweight design. As they say, it's what's inside that counts - so inside you'll find a moisture-wicking lining that quarantines stank and keeps your feet fresh as that mountaintop breeze. Remember the fear of slippery surfaces? With these boots, you can finally tell it to 'take a hike'! Their shock-absorbing midsoles and excellent traction capabilities promise stability at your every step. Beautifully finished in a traditional lace-up system, every adventurer deserves a pair of TrekReady Hiking Boots. Hike more, worry less!\"\n5,BaseCamp Folding Table,60.0,Camping Tables,CampBuddy,\"CampBuddy's BaseCamp Folding Table is an adventurer's best friend. Lightweight yet powerful, the table is a testament to fun-meets-function and will elevate any outing to new heights. Crafted from resilient, rust-resistant aluminum, the table boasts a generously sized 48 x 24 inches tabletop, perfect for meal times, games and more. The foldable design is a godsend for on-the-go explorers. Adjustable legs rise to the occasion to conquer uneven terrains and offer height versatility, while the built-in handle simplifies transportation. Additional features like non-slip feet, integrated cup holders and mesh pockets add a pinch of finesse. Quick to set up without the need for extra tools, this table is a silent yet indispensable sidekick during camping, picnics, and other outdoor events. Don't miss out on the opportunity to take your outdoor experiences to a new level with the BaseCamp Folding Table. Get yours today and embark on new adventures tomorrow! \"\n6,EcoFire Camping Stove,80.0,Camping Stoves,EcoFire,\"Introducing EcoFire's Camping Stove, your ultimate companion for every outdoor adventure! This portable wonder is precision-engineered with a lightweight and compact design, perfect for capturing that spirit of wanderlust. Made from high-quality stainless steel, it promises durability and steadfast performance. This stove is not only fuel-efficient but also offers an easy, intuitive operation that ensures hassle-free cooking. Plus, it's flexible, accommodating a variety of cooking methods whether you're boiling, grilling, or simmering under the starry sky. Its stable construction, quick setup, and adjustable flame control make cooking a breeze, while safety features protect you from any potential mishaps. And did we mention it also includes an effective wind protector and a carry case for easy transportation? But that's not all! The EcoFire Camping Stove is eco-friendly, designed to minimize environmental impact. So get ready to enhance your camping experience and enjoy delicious outdoor feasts with this unique, versatile stove!\"\n7,CozyNights Sleeping Bag,100.0,Sleeping Bags,CozyNights,\"Embrace the great outdoors in any season with the lightweight CozyNights Sleeping Bag! This durable three-season bag is superbly designed to give hikers, campers, and backpackers comfort and warmth during spring, summer, and fall. With a compact design that folds down into a convenient stuff sack, you can whisk it away on any adventure without a hitch. The sleeping bag takes comfort seriously, featuring a handy hood, ample room and padding, and a reliable temperature rating. Crafted from high-quality polyester, it ensures long-lasting use and can even be zipped together with another bag for shared comfort. Whether you're gazing at stars or catching a quick nap between trails, the CozyNights Sleeping Bag makes it a treat. Don't just sleep\u2014 dream with CozyNights.\"\n8,Alpine Explorer Tent,350.0,Tents,AlpineGear,\"Welcome to the joy of camping with the Alpine Explorer Tent! This robust, 8-person, 3-season marvel is from the responsible hands of the AlpineGear brand. Promising an enviable setup that is as straightforward as counting sheep, your camping experience is transformed into a breezy pastime. Looking for privacy? The detachable divider provides separate spaces at a moment's notice. Love a tent that breathes? The numerous mesh windows and adjustable vents fend off any condensation dragon trying to dampen your adventure fun. The waterproof assurance keeps you worry-free during unexpected rain dances. With a built-in gear loft to stash away your outdoor essentials, the Alpine Explorer Tent emerges as a smooth balance of privacy, comfort, and convenience. Simply put, this tent isn't just a shelter - it's your second home in the heart of nature! Whether you're a seasoned camper or a nature-loving novice, this tent makes exploring the outdoors a joyous journey.\"\n9,SummitClimber Backpack,120.0,Backpacks,HikeMate,\"Adventure waits for no one! Introducing the HikeMate SummitClimber Backpack, your reliable partner for every exhilarating journey. With a generous 60-liter capacity and multiple compartments and pockets, packing is a breeze. Every feature points to comfort and convenience; the ergonomic design and adjustable hip belt ensure a pleasantly personalized fit, while padded shoulder straps protect you from the burden of carrying. Venturing into wet weather? Fear not! The integrated rain cover has your back, literally. Stay hydrated thanks to the backpack's hydration system compatibility. Travelling during twilight? Reflective accents keep you visible in low-light conditions. The SummitClimber Backpack isn't merely a carrier; it's a wearable base camp constructed from ruggedly durable nylon and thoughtfully designed for the great outdoors adventurer, promising to withstand tough conditions and provide years of service. So, set off on that quest - the wild beckons! The SummitClimber Backpack - your hearty companion on every expedition!\"\n10,TrailBlaze Hiking Pants,75.0,Hiking Clothing,MountainStyle,\"Meet the TrailBlaze Hiking Pants from MountainStyle, the stylish khaki champions of the trails. These are not just pants; they're your passport to outdoor adventure. Crafted from high-quality nylon fabric, these dapper troopers are lightweight and fast-drying, with a water-resistant armor that laughs off light rain. Their breathable design whisks away sweat while their articulated knees grant you the flexibility of a mountain goat. Zippered pockets guard your essentials, making them a hiker's best ally. Designed with durability for all your trekking trials, these pants come with a comfortable, ergonomic fit that will make you forget you're wearing them. Sneak a peek, and you are sure to be tempted by the sleek allure that is the TrailBlaze Hiking Pants. Your outdoors wardrobe wouldn't be quite complete without them.\"\n11,TrailWalker Hiking Shoes,110.0,Hiking Footwear,TrekReady,\"Meet the TrekReady TrailWalker Hiking Shoes, the ideal companion for all your outdoor adventures. Constructed with synthetic leather and breathable mesh, these shoes are tough as nails yet surprisingly airy. Their cushioned insoles offer fabulous comfort for long hikes, while the supportive midsoles and traction outsoles with multidirectional lugs ensure stability and excellent grip. A quick-lace system, padded collar and tongue, and reflective accents make these shoes a dream to wear. From combating rough terrain with the reinforced toe cap and heel, to keeping off trail debris with the protective mudguard, the TrailWalker Hiking Shoes have you covered. These waterproof warriors are made to endure all weather conditions. But they're not just about being rugged, they're light as a feather too, minimizing fatigue during epic hikes. Each pair can be customized for a perfect fit with removable insoles and availability in multiple sizes and widths. Navigate hikes comfortably and confidently with the TrailWalker Hiking Shoes. Adventure, here you come!\"\n12,TrekMaster Camping Chair,50.0,Camping Tables,CampBuddy,\"Gravitate towards comfort with the TrekMaster Camping Chair from CampBuddy. This trusty outdoor companion boasts sturdy construction using high-quality materials that promise durability and enjoyment for seasons to come. Impeccably lightweight and portable, it's designed to be your go-to seat whether you're camping, at a picnic, cheering at a sporting event, or simply relishing in your backyard pleasures. Beyond its foldable design ensuring compact storage and easy transportation, its ergonomic magic is in the details. An adjustable recline, padded seat and backrest, integrated cup holder, and side pockets ensure the greatest outdoor comfort. Weather resistant, easy to clean, and capable of supporting diverse body types, this versatile chair also comes with a carry bag, ready for your next adventure.\"\n13,PowerBurner Camping Stove,100.0,Camping Stoves,PowerBurner,\"Unleash your inner explorer with the PowerBurner Dual Burner Camping Stove. It's designed for the adventurous heart, with sturdy construction and a high heat output that makes boiling and cooking a breeze. This stove isn't just about strength\u2014it's got finesse too. With adjustable flame control, you can simmer, saut\u00e9, or sizzle with absolute precision. Its compact design and integrated carrying handle make transportation effortless. Moreover, it's crafted to defy the elements, boasting a wind-resistant exterior and piezo ignition system for quick, reliable starts. And when the cooking's done, its removable grates make cleanup swift and easy. Rugged, versatile and reliable, the PowerBurner marks a perfect blend of practicality and performance. So, why wait? Let's turn up the heat on your outdoor culinary adventures today.\"\n14,MountainDream Sleeping Bag,130.0,Sleeping Bags,MountainDream,\"Meet the MountainDream Sleeping Bag: your new must-have companion for every outdoor adventure. Designed to handle 3-season camping with ease, it comes equipped with a premium synthetic insulation that will keep you cozy even when temperatures fall down to 15\u00b0F! Sporting a durable water-resistant nylon shell and soft breathable polyester lining, this bag doesn't sacrifice comfort for toughness. The star of the show is the contoured mummy shape that not only provides optimal heat retention but also cuts down on the weight. A smooth, snag-free YKK zipper with a unique anti-snag design allows for hassle-free operation, while the adjustable hood and full-length zipper baffle work together to ensure you stay warm all night long. Need to bring along some essentials? Not to worry! There's an interior pocket just for that. And when it's time to pack up? Just slip it into the included compression sack for easy storage and transport. Whether you're a backpacking pro or a camping novice, the MountainDream Sleeping Bag is the perfect blend of durability, warmth, and comfort that you've been looking for.\"\n15,SkyView 2-Person Tent,200.0,Tents,OutdoorLiving,\"Introducing the OutdoorLiving SkyView 2-Person Tent, a perfect companion for your camping and hiking adventures. This tent offers a spacious interior that houses two people comfortably, with room to spare. Crafted from durable waterproof materials to shield you from the elements, it is the fortress you need in the wild. Setup is a breeze thanks to its intuitive design and color-coded poles, while two large doors allow for easy access. Stay organized with interior pockets, and store additional gear in its two vestibules. The tent also features mesh panels for effective ventilation, and it comes with a rainfly for extra weather protection. Light enough for on-the-go adventurers, it packs compactly into a carrying bag for seamless transportation. Reflective guy lines ensure visibility at night for added safety, and the tent stands freely for versatile placement. Experience the reliability of double-stitched seams that guarantee increased durability, and rest easy under the stars with OutdoorLiving's SkyView 2-Person Tent. It's not just a tent; it's your home away from home.\"\n16,TrailLite Daypack,60.0,Backpacks,HikeMate,\"Step up your hiking game with HikeMate's TrailLite Daypack. Built for comfort and efficiency, this lightweight and durable backpack offers a spacious main compartment, multiple pockets, and organization-friendly features all in one sleek package. The adjustable shoulder straps and padded back panel ensure optimal comfort during those long exhilarating treks. Course through nature without worry as the daypack's water-resistant fabric protects your essentials from unexpected showers. Plus, never run dry with the integrated hydration system. And did we mention it comes in a plethora of colors and designs? So you can choose one that truly speaks to your outdoorsy soul! Keeping your visibility in mind, we've added reflective accents that light up in low-light conditions. Don't just carry a backpack, adorn a companion that takes you a step ahead in your adventures. Trust the TrailLite Daypack for a hassle-free, enjoyable hiking experience.\"\n17,RainGuard Hiking Jacket,110.0,Hiking Clothing,MountainStyle,\"Introducing the MountainStyle RainGuard Hiking Jacket - the ultimate solution for weatherproof comfort during your outdoor undertakings! Designed with waterproof, breathable fabric, this jacket promises an outdoor experience that's as dry as it is comfortable. The rugged construction assures durability, while the adjustable hood provides a customizable fit against wind and rain. Featuring multiple pockets for safe, convenient storage and adjustable cuffs and hem, you can tailor the jacket to suit your needs on-the-go. And, don't worry about overheating during intense activities - it's equipped with ventilation zippers for increased airflow. Reflective details ensure visibility even during low-light conditions, making it perfect for evening treks. With its lightweight, packable design, carrying it inside your backpack requires minimal effort. With options for men and women, the RainGuard Hiking Jacket is perfect for hiking, camping, trekking and countless other outdoor adventures. Don't let the weather stand in your way - embrace the outdoors with MountainStyle RainGuard Hiking Jacket!\"\n18,TrekStar Hiking Sandals,70.0,Hiking Footwear,TrekReady,\"Meet the TrekStar Hiking Sandals from TrekReady - the ultimate trail companion for your feet. Designed for comfort and durability, these lightweight sandals are perfect for those who prefer to see the world from a hiking trail. They feature adjustable straps for a snug, secure fit, perfect for adapting to the contours of your feet. With a breathable design, your feet will stay cool and dry, escaping the discomfort of sweaty hiking boots on long summer treks. The deep tread rubber outsole ensures excellent traction on any terrain, while the cushioned footbed promises enhanced comfort with every step. For those wild and unpredictable trails, the added toe protection and shock-absorbing midsole protect your feet from rocky surprises. Ingeniously, the removable insole makes for easy cleaning and maintenance, extending the lifespan of your sandals. Available in various sizes and a handsome brown color, the versatile TrekStar Hiking Sandals are just as comfortable on a casual walk in the park as they are navigating rocky slopes. Explore more with TrekReady!\"\n19,Adventure Dining Table,90.0,Camping Tables,CampBuddy,\"Discover the joy of outdoor adventures with the CampBuddy Adventure Dining Table. This feature-packed camping essential brings both comfort and convenience to your memorable trips. Made from high-quality aluminum, it promises long-lasting performance, weather resistance, and easy maintenance - all key for the great outdoors! It's light, portable, and comes with adjustable height settings to suit various seating arrangements and the spacious surface comfortably accommodates meals, drinks, and other essentials. The sturdy yet lightweight frame holds food, dishes, and utensils with ease. When it's time to pack up, it fold and stows away with no fuss, ready for the next adventure!  Perfect for camping, picnics, barbecues, and beach outings - its versatility shines as brightly as the summer sun! Durable, sturdy and a breeze to set up, the Adventure Dining Table will be a loyal companion on every trip. Embark on your next adventure and make lifetime memories with CampBuddy. As with all good experiences, it'll leave you wanting more! \"\n20,CompactCook Camping Stove,60.0,Camping Stoves,CompactCook,\"Step into the great outdoors with the CompactCook Camping Stove, a convenient, lightweight companion perfect for all your culinary camping needs. Boasting a robust design built for harsh environments, you can whip up meals anytime, anywhere. Its wind-resistant and fuel-versatile features coupled with an efficient cooking performance, ensures you won't have to worry about the elements or helpless taste buds while on adventures. The easy ignition technology and adjustable flame control make cooking as easy as a walk in the park, while its compact, foldable design makes packing a breeze. Whether you're camping with family or hiking solo, this reliable, portable stove is an essential addition to your gear. With its sturdy construction and safety-focused design, the CompactCook Camping Stove is a step above the rest, providing durability, quality, and peace of mind. Be wild, be free, be cooked for with the CompactCook Camping Stove!\"\n
"},{"location":"0-Workshop/3-Ideate/02/","title":"3.2 Create Search Index","text":""},{"location":"0-Workshop/3-Ideate/02/#1-create-search-index-script","title":"1. Create Search Index Script","text":"

Let's copy over the create-search-index.py script into our application source folder.

cp src.sample/api/create-search-index.py  src/api/.\n
"},{"location":"0-Workshop/3-Ideate/02/#2-understand-index-creation","title":"2. Understand Index Creation","text":"

Now, let's take a look at what this does.

Click to expand and view the Python Script to create the search index src/api/create-search-index.py
    import os\n    from azure.ai.projects import AIProjectClient\n    from azure.ai.projects.models import ConnectionType\n    from azure.identity import DefaultAzureCredential\n    from azure.core.credentials import AzureKeyCredential\n    from azure.search.documents import SearchClient\n    from azure.search.documents.indexes import SearchIndexClient\n    from config import get_logger\n\n    # initialize logging object\n    logger = get_logger(__name__)\n\n    # create a project client using environment variables loaded from the .env file\n    project = AIProjectClient.from_connection_string(\n        conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n    )\n\n    # create a vector embeddings client that will be used to generate vector embeddings\n    embeddings = project.inference.get_embeddings_client()\n\n    # use the project client to get the default search connection\n    search_connection = project.connections.get_default(\n        connection_type=ConnectionType.AZURE_AI_SEARCH, include_credentials=True\n    )\n\n    # Create a search index client using the search connection\n    # This client will be used to create and delete search indexes\n    index_client = SearchIndexClient(\n        endpoint=search_connection.endpoint_url, credential=AzureKeyCredential(key=search_connection.key)\n    )\n

First the script sets up a search index_client:

  1. Creates an Azure AI Project Client instance (configured with connection string)
  2. Retrieves an embeddings inference client from the AI project (maps to that model)
  3. Retrieves a search_connection object from the AI project instance
  4. Creates an index_client search index client using the search connection (key, endpoint)

First it defines the index based on a vector derived from product data fields.

  1. It maps product name to a title property
  2. It maps product description to a content property
  3. It uses HNSW algorithm (cosine distance) for similarity
  4. It prioritizes \"content\" for semantic ranking

It then creates the index from CSV and populates it using the index_client.

  1. It defines an index using the specified name and embeddings model
  2. It loads CSV and generates vector embeddings for each description
  3. It uploads each vectorized document into the pre-defined search index
"},{"location":"0-Workshop/3-Ideate/02/#3-run-index-creation-script","title":"3. Run Index Creation Script","text":"

To get the index created in Azure AI Search, run the script described above.

python create_search_index.py\n
"},{"location":"0-Workshop/3-Ideate/02/#4-verify-search-index","title":"4. Verify Search Index","text":"

Then verify that the index was created successfully:

  1. Visit the Azure Portal and look up your Resource Group
  2. Visit the Azure AI Search resource page from that RG
  3. Click on \"Search Explorer\" from the resource overview page
  4. Click \"Search\" - verify that you see indexed products
"},{"location":"0-Workshop/3-Ideate/03/","title":"3.3 Retrieve Related Products","text":""},{"location":"0-Workshop/3-Ideate/03/#1-add-docs-retrieval-script","title":"1. Add Docs Retrieval Script","text":"

Let's copy over the get_product_documents.py script into our application source folder.

cp src.sample/api/get_product_documents.py  src/api/.\n
"},{"location":"0-Workshop/3-Ideate/03/#2-understand-docs-retrieval","title":"2. Understand Docs Retrieval","text":"

Let's start with a sample user query like this:

 I need a new tent for 4 people, what would you recommend?\n

Different users could phrase the question in different ways, with different levels of information. But we need to map all these queries to a search query that works on the product database. How do we do that? We use a 3-step process:

  1. We teach an AI to extract user intent from an input text query
  2. We teach the AI to map user intent to a search query on products
  3. We use the search index to retrieve product documents matching query.

Let's see how we do this.

"},{"location":"0-Workshop/3-Ideate/03/#3-create-ai-project-client","title":"3. Create AI Project Client","text":"
  1. Create an Azure AI Project client using the connection string
  2. Use the client to retrieve a chat_completions inference client
  3. Use the client to retrieve an embeddings inference client
  4. Use the client to setup a search_client using the search connection
Click to expand and view the Python script src/api/get_product_documents.py - Part 1
import os\nfrom pathlib import Path\nfrom opentelemetry import trace\nfrom azure.ai.projects import AIProjectClient\nfrom azure.ai.projects.models import ConnectionType\nfrom azure.identity import DefaultAzureCredential\nfrom azure.core.credentials import AzureKeyCredential\nfrom azure.search.documents import SearchClient\nfrom config import ASSET_PATH, get_logger\n\n# initialize logging and tracing objects\nlogger = get_logger(__name__)\ntracer = trace.get_tracer(__name__)\n\n# create a project client using environment variables loaded from the .env file\nproject = AIProjectClient.from_connection_string(\n    conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n)\n\n# create a vector embeddings client that will be used to generate vector embeddings\nchat = project.inference.get_chat_completions_client()\nembeddings = project.inference.get_embeddings_client()\n\n# use the project client to get the default search connection\nsearch_connection = project.connections.get_default(\n    connection_type=ConnectionType.AZURE_AI_SEARCH, include_credentials=True\n)\n\n# Create a search index client using the search connection\n# This client will be used to create and delete search indexes\nsearch_client = SearchClient(\n    index_name=os.environ[\"AISEARCH_INDEX_NAME\"],\n    endpoint=search_connection.endpoint_url,\n    credential=AzureKeyCredential(key=search_connection.key),\n)\n
"},{"location":"0-Workshop/3-Ideate/03/#4-get-docs-for-user-intent","title":"4. Get Docs For User Intent","text":"
  1. First, receive input text string (user query)
  2. Then, map user query text into a clear intent (search query)
  3. Then, vectorize the search query (to support retrieval)
  4. Then, search the product index for matches (by cosine similarity)
  5. Then, for each matching product, retrieve its document (content)
  6. Return the collection of documents to the user.
Click to expand and view the Python script src/api/get_product_documents.py - Part 2
    from azure.ai.inference.prompts import PromptTemplate\n    from azure.search.documents.models import VectorizedQuery\n\n    @tracer.start_as_current_span(name=\"get_product_documents\")\n    def get_product_documents(messages: list, context: dict = None) -> dict:\n        if context is None:\n            context = {}\n\n        overrides = context.get(\"overrides\", {})\n        top = overrides.get(\"top\", 5)\n\n        # generate a search query from the chat messages\n        intent_prompty = PromptTemplate.from_prompty(Path(ASSET_PATH) / \"intent_mapping.prompty\")\n\n        intent_mapping_response = chat.complete(\n            model=os.environ[\"INTENT_MAPPING_MODEL\"],\n            messages=intent_prompty.create_messages(conversation=messages),\n            **intent_prompty.parameters,\n        )\n\n        search_query = intent_mapping_response.choices[0].message.content\n        logger.debug(f\"\ud83e\udde0 Intent mapping: {search_query}\")\n\n        # generate a vector representation of the search query\n        embedding = embeddings.embed(model=os.environ[\"EMBEDDINGS_MODEL\"], input=search_query)\n        search_vector = embedding.data[0].embedding\n\n        # search the index for products matching the search query\n        vector_query = VectorizedQuery(vector=search_vector, k_nearest_neighbors=top, fields=\"contentVector\")\n\n        search_results = search_client.search(\n            search_text=search_query, vector_queries=[vector_query], select=[\"id\", \"content\", \"filepath\", \"title\", \"url\"]\n        )\n\n        documents = [\n            {\n                \"id\": result[\"id\"],\n                \"content\": result[\"content\"],\n                \"filepath\": result[\"filepath\"],\n                \"title\": result[\"title\"],\n                \"url\": result[\"url\"],\n            }\n            for result in search_results\n        ]\n\n        # add results to the provided context\n        if \"thoughts\" not in context:\n            context[\"thoughts\"] = []\n\n        # add thoughts and documents to the context object so it can be returned to the caller\n        context[\"thoughts\"].append(\n            {\n                \"title\": \"Generated search query\",\n                \"description\": search_query,\n            }\n        )\n\n        if \"grounding_data\" not in context:\n            context[\"grounding_data\"] = []\n        context[\"grounding_data\"].append(documents)\n\n        logger.debug(f\"\ud83d\udcc4 {len(documents)} documents retrieved: {documents}\")\n        return documents\n
"},{"location":"0-Workshop/3-Ideate/03/#5-run-docs-retrieval-script","title":"5. Run Docs Retrieval Script","text":"

Before we can run this script, we need to create the Intent Mapper template for step 2. Let's do that next.

"},{"location":"0-Workshop/3-Ideate/04/","title":"3.4 Understand Intent Mapping","text":""},{"location":"0-Workshop/3-Ideate/04/#1-add-intent-mapping-prompty","title":"1. Add Intent Mapping Prompty","text":"

Let's copy over the intent_mapping.prompty prompt template into our assets folder.

cp src.sample/api/assets/intent_mapping.prompty src/api/assets/.\n
"},{"location":"0-Workshop/3-Ideate/04/#2-run-docs-retrieval-script","title":"2. Run Docs Retrieval Script","text":"

Before we dive into the details of that file, let's first run the document retrieval script and see what happens. Here's an example with the question we discussed earlier.

python get_product_documents.py \\\n    --query \"I need a new tent for 4 people, what would you recommend?\"\n\n\ud83e\udde0 Intent mapping: {\n    \"intent\": \"The user is looking for recommendations for a tent suitable for 4 people.\",\n    \"search_query\": \"best tents for 4 people\"\n}\n

The script output shows that it extracted the user intent and formulated a search query from it that related to a product (\"best tents for 4 people\") - and can be answered by the search index.

"},{"location":"0-Workshop/3-Ideate/04/#3-intent-mapping-in-action","title":"3. Intent Mapping In Action","text":"

The intent mapping is achieved using a Prompty asset - a YAML file that consists of frontmatter (prompt metdata) and content (prompt template)

  • Frontmatter defines model configuration and prompt inputs
  • Template defines prompt structure, context and instructions

Looking into the details of the prompt template, we can see it employs a few-shot learning technique where it attempts to teach the AI to execute a specific task (extract intent) by providing it with examples of inputs and expected responses.

Click to expand and view Intent Mapping Prompty src/api/assets/intent_mapping.prompty
 ---\nname: Chat Prompt\ndescription: A prompty that extract users query intent based on the current_query and chat_history of the conversation\nmodel:\n    api: chat\n    configuration:\n        azure_deployment: gpt-4o\ninputs:\n    conversation:\n        type: array\n---\nsystem:\n# Instructions\n- You are an AI assistant reading a current user query and chat_history.\n- Given the chat_history, and current user's query, infer the user's intent expressed in the current user query.\n- Once you infer the intent, respond with a search query that can be used to retrieve relevant documents for the current user's query based on the intent\n- Be specific in what the user is asking about, but disregard parts of the chat history that are not relevant to the user's intent.\n- Provide responses in json format\n\n# Examples\nExample 1:\nWith a conversation like below:\n    ```\n    - user: are the trailwalker shoes waterproof?\n    - assistant: Yes, the TrailWalker Hiking Shoes are waterproof. They are designed with a durable and waterproof construction to withstand various terrains and weather conditions.\n    - user: how much do they cost?\n    ```\nRespond with:\n{\n    \"intent\": \"The user wants to know how much the Trailwalker Hiking Shoes cost.\",\n    \"search_query\": \"price of Trailwalker Hiking Shoes\"\n}\n\nExample 2:\nWith a conversation like below:\n    ```\n    - user: are the trailwalker shoes waterproof?\n    - assistant: Yes, the TrailWalker Hiking Shoes are waterproof. They are designed with a durable and waterproof construction to withstand various terrains and weather conditions.\n    - user: how much do they cost?\n    - assistant: The TrailWalker Hiking Shoes are priced at $110.\n    - user: do you have waterproof tents?\n    - assistant: Yes, we have waterproof tents available. Can you please provide more information about the type or size of tent you are looking for?\n    - user: which is your most waterproof tent?\n    - assistant: Our most waterproof tent is the Alpine Explorer Tent. It is designed with a waterproof material and has a rainfly with a waterproof rating of 3000mm. This tent provides reliable protection against rain and moisture.\n    - user: how much does it cost?\n    ```\nRespond with:\n{\n    \"intent\": \"The user would like to know how much the Alpine Explorer Tent costs.\",\n    \"search_query\": \"price of Alpine Explorer Tent\"\n}\n\nuser:\nReturn the search query for the messages in the following conversation:\n{{#conversation}}\n- {{role}}: {{content}}\n{{/conversation}}\n
"},{"location":"0-Workshop/3-Ideate/05/","title":"3.5 Retrieve Related Knowledge","text":""},{"location":"0-Workshop/3-Ideate/05/#1-add-chat-with-products-script","title":"1. Add Chat With Products Script","text":"

Let's copy over the chat_with_products.py script into our application source.

cp src.sample/api/chat_with_products.py src/api/.\n
"},{"location":"0-Workshop/3-Ideate/05/#2-understand-rag-workflow","title":"2. Understand RAG Workflow","text":"

This script is the core orchestrator for our RAG workflow, executing the following steps:

  1. Create an Azure AI Project client (with connection string)
  2. Retrieve the inference client for chat completions model
  3. Use incoming user messages to retrieve related products
  4. Use this knowledge to populate a grounded chat template
  5. Call the chat completions client with this prompt template
Click to expand and view Chat With Products script (segment) src/api/chat_with_products.py
from azure.ai.inference.prompts import PromptTemplate\n\n\n@tracer.start_as_current_span(name=\"chat_with_products\")\ndef chat_with_products(messages: list, context: dict = None) -> dict:\n    if context is None:\n        context = {}\n\n    documents = get_product_documents(messages, context)\n\n    # do a grounded chat call using the search results\n    grounded_chat_prompt = PromptTemplate.from_prompty(Path(ASSET_PATH) / \"grounded_chat.prompty\")\n\n    system_message = grounded_chat_prompt.create_messages(documents=documents, context=context)\n    response = chat.complete(\n        model=os.environ[\"CHAT_MODEL\"],\n        messages=system_message + messages,\n        **grounded_chat_prompt.parameters,\n    )\n    logger.info(f\"\ud83d\udcac Response: {response.choices[0].message}\")\n\n    # Return a chat protocol compliant response\n    return {\"message\": response.choices[0].message, \"context\": context}\n

In the next section, we'll look at the prompt template and run a test with a sample query.

"},{"location":"0-Workshop/3-Ideate/05/#3-run-chat-with-products-script","title":"3. Run Chat With Products Script","text":"

Before we can run this script, we need to create the Grounded Chat Prompt template for step 4. Let's do that next.

"},{"location":"0-Workshop/3-Ideate/06/","title":"3.6 Design Grounded Prompt","text":""},{"location":"0-Workshop/3-Ideate/06/#1-add-grounded-chat-prompty","title":"1. Add Grounded Chat Prompty","text":"

Let's copy over the grounded_chat.prompty prompt template into our assets folder.

cp src.sample/api/assets/grounded_chat.prompty src/api/assets/.\n
"},{"location":"0-Workshop/3-Ideate/06/#2-understand-grounding-context","title":"2. Understand Grounding Context","text":"

Explore the contents of this template. Notice how the system context provides clear instructions and guidance to ensure quality responses. This includes grounding responses in context (when query is relevant) and declining to provide responses (when query is irrelevant).

Click to expand and view Grounded Chat Prompty src/api/assets/grounded_chat.prompty
    ---\n    name: Chat with documents\n    description: Uses a chat completions model to respond to queries grounded in relevant documents\n    model:\n        api: chat\n        configuration:\n            azure_deployment: gpt-4o\n    inputs:\n        conversation:\n            type: array\n    ---\n    system:\n    You are an AI assistant helping users with queries related to outdoor outdooor/camping gear and clothing.\n    If the question is not related to outdoor/camping gear and clothing, just say 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?'\n    Don't try to make up any answers.\n    If the question is related to outdoor/camping gear and clothing but vague, ask for clarifying questions instead of referencing documents. If the question is general, for example it uses \"it\" or \"they\", ask the user to specify what product they are asking about.\n    Use the following pieces of context to answer the questions about outdoor/camping gear and clothing as completely, correctly, and concisely as possible.\n    Do not add documentation reference in the response.\n\n    # Documents\n\n    {{#documents}}\n\n    ## Document {{id}}: {{title}}\n    {{content}}\n    {{/documents}}\n
"},{"location":"0-Workshop/3-Ideate/06/#3-chat-with-products-relevant","title":"3. Chat With Products - Relevant","text":"

Run the script with a test query (from the src/api folder). 

python chat_with_products.py --query \"I need a new tent for 4 people, what would you recommend?\"\n

Observe the response. It may look like this:

\ud83d\udcac Response: {'content': \"I recommend the TrailMaster X4 Tent. It is specifically designed to accommodate four occupants comfortably. The tent features durable water-resistant construction, multiple doors for easy access, and mesh panels for ventilation and bug protection. Additionally, it has a freestanding design for easy setup and relocation, as well as interior pockets for organization. It's a great choice for your camping adventures!\", 'role': 'assistant'}\n

Is the response grounded in product data from the catalog?

"},{"location":"0-Workshop/3-Ideate/06/#4-chat-with-products-irrelevant","title":"4. Chat With Products - Irrelevant","text":"

Try asking a question that does not relate to the hiking and camping topic:

python chat_with_products.py --query \"I am looking for a recipe for spicy bean burgers\"\n

Observe the response. It may look like this:

\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n

Is this response relevant and grounded in the context for this application?

"},{"location":"0-Workshop/3-Ideate/07/","title":"3.7 Test with Observability","text":"

Recall that we activated Application Insights during the setup phase of our projec. This now allows to ask questions with telemetry enabled and get observable traces using open telemetry, to help us analyze the cost or performance of our workflows.

Let's see this in action.

"},{"location":"0-Workshop/3-Ideate/07/#1-enable-telemetry-on-test","title":"1. Enable Telemetry on Test","text":"

Run the Chat with Products script with --enable-telemetry as shown.

python chat_with_products.py --query \"I need hiking gear for a trip to Andalusia - what tents and boots do you recommend?\" --enable-telemetry\n

You should see something like this in the console. Let's explore this next. 

Enabled telemetry logging to project, view traces at:\nhttps://ai.azure.com/tracing?wsid=/subscriptions/XXXX/resourceGroups/ninarasi-ragchat-rg/providers/Microsoft.MachineLearningServices/workspaces/ninarasi-ragchat-v1\n\ud83d\udcac Response: {'content': \"For your trip to Andalusia, I recommend the following tents and boots:\\n\\n**Tents:**\\n1. **Alpine Explorer Tent**: This robust, 8-person, 3-season tent is perfect for group camping. It has multiple mesh windows for ventilation and a detachable divider for privacy. Its waterproof feature ensures you stay dry during unexpected rain.\\n\\n2. **SkyView 2-Person Tent**: If you're looking for a smaller option, this tent comfortably houses two people and is made from durable waterproof materials. It also features an intuitive setup system, effective ventilation, and a rainfly for extra weather protection, making it great for hiking and camping.\\n\\n**Boots:**\\n1. **TrekReady Hiking Boots**: These boots are crafted from leather, ensuring durability and comfort on long hikes. They have a moisture-wicking lining, shock-absorbing midsoles, and excellent traction, making them suitable for various terrains.\\n\\n2. **TrekStar Hiking Sandals**: If you prefer something lighter and more breathable, consider these lightweight sandals. They offer adjustable straps, excellent traction, toe protection, and a cushioned footbed for comfort during summer treks.\\n\\nChoose based on your group size and hiking preferences, and you'll be well-prepared for your adventure in Andalusia!\", 'role': 'assistant'}\n

"},{"location":"0-Workshop/3-Ideate/07/#2-view-traces-detail","title":"2. View Traces Detail","text":"

Look for the output section with a link as shown below, and navigate to that URL in the browser.

Enabled telemetry logging to project, view traces at:\nhttps://ai.azure.com/tracing?....\n

You should see something like this reflecting the latest run of the Chat with Products script above. Click to expand the tree of nodes on the left and observe the details provided in the panel on the right, as you step through them.

  1. We can trace the flow of control from the user query to the returned response
  2. We can measure the time taken for each step to execute (in seconds)
  3. We can observe the cost for model invocations (in tokens) in the GenAI rows
  4. For a given model interaction, we can explore details (system, user, assistant) to debug

"},{"location":"0-Workshop/3-Ideate/07/#3-view-traces-dashboard","title":"3. View Traces Dashboard","text":"

Click on the Tracing menu option in the sidebar to get the historical logs from previous runs. This is an effective way to analyze issues in cost or performance, make changes, then compare trace runs to see if the iterations had any impact.

"},{"location":"0-Workshop/3-Ideate/07/#4-view-traces-insights","title":"4. View Traces Insights","text":"

You can also click on the Insights for Generative AI Applications Dashboard link (top of screen) to get a more actionable Generative AI Application Insights (Preview) dashboard for real-time insights and analysis of usage patterns.

"},{"location":"0-Workshop/4-Evaluate/01/","title":"4.1 Create Evaluation Dataset","text":"

This begins Part 3 of the tutorial. This stage is completed USING THE AZURE AI FOUNDRY SDK.

At the end of this section, you should have established an evaluation inputs dataset, created an evaluation script for AI-assisted evaluation, and run the script to assess the chat AI app for quality. You will then learn to explore the evaluation outcomes locally, and via the Azure AI Foundry portal, and understand how to customize and automate the process for rapid iteration and improvement of your application quality.

"},{"location":"0-Workshop/4-Evaluate/01/#1-evaluating-generative-ai-apps","title":"1. Evaluating Generative AI Apps","text":"

The evaluation phase is critical to assessing the quality and safety of your generative AI application. The Azure AI Foundry provides a comprehensive set of tools and guidance to support evaluation in three dimensions. Learn More Here.

  1. Risk and safety evaluators: Evaluating potential risks associated with AI-generated content. Ex: evaluating AI predisposition towards generating harmful or inappropriate content.
  2. Performance and quality evaluators: Assessing accuracy, groundedness, and relevance of generated content using robust AI-assisted and Natural Language Processing (NLP) metrics.
  3. Custom evaluators: Tailored evaluation metrics that allow for more detailed and specific analyses. Ex: addressing app-specific requirements not covered by standard metrics.

AI-Assisted Evaluators are available only in select regions (Recommended: East US 2)

"},{"location":"0-Workshop/4-Evaluate/01/#2-ai-assisted-evaluation-flow","title":"2. AI-Assisted Evaluation Flow","text":"

So far, we've tested the chat application interactively (command-line) using a single test prompt. Now, we want to evaluate the responses on a larger and more diverse set of test prompts including edge cases. We can then use those results to iterate on our application (e.g., prompt template, data sources) till evaluations pass our acceptance criteria.

To do this we use AI-Assisted Evaluation - also referred to as LLM-as-a-judge - where we ask a second AI model (\"judge\") to grade the responses of the first AI model (\"target\"). The workflow is as shown below:

  1. First, create an evaluation dataset that consists of diverse queries for testing.
  2. Next, have the target AI (app) generate responses for each query
  3. Next, have the judge AI (assessor) grade tge {query, response} pairs
  4. Finally, visualize results (individual vs. aggregate metrics) for review.

"},{"location":"0-Workshop/4-Evaluate/01/#3-create-evaluation-dataset","title":"3. Create Evaluation Dataset","text":"

Let's copy over the chat_eval_data.jsonl dataset into our assets folder.

cp src.sample/api/assets/chat_eval_data.jsonl src/api/assets/.\n
"},{"location":"0-Workshop/4-Evaluate/01/#4-review-evaluation-dataset","title":"4. Review Evaluation Dataset","text":"

The Azure AI Foundry supports different data formats for evaluation including:

  1. Query/Response - each result has the query, response, and ground truth.
  2. Conversation (single/multi-turn) - messages (with content, role, optional context)
Click to expand and view the evaluation dataset src/api/assets/chat_eval_data.jsonl
{\"query\": \"Which tent is the most waterproof?\", \"truth\": \"The Alpine Explorer Tent has the highest rainfly waterproof rating at 3000m\"}\n{\"query\": \"Which camping table holds the most weight?\", \"truth\": \"The Adventure Dining Table has a higher weight capacity than all of the other camping tables mentioned\"}\n{\"query\": \"How much do the TrailWalker Hiking Shoes cost? \", \"truth\": \"The Trailewalker Hiking Shoes are priced at $110\"}\n{\"query\": \"What is the proper care for trailwalker hiking shoes? \", \"truth\": \"After each use, remove any dirt or debris by brushing or wiping the shoes with a damp cloth.\"}\n{\"query\": \"What brand is TrailMaster tent? \", \"truth\": \"OutdoorLiving\"}\n{\"query\": \"How do I carry the TrailMaster tent around? \", \"truth\": \" Carry bag included for convenient storage and transportation\"}\n{\"query\": \"What is the floor area for Floor Area? \", \"truth\": \"80 square feet\"}\n{\"query\": \"What is the material for TrailBlaze Hiking Pants?\", \"truth\": \"Made of high-quality nylon fabric\"}\n{\"query\": \"What color does TrailBlaze Hiking Pants come in?\", \"truth\": \"Khaki\"}\n{\"query\": \"Can the warrenty for TrailBlaze pants be transfered? \", \"truth\": \"The warranty is non-transferable and applies only to the original purchaser of the TrailBlaze Hiking Pants. It is valid only when the product is purchased from an authorized retailer.\"}\n{\"query\": \"How long are the TrailBlaze pants under warranty for? \", \"truth\": \" The TrailBlaze Hiking Pants are backed by a 1-year limited warranty from the date of purchase.\"}\n{\"query\": \"What is the material for PowerBurner Camping Stove? \", \"truth\": \"Stainless Steel\"}\n{\"query\": \"Is France in Europe?\", \"truth\": \"Sorry, I can only queries related to outdoor/camping gear and equipment\"}\n

Our dataset reflects the first format, where the test prompts contain a query with the ground truth for evaluating responses. The chat AI will then generate a response (based on query) that gets added to this record, to create the evaluation dataset that is sent to the \"judge\" AI.

 {\n    \"query\": \"Which tent is the most waterproof?\", \n    \"truth\": \"The Alpine Explorer Tent has the highest rainfly waterproof rating at 3000m\"\n}\n
Let's look at the evaluation script that orchestrates this workflow, next

"},{"location":"0-Workshop/4-Evaluate/02/","title":"4.2 Create Evaluation Script","text":""},{"location":"0-Workshop/4-Evaluate/02/#1-create-evaluation-script","title":"1. Create Evaluation Script","text":"

Let's copy over the evaluate.py script into our application source folder.

cp src.sample/api/evaluate.py  src/api/.\n
"},{"location":"0-Workshop/4-Evaluate/02/#2-review-evaluation-workflow","title":"2. Review Evaluation Workflow","text":"

Let's review the workflow required for AI-Assisted evaluation:

  1. We have an evaluation test dataset containing query/truth pairs
  2. We have a target AI (applicationl) that will generate the responses
  3. We have a judge AI (evaluator) that will grade those responses
  4. The judge has scoring criteria they use to generate evaluation metrics
  5. The evaluation results are published locally, or to Azure AI Foundry

"},{"location":"0-Workshop/4-Evaluate/02/#3-unpack-evaluation-script","title":"3. Unpack Evaluation Script","text":""},{"location":"0-Workshop/4-Evaluate/02/#31-create-ai-project-client","title":"3.1 Create AI Project Client","text":"src/api/evaluation.py
    # ----------------------------------------------\n    # 1. Create AI Project Client \n    # ----------------------------------------------\n    import os\n    import pandas as pd\n    from azure.ai.projects import AIProjectClient\n    from azure.ai.projects.models import ConnectionType\n    from azure.ai.evaluation import evaluate, GroundednessEvaluator\n    from azure.identity import DefaultAzureCredential\n\n    from chat_with_products import chat_with_products\n\n    # load environment variables from the .env file at the root of this repo\n    from dotenv import load_dotenv\n    load_dotenv()\n\n    # create a project client using environment variables loaded from the .env file\n    project = AIProjectClient.from_connection_string(\n        conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n    )\n\n    connection = project.connections.get_default(connection_type=ConnectionType.AZURE_OPEN_AI, include_credentials=True)\n
"},{"location":"0-Workshop/4-Evaluate/02/#32-specify-model-evaluators","title":"3.2 Specify Model & Evaluators","text":"src/api/evaluation.py
    # ----------------------------------------------\n    # 2. Define Evaluator Model to use\n    # ----------------------------------------------\n    evaluator_model = {\n        \"azure_endpoint\": connection.endpoint_url,\n        \"azure_deployment\": os.environ[\"EVALUATION_MODEL\"],\n        \"api_version\": \"2024-06-01\",\n        \"api_key\": connection.key,\n    }\n\n    groundedness = GroundednessEvaluator(evaluator_model)\n
"},{"location":"0-Workshop/4-Evaluate/02/#33-create-evaluation-wrapper","title":"3.3 Create Evaluation Wrapper","text":"src/api/evaluation.py
    # ----------------------------------------------\n    # 3. Create Eval Wrapper Function for Query\n    # ----------------------------------------------\n    def evaluate_chat_with_products(query):\n        response = chat_with_products(messages=[{\"role\": \"user\", \"content\": query}])\n        return {\"response\": response[\"message\"].content, \"context\": response[\"context\"][\"grounding_data\"]}\n
"},{"location":"0-Workshop/4-Evaluate/02/#34-run-evaluation-print","title":"3.4 Run Evaluation & Print","text":"

This is the entry point for the evaluation script.

  • It uses the evaluate function from the azure.ai.evaluation package to run a built-in evaluator (GroundednessEvaluator) to score the responses from the target app.
  • If both the data (test) and target (function) are provided, it will first invoke the target with that data - and then run evaluations on the results.
  • If azure_ai_project is set, then the evaluation results are also logged to Azure AI Foundry
src/api/evaluation.py
    # ----------------------------------------------\n    # 4. Run the Evaluation\n    #    View Results Locally (Saved as JSON)\n    #    Get Link to View Results in Foundry Portal\n    #    NOTE:\n    #    Evaluation takes more tokens \n    #    Try to increase Rwate limit (Tokens per minute)\n    #    Script should handle limit errors if needed\n    # ----------------------------------------------\n    # Evaluate must be called inside of __main__, not on import\n\n    if __name__ == \"__main__\":\n        from config import ASSET_PATH\n\n        # workaround for multiprocessing issue on linux\n        from pprint import pprint\n        from pathlib import Path\n        import multiprocessing\n        import contextlib\n\n        with contextlib.suppress(RuntimeError):\n            multiprocessing.set_start_method(\"spawn\", force=True)\n\n        # run evaluation with a dataset and target function, \n        # log to the project\n        result = evaluate(\n            data=Path(ASSET_PATH) / \"chat_eval_data.jsonl\",\n            target=evaluate_chat_with_products,\n            evaluation_name=\"evaluate_chat_with_products\",\n            evaluators={\n                \"groundedness\": groundedness,\n            },\n            evaluator_config={\n                \"default\": {\n                    \"query\": {\"${data.query}\"},\n                    \"response\": {\"${target.response}\"},\n                    \"context\": {\"${target.context}\"},\n                }\n            },\n            azure_ai_project=project.scope,\n            output_path=\"./myevalresults.json\",\n        )\n\n        tabular_result = pd.DataFrame(result.get(\"rows\"))\n\n        pprint(\"-----Summarized Metrics-----\")\n        pprint(result[\"metrics\"])\n        pprint(\"-----Tabular Result-----\")\n        pprint(tabular_result)\n        pprint(f\"View evaluation results in AI Studio: {result['studio_url']}\")\n
"},{"location":"0-Workshop/4-Evaluate/03/","title":"4.3 Configure Evaluation Model","text":""},{"location":"0-Workshop/4-Evaluate/04/","title":"4.4 Run Evaluation Script","text":""},{"location":"0-Workshop/4-Evaluate/05/","title":"4.5 View Results Locally","text":""},{"location":"0-Workshop/4-Evaluate/06/","title":"4.6 View Results In Portal","text":""},{"location":"0-Workshop/4-Evaluate/07/","title":"4.7 Validate The Evaluation","text":"Text Only
# ----------------------------------------------\n# Run the Evaluation using this command\n#    python evaluate.py \n#\n# You should see something like this:\n\n'''\nStarting prompt flow service...\nStart prompt flow service on 127.0.0.1:23333, version: 1.16.2.\nYou can stop the prompt flow service with the following command:'pf service stop'.\n\nYou can view the traces in local from http://127.0.0.1:23333/v1.0/ui/traces/?#run=main_evaluate_chat_with_products_rxna_3r9_20241216_163719_733780\n[2024-12-16 16:37:42 +0000][promptflow._sdk._orchestrator.run_submitter][INFO] - Submitting run main_evaluate_chat_with_products_rxna_3r9_20241216_163719_733780, log path: /home/vscode/.promptflow/.runs/main_evaluate_chat_with_products_rxna_3r9_20241216_163719_733780/logs.txt\n\ud83d\udcac Response: {'content': 'Could you please specify which camping table you are referring to? There are multiple options available, and I can provide information on them.', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': \"Could you specify what aspect of care you're asking about? Are you looking for cleaning instructions, storage tips, or something else for the TrailWalker Hiking Shoes?\", 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': \"Could you please specify which tents you are comparing, or do you want information about a specific tent's waterproof features?\", 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'The TrailBlaze Hiking Pants are crafted from high-quality nylon fabric.', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'The TrailMaster X4 Tent comes with an included carry bag, which makes transporting the tent easy and convenient. You can simply pack the tent into the carry bag and carry it as needed for your camping adventure. If you have any more specific questions about the tent or its features, feel free to ask!', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n...\n...\n...\n'''\n\n#This is what a rate limit error looks like:\n\n'''\n[2024-12-16 16:44:07 +0000][promptflow.core._prompty_utils][ERROR] - Exception occurs: RateLimitError: Error code: 429 - {'error': {'code': '429', 'message': 'Requests to the ChatCompletions_Create Operation under Azure OpenAI API version 2024-06-01 have exceeded token rate limit of your current AIServices S0 pricing tier. Please retry after 60 seconds. Please contact Azure support service if you would like to further increase the default rate limit.'}}\n[2024-12-16 16:44:07 +0000][promptflow.core._prompty_utils][WARNING] - RateLimitError #2, Retry-After=60, Back off 60.0 seconds for retry.\n'''\n\n# This is what the final result looks like\n\n'''\n======= Run Summary =======\n\nRun name: \"azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_rvrjml8t_20241216_164205_696721\"\nRun status: \"Completed\"\nStart time: \"2024-12-16 16:42:05.695977+00:00\"\nDuration: \"0:03:06.490785\"\nOutput path: \"/home/vscode/.promptflow/.runs/azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_rvrjml8t_20241216_164205_696721\"\n\n======= Combined Run Summary (Per Evaluator) =======\n\n{\n    \"groundedness\": {\n        \"status\": \"Completed\",\n        \"duration\": \"0:03:06.490785\",\n        \"completed_lines\": 13,\n        \"failed_lines\": 0,\n        \"log_path\": \"/home/vscode/.promptflow/.runs/azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_rvrjml8t_20241216_164205_696721\"\n    }\n}\n\n====================================================\n\nEvaluation results saved to \"/workspaces/learns-azure-ai-foundry/src/api/myevalresults.json\".\n\n'-----Summarized Metrics-----'\n{'groundedness.gpt_groundedness': 1.4615384615384615,\n'groundedness.groundedness': 1.4615384615384615}\n'-----Tabular Result-----'\n                                    outputs.response  ... line_number\n0   Could you please specify which tents you are c...  ...           0\n1   Could you please specify which camping table y...  ...           1\n2   Sorry, I only can answer queries related to ou...  ...           2\n3   Could you specify what aspect of care you're a...  ...           3\n4   Sorry, I only can answer queries related to ou...  ...           4\n5   The TrailMaster X4 Tent comes with an included...  ...           5\n6   Sorry, I only can answer queries related to ou...  ...           6\n7   The TrailBlaze Hiking Pants are crafted from h...  ...           7\n8   Sorry, I only can answer queries related to ou...  ...           8\n9   Sorry, I only can answer queries related to ou...  ...           9\n10  Sorry, I only can answer queries related to ou...  ...          10\n11  Sorry, I only can answer queries related to ou...  ...          11\n12  Sorry, I only can answer queries related to ou...  ...          12\n\n[13 rows x 8 columns]\n('View evaluation results in AI Studio: '\n'https://ai.azure.com/build/evaluation/XXXXXXX?wsid=/subscriptions/XXXXXXXX/resourceGroups/ninarasi-ragchat-rg/providers/Microsoft.MachineLearningServices/workspaces/ninarasi-ragchat-v1')\n'''\n\n# ----------------------------------------------\n\n```\n
"},{"location":"0-Workshop/5-Evolve/01/","title":"5.1 Recap: Build A Copilot","text":"

PLACEHOLDER \ud83d\udc49\ud83c\udffd EXPLAIN WHAT WE COVERED (AND WHAT WE DIDN'T)

"},{"location":"0-Workshop/5-Evolve/02/","title":"5.2 Refactor: Make it Better","text":"

PLACEHOLDER \ud83d\udc49\ud83c\udffd EXPLAIN HOW CONTOSO CHAT STREAMLINES E2E

"},{"location":"0-Workshop/5-Evolve/03/","title":"5.3 Resources: Learn More","text":"

PLACEHOLDER \ud83d\udc49\ud83c\udffd EXPLAIN WHERE TO GO FOR ADVANCED TUTORIALS

"}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Introduction","text":"

This repository contains the source and instructions guide for a workshop on building a RAG-based chat app using Azure AI Foundry. The workshop is derived from the official 3-part tutorial and adapted to provide additional resources and suggestions for self-guided learners.

"},{"location":"#pre-requisites","title":"Pre-Requisites","text":"

To get the most from this lab, you will need the following:

  1. An Azure subscription - Get one for free
  2. A GitHub account - Get one for free
  3. Familiarity with VS Code, Github & Azure.
  4. Familiarity with Python and Jupyter Notebooks.

Verify: Your Azure subscription has sufficient quota to deploy these models:

  1. Chat: gpt-4o-mini
  2. Embeddings: text-embedding-ada-002

Verify: Your Azure account is authorized to make role assignments for Azure AI resources. This may involve having a Privileged role like Owner, User Access Admin or RBAC Admin.

"},{"location":"#getting-started","title":"Getting Started","text":"

This repository is instrumented with a devcontainer.json to give you a development environment with all required dependencies pre-installed. To get started:

  1. Fork the repository to your personal profile. Visit your fork in the browser.

  2. Launch Codespaces on that fork. Setup will take a few minutes.

  3. Type this command into the VS Code terminal when ready. A dialog will pop up.

    mkdocs serve\n

  4. Select the \"View in browser\" option in the pop-up dialog. You should see this guide.

  5. Open a second VS Code terminal pane. Use that for all further instructions.

"},{"location":"0-Workshop/1-Overview/00/","title":"1.1 Learning Objectives","text":"

This lab takes you through the \"Build a custom chat app with the Azure AI Foundry SDK with a devcontainer configuration for fast setup. Fork the repo and launch GitHub Codespaces to work through lab exercises.

The lab teaches you to build a RAG-based copilot using the Azure AI Foundry platform By completing this lab, you will learn to do the following tasks:

  1. Use the Azure AI Foundry Portal to
    • start a new Azure AI project
    • discover and deploy AI models to the project
    • add an Azure AI Search resource to the project
    • setup an Application Insights resource for tracing
  2. Use the Azure AI Foundry SDK to
    • create a new search index with your data
    • extract customer intent with an intent mapping prompt template
    • retrieve relevant knowledge using the search index
    • create a chat agent with a grounded prompt template
    • process customer queries returning responses grounded in my data.
  3. Use the Azure AI Evaluation SDK to assess app quality by
    • creating a custom evaluation dataset.
    • executing a custom evaluation workflow.
    • viewing evaluation results, in your local environment.
    • viewing evaluation results, in Azure AI Foundry portal.

This sets you up with a \"sandbox\" project that you can use to explore other features or tooling in Azure AI Foundry. For example - try bringing your own application data, or adding secondary data sources for knowledge retrieval as part of the RAG flow.

BONUS: Use what you learn here to explore Contoso Chat next! (click to expand)

Contoso Chat is an open-source reference sample that implements the retail RAG-based copilot (using the same data) in a code-first approach from initial provisioning to final production deployment.

  • The current version (v3) provisions the Azure AI Foundry resources using the Azure Developer CLI (azd provision) and deploys the final application to Azure Container Apps for a hosted API endpoint.
  • The next version (v4) will incorporate more elements from the Azure AI Foundry SDKs that we highlight in this RAG tutorial, including use of the Azure AI Model Inference API and the Azure AI Evaluation SDK.
"},{"location":"0-Workshop/1-Overview/01/","title":"1.2 Application Scenario","text":"

In this tutorial, we build a retail chat AI (copilot) that uses Retrieval Augmented Generation (RAG) to ground the chat responses in the retailer's own data. Let's briefly review what this means.

"},{"location":"0-Workshop/1-Overview/01/#1-rag-chat-app-tutorial","title":"1. RAG Chat App (tutorial)","text":"

This RAG Chat tutorial provides a quickstart for building and evaluating a basic RAG-based copilot using the Azure AI Foundry portal and SDK. The tutorial is grounded in the Contoso Outdoors retailer data and combines both low-code (Portal) and code-first (SDK) steps to teach the latest Azure AI Foundry tools and features. Think of this as a sandbox for open exploration

The figure explains the RAG pattern visually:

  1. The user question (prompt) is received at the copilot hosted endpoint
  2. The question is used to retrieve related knowledge from relevant sources.
  3. The prompt is then augmented with knowledge as context, and sent to the model.
  4. The model now generates a response that is \"grounded\" in this knowledge context.

"},{"location":"0-Workshop/1-Overview/01/#2-contoso-outdoor-chat-ui","title":"2. Contoso Outdoor (chat UI)","text":"

Contoso Outdoor is a fictitious enterprise retailer specializing in hiking and camping gear for outdoor enthusiasts. Their website (chat UI) provides customers with a catalog of their products, with product pages offering detailed information for user review. We'll look at the retailer data sources in the next section.

"},{"location":"0-Workshop/1-Overview/01/#3-contoso-chat-chat-ai","title":"3. Contoso Chat (chat AI)","text":"

The chat UI shown is not used in THIS workshop - but code is open-source if useful.

Contoso Chat is the open-source reference implementation of a custom RAG-based retail copilot based on the Contoso Outdoor retail scenario. It is implemented as an AI App Template that can be provisioned and deployed to Azure Container Apps to provide a hosted API endpoint. Customer requests on the chat UI (website) are now directed to this chat AI (endpoint) for processing, allowing for the user experience shown below.

"},{"location":"0-Workshop/1-Overview/02/","title":"1.3 Application Data","text":"

The Retrieval Augmented Generation (RAG) design pattern allows us to customize the AI by enhancing the user prompt with dynamically retrieved knowledge that grounds the responses in the provided context. Let's understand the shape of the data available to us - and think proactively about how you could bring your data into this mix.

"},{"location":"0-Workshop/1-Overview/02/#1-customer-info","title":"1. Customer Info","text":"

This record represents a single customer, providing their profile information (\"id\", name, contact info) and their purchase history (\"orders\"). This JSON data may be stored in a noSQL datbase like Azure CosmosDB and retrieved dynamically by the chat AI.

SAMPLE DATA (JSON) - click to expand 
{\n    \"id\": \"1\",\n    \"firstName\": \"John\",\n    \"lastName\": \"Smith\",\n    \"age\": 35,\n    \"email\": \"johnsmith@example.com\",\n    \"phone\": \"555-123-4567\",\n    \"address\": \"123 Main St,  Anytown USA, 12345\",\n    \"membership\": \"Base\",\n\n    \"orders\": [\n    {\n        \"id\": 29,\n        \"productId\": 8,\n        \"quantity\": 2,\n        \"total\": 700.0,\n        \"date\": \"2/10/2023\",\n        \"name\": \"Alpine Explorer Tent\",\n        \"unitprice\": 350.0,\n        \"category\": \"Tents\",\n        \"brand\": \"AlpineGear\",\n        \"description\": \"Welcome to the joy of camping with the Alpine Explorer Tent! This robust, 8-person, 3-season marvel is from the responsible hands of the AlpineGear brand. Promising an enviable setup that is as straightforward as counting sheep, your camping experience is transformed into a breezy pastime. Looking for privacy? The detachable divider provides separate spaces at a moment's notice. Love a tent that breathes? The numerous mesh windows and adjustable vents fend off any condensation dragon trying to dampen your adventure fun. The waterproof assurance keeps you worry-free during unexpected rain dances. With a built-in gear loft to stash away your outdoor essentials, the Alpine Explorer Tent emerges as a smooth balance of privacy, comfort, and convenience. Simply put, this tent isn't just a shelter - it's your second home in the heart of nature! Whether you're a seasoned camper or a nature-loving novice, this tent makes exploring the outdoors a joyous journey.\"\n    },\n    {\n        \"id\": 1,\n        \"productId\": 1,\n        \"quantity\": 2,\n        \"total\": 500.0,\n        \"date\": \"1/5/2023\",\n        \"name\": \"TrailMaster X4 Tent\",\n        \"unitprice\": 250.0,\n        \"category\": \"Tents\",\n        \"brand\": \"OutdoorLiving\",\n        \"description\": \"Unveiling the TrailMaster X4 Tent from OutdoorLiving, your home away from home for your next camping adventure. Crafted from durable polyester, this tent boasts a spacious interior perfect for four occupants. It ensures your dryness under drizzly skies thanks to its water-resistant construction, and the accompanying rainfly adds an extra layer of weather protection. It offers refreshing airflow and bug defence, courtesy of its mesh panels. Accessibility is not an issue with its multiple doors and interior pockets that keep small items tidy. Reflective guy lines grant better visibility at night, and the freestanding design simplifies setup and relocation. With the included carry bag, transporting this convenient abode becomes a breeze. Be it an overnight getaway or a week-long nature escapade, the TrailMaster X4 Tent provides comfort, convenience, and concord with the great outdoors. Comes with a two-year limited warranty to ensure customer satisfaction.\"\n    },\n    {\n        \"id\": 19,\n        \"productId\": 5,\n        \"quantity\": 1,\n        \"total\": 60.0,\n        \"date\": \"1/25/2023\",\n        \"name\": \"BaseCamp Folding Table\",\n        \"unitprice\": 60.0,\n        \"category\": \"Camping Tables\",\n        \"brand\": \"CampBuddy\",\n        \"description\": \"CampBuddy's BaseCamp Folding Table is an adventurer's best friend. Lightweight yet powerful, the table is a testament to fun-meets-function and will elevate any outing to new heights. Crafted from resilient, rust-resistant aluminum, the table boasts a generously sized 48 x 24 inches tabletop, perfect for meal times, games and more. The foldable design is a godsend for on-the-go explorers. Adjustable legs rise to the occasion to conquer uneven terrains and offer height versatility, while the built-in handle simplifies transportation. Additional features like non-slip feet, integrated cup holders and mesh pockets add a pinch of finesse. Quick to set up without the need for extra tools, this table is a silent yet indispensable sidekick during camping, picnics, and other outdoor events. Don't miss out on the opportunity to take your outdoor experiences to a new level with the BaseCamp Folding Table. Get yours today and embark on new adventures tomorrow!\"\n    }]\n}\n
"},{"location":"0-Workshop/1-Overview/02/#2-product-manual-info","title":"2. Product Manual Info","text":"

This record represents a single product in the retailer's catalog with extensive text (formatted as Markdown) covering information like brand, category, features, technical specs, user guide, cautions, warranty information, return policy, reviews, FAQ. This information may be used for building the Contoso Web UI, and potentially for grounding responses related to richer QA later.

The product info has been rendered as a Markmap for visual clarity. Simply zoom in/out or pan in/out to explore the content. You can click on any node (circle) to expand/collapse its sub-tree. You may need to refresh or reload page to re-render the tree.

SAMPLE RECORD (Markdown) - click to expand # Information about product item_number: 1 TrailMaster X4 Tent, price $250, ## Brand OutdoorLiving ## Category Tents ## Features - Polyester material for durability - Spacious interior to accommodate multiple people - Easy setup with included instructions - Water-resistant construction to withstand light rain - Mesh panels for ventilation and insect protection - Rainfly included for added weather protection - Multiple doors for convenient entry and exit - Interior pockets for organizing small items - Reflective guy lines for improved visibility at night - Freestanding design for easy setup and relocation - Carry bag included for convenient storage and transportation ## Technical Specs **Best Use**: Camping **Capacity**: 4-person **Season Rating**: 3-season **Setup**: Freestanding **Material**: Polyester **Waterproof**: Yes **Floor Area**: 80 square feet **Peak Height**: 6 feet **Number of Doors**: 2 **Color**: Green **Rainfly**: Included **Rainfly Waterproof Rating**: 2000mm **Tent Poles**: Aluminum **Pole Diameter**: 9mm **Ventilation**: Mesh panels and adjustable vents **Interior Pockets**: Yes (4 pockets) **Gear Loft**: Included **Footprint**: Sold separately **Guy Lines**: Reflective **Stakes**: Aluminum **Carry Bag**: Included **Dimensions**: 10ft x 8ft x 6ft (length x width x peak height) **Packed Size**: 24 inches x 8 inches **Weight**: 12 lbs ## User Guide ### Introduction Thank you for choosing the TrailMaster X4 Tent. This user guide provides instructions on how to set up, use, and maintain your tent effectively. Please read this guide thoroughly before using the tent. ### Package Contents Ensure that the package includes the following components: - TrailMaster X4 Tent body - Tent poles - Rainfly (if applicable) - Stakes and guy lines - Carry bag - User Guide If any components are missing or damaged, please contact our customer support immediately. ### Tent Setup #### Step 1: Selecting a Suitable Location - Find a level and clear area for pitching the tent. - Remove any sharp objects or debris that could damage the tent floor. #### Step 2: Unpacking and Organizing Components - Lay out all the tent components on the ground. - Familiarize yourself with each part, including the tent body, poles, rainfly, stakes, and guy lines. #### Step 3: Assembling the Tent Poles - Connect the tent poles according to their designated color codes or numbering. - Slide the connected poles through the pole sleeves or attach them to the tent body clips. #### Step 4: Setting up the Tent Body - Begin at one end and raise the tent body by pushing up the poles. - Ensure that the tent body is evenly stretched and centered. - Secure the tent body to the ground using stakes and guy lines as needed. #### Step 5: Attaching the Rainfly (if applicable) - If your tent includes a rainfly, spread it over the tent body. - Attach the rainfly to the tent corners and secure it with the provided buckles or clips. - Adjust the tension of the rainfly to ensure proper airflow and weather protection. #### Step 6: Securing the Tent - Stake down the tent corners and guy out the guy lines for additional stability. - Adjust the tension of the guy lines to provide optimal stability and wind resistance. ### Tent Takedown and Storage #### Step 1: Removing Stakes and Guy Lines - Remove all stakes from the ground. - Untie or disconnect the guy lines from the tent and store them separately. #### Step 2: Taking Down the Tent Body - Start by collapsing the tent poles carefully. - Remove the poles from the pole sleeves or clips. - Collapse the tent body and fold it neatly. #### Step 3: Disassembling the Tent Poles - Disconnect and separate the individual pole sections. - Store the poles in their designated bag or sleeve. #### Step 4: Packing the Tent - Fold the tent body tightly and place it in the carry bag. - Ensure that the rainfly and any other components are also packed securely. ### Tent Care and Maintenance - Clean the tent regularly with mild soap and water. - Avoid using harsh chemicals or abrasive cleaners. - Allow the tent to dry completely before storing it. - Store the tent in a cool, dry place away from direct sunlight. ## Cautions 1. **Avoid Uneven or Rocky Surfaces**: Do not place the tent on uneven or rocky surfaces. 2. **Stay Clear of Hazardous Areas**: Avoid setting up the tent near hazardous areas. 3. **No Open Flames or Heat Sources**: Do not use open flames, candles, or any other flammable heat sources near the tent. 4. **Avoid Overloading**: Do not exceed the maximum weight capacity or overload the tent with excessive gear or equipment. 5. **Don't Leave Unattended**: Do not leave the tent unattended while open or occupied. 6. **Avoid Sharp Objects**: Keep sharp objects away from the tent to prevent damage to the fabric or punctures. 7. **Avoid Using Harsh Chemicals**: Do not use harsh chemicals or abrasive cleaners on the tent, as they may damage the material. 8. **Don't Store Wet**: Do not store the tent when it is wet or damp, as it can lead to mold, mildew, or fabric deterioration. 9. **Avoid Direct Sunlight**: Avoid prolonged exposure of the tent to direct sunlight, as it can cause fading or weakening of the fabric. 10. **Don't Neglect Maintenance**: Regularly clean and maintain the tent according to the provided instructions to ensure its longevity and performance. ## Warranty Information Thank you for purchasing the TrailMaster X4 Tent. We are confident in the quality and durability of our product. This warranty provides coverage for any manufacturing defects or issues that may arise during normal use of the tent. Please read the terms and conditions of the warranty below: 1. **Warranty Coverage**: The TrailMaster X4 Tent is covered by a **2-year limited warranty** from the date of purchase. This warranty covers manufacturing defects in materials and workmanship. 2. **What is Covered**: - Seam or fabric tears that occur under normal use and are not a result of misuse or abuse. - Issues with the tent poles, zippers, buckles, or other hardware components that affect the functionality of the tent. - Problems with the rainfly or other included accessories that impact the performance of the tent. 3. **What is Not Covered**: - Damage caused by misuse, abuse, or improper care of the tent. - Normal wear and tear or cosmetic damage that does not affect the functionality of the tent. - Damage caused by extreme weather conditions, natural disasters, or acts of nature. - Any modifications or alterations made to the tent by the user. 4. **Claim Process**: - In the event of a warranty claim, please contact our customer support (contact details provided in the user guide) to initiate the process. - Provide proof of purchase, including the date and place of purchase, along with a detailed description and supporting evidence of the issue. 5. **Resolution Options**: - Upon receipt of the warranty claim, our customer support team will assess the issue and determine the appropriate resolution. - Options may include repair, replacement of the defective parts, or, if necessary, replacement of the entire tent. 6. **Limitations and Exclusions**: - Our warranty is non-transferable and applies only to the original purchaser of the TrailMaster X4 Tent. - The warranty does not cover any incidental or consequential damages resulting from the use or inability to use the tent. - Any unauthorized repairs or alterations void the warranty. ### Contact Information If you have any questions or need further assistance, please contact our customer support: - Customer Support Phone: +1-800-123-4567 - Customer Support Email: support@example.com ## Return Policy - **If Membership status \"None \":** Returns are accepted within 30 days of purchase, provided the tent is unused, undamaged and in its original packaging. Customer is responsible for the cost of return shipping. Once the returned item is received, a refund will be issued for the cost of the item minus a 10% restocking fee. If the item was damaged during shipping or if there is a defect, the customer should contact customer service within 7 days of receiving the item. - **If Membership status \"Gold\":** Returns are accepted within 60 days of purchase, provided the tent is unused, undamaged and in its original packaging. Free return shipping is provided. Once the returned item is received, a full refund will be issued. If the item was damaged during shipping or if there is a defect, the customer should contact customer service within 7 days of receiving the item. - **If Membership status \"Platinum\":** Returns are accepted within 90 days of purchase, provided the tent is unused, undamaged and in its original packaging. Free return shipping is provided, and a full refund will be issued. If the item was damaged during shipping or if there is a defect, the customer should contact customer service within 7 days of receiving the item. ## Reviews 1) **Rating:** 5 **Review:** I am extremely happy with my TrailMaster X4 Tent! It's spacious, easy to set up, and kept me dry during a storm. The UV protection is a great addition too. Highly recommend it to anyone who loves camping! 2) **Rating:** 3 **Review:** I bought the TrailMaster X4 Tent, and while it's waterproof and has a spacious interior, I found it a bit difficult to set up. It's a decent tent, but I wish it were easier to assemble. 3) **Rating:** 5 **Review:** The TrailMaster X4 Tent is a fantastic investment for any serious camper. The easy setup and spacious interior make it perfect for extended trips, and the waterproof design kept us dry in heavy rain. 4) **Rating:** 4 **Review:** I like the TrailMaster X4 Tent, but I wish it came in more colors. It's comfortable and has many useful features, but the green color just isn't my favorite. Overall, it's a good tent. 5) **Rating:** 5 **Review:** This tent is perfect for my family camping trips. The spacious interior and convenient storage pocket make it easy to stay organized. It's also super easy to set up, making it a great addition to our gear. ## FAQ 1) Can the TrailMaster X4 Tent be used in winter conditions? The TrailMaster X4 Tent is designed for 3-season use and may not be suitable for extreme winter conditions with heavy snow and freezing temperatures. 2) How many people can comfortably sleep in the TrailMaster X4 Tent? The TrailMaster X4 Tent can comfortably accommodate up to 4 people with room for their gear. 3) Is there a warranty on the TrailMaster X4 Tent? Yes, the TrailMaster X4 Tent comes with a 2-year limited warranty against manufacturing defects. 4) Are there any additional accessories included with the TrailMaster X4 Tent? The TrailMaster X4 Tent includes a rainfly, tent stakes, guy lines, and a carry bag for easy transport. 5) Can the TrailMaster X4 Tent be easily carried during hikes? Yes, the TrailMaster X4 Tent weighs just 12lbs, and when packed in its carry bag, it can be comfortably carried during hikes."},{"location":"0-Workshop/1-Overview/02/#3-product-catalog-info","title":"3. Product Catalog Info","text":"

This record represents a single product item in the product catalog database, with a unique product ID. The products.csv file contains a collection of these records, representing the entire Contoso Outdoors product catalog at a high level.

Each product ID has a corresponding \"product manual\" record that provides more extensive detail (e.g, in website pages). The product catalog entry itself contains just the {id, name, price, category, brand, description} information required for creating product indexes and searching for matching results (for later retrieval) based on a customer query.

The catalog record below corresponds to the product manual record above.

SAMPLE RECORD (CSV) - click to expand 
id = 1,\nname = TrailMaster X4 Tent,\nprice = 250.0,\ncategory = Tents,\nbrand = OutdoorLiving,\ndescription = \"Unveiling the TrailMaster X4 Tent from \\\n    OutdoorLiving, your home away from home for your next \\\n    camping adventure. Crafted from durable polyester, \\\n    this tent boasts a spacious interior perfect for four \\\n    occupants. It ensures your dryness under drizzly skies \\\n    thanks to its water-resistant construction, and the \\\n    accompanying rainfly adds an extra layer of weather \\\n    protection. It offers refreshing airflow and bug defence, \\\n    courtesy of its mesh panels. Accessibility is not an issue \\\n    with its multiple doors and interior pockets that keep \\\n    small items tidy. Reflective guy lines grant better \\\n    visibility at night, and the freestanding design \\\n    simplifies setup and relocation. With the included \\\n    carry bag, transporting this convenient abode becomes \\\n    a breeze. Be it an overnight getaway or a week-long nature \\\n    escapade, the TrailMaster X4 Tent provides comfort, \\\n    convenience, and concord with the great outdoors. Comes with \\\n    a two-year limited warranty to ensure customer satisfaction.\"\n
"},{"location":"0-Workshop/1-Overview/03/","title":"1.4 RAG Chat Tutorial","text":"

This markmap provides the big picture for navigating this RAG Chat tutorial. Learn to build a minimal RAG-based copilot experience using Azure AI Foundry Portal (for setup) and Azure AI Foundry SDK (for ideation and evaluation).

RAG Chat Tutorial Markmap

# RAG Copilot ## 1. Overview ### 1.0 Pre-Requisites - Azure Subscription (Roles) - GitHub Account (Codespaces) - AI Models (Chat, Embedding) - Application Data (RAG) ### 1.1 Concepts - Generative AI Ops (GenAIOps) - Custom Copilot (Chat AI) - Prompt Template (Asset Format) - Retrieval Augmented Generation (RAG) - AI-Assisted Evaluation (LLM-As-Judge) - [Azure OpenAI Deployment types](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/deployment-types#global-standard) - [Data Residency in Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/data-residency/) ### 1.2 Technologies - Azure AI Foundry Portal - Azure AI Project Resource - Azure AI Hub Resource - Azure AI Search Service - Azure OpenAI Service ### 1.3 Dev Tools - GitHub Codespaces (Dev Env) - Visual Studio Code (Dev IDE) - Azure CLI (Configure) --- ## 2. [Setup](https://learn.microsoft.com/en-us/azure/ai-studio/tutorials/copilot-sdk-create-resources) ### 2.1 Create AI Project ### 2.2 Deploy AI Models ### 2.3 Add Azure AI Search ### 2.4 Setup Local Environment ### 2.5 Configure Env Variables ### 2.6 Validate Env Setup ### 2.7 Connect The Dots - What did we do? - Why did we do it? - How can we improve? --- ## 3. [Ideate](https://learn.microsoft.com/en-us/azure/ai-studio/tutorials/copilot-sdk-build-rag) ### 3.1 Add Application Data ### 3.2 Create Search Index ### 3.3 Retrieve Related Products ### 3.4 Extract Customer Intent ### 3.5 Retrieve Related Knowledge ### 3.6 Design Grounded Prompt ### 3.7 Validate the Prototype ### 3.8 Connect The Dots - What did we do? - Why did we do it? - How can we improve? ## 4. [Evaluate](https://learn.microsoft.com/en-us/azure/ai-studio/tutorials/copilot-sdk-evaluate) ### 4.1 Create Evaluation Dataset ### 4.2 Create Evaluation Script ### 4.3 Configure Evaluation Model ### 4.4 Run Evaluation Script ### 4.5 View Results Locally ### 4.6 View Results in Portal ### 4.7 Validate Evaluation ### 4.8 Connect The Dots ## 5. Evolve ### 5.1 Recap: Build a Copilot ### 5.2 Refactor: Make it Better ### 5.3 Resources: Learn More"},{"location":"0-Workshop/1-Overview/04/","title":"1.5 Contoso Chat Sample","text":"

Once you've familiarized yourself with the basic setup, ideation and evaluation capabilities, you can explore the Contoso Chat sample for more advanced insights, on your own.

This sample uses the same retail dataset used in this RAG Chat workshop, but streamlines the end-to-end development workflow with core developer tools like Azure Developer CLI, to automate provisioning and deployment. The sample also teaches you how to package up your application prototype and deploy it into production using Azure Container Apps.

Contoso Chat is currently in v3. An updated v4 (Jan 2025) will reflect recent Azure AI SDK updates

"},{"location":"0-Workshop/2-Setup/01/","title":"2.1 Create AI Project","text":"

This is Part 1 of the tutorial. This stage is completed USING THE AZURE AI FOUNDRY PORTAL

At the end of this section, you should have provisioned an Azure AI Hub and Azure AI project resource, setup an Azure AI Search resource and deployed two Azure OpenAI models for implementing the RAG-based copilot. You should also have launched GitHub Codespaces and configured your development environment to work with your provisioned Azure infrastructure.

"},{"location":"0-Workshop/2-Setup/01/#1-log-into-azure-ai-foundry","title":"1. Log Into Azure AI Foundry","text":"
  1. Open a private browser and navigate to https://ai.azure.com.
  2. Log in with an active Azure subscription. Note the tenant ID if multi-tenant.
  3. You should see a landing page with a blue \"+ Create project\" button as shown below.

Note: If you had previously created projects, those will be listed as shown above. Don't worry if you don't see any listed for your profile. We are going to create a new project next.

"},{"location":"0-Workshop/2-Setup/01/#2-create-a-new-project","title":"2. Create a new project","text":"

  1. Click the \"+ Create project\" button. You should see a dialog popup like this. Your default project name and hub information will be different and reflect your prior activity.

  2. Change the default project name to something memorable - I used ninarasi-ragchat-v1.

  3. We also want to create a new hub for our new AI project - let's fix that next in the dialog.
"},{"location":"0-Workshop/2-Setup/01/#3-create-new-hub","title":"3. Create new hub","text":"
  1. Click the Create new hub (blue lettering) in dialog above.
  2. Pick a memorable name that reflects the project - I used ninarasi-ragchat-hub
  3. Click \"Next\". It returns you to the previous dialog, with an enhanced view as shown
"},{"location":"0-Workshop/2-Setup/01/#4-customize-the-hub","title":"4. Customize the hub","text":"
  1. Click the \"Customize\" button in that dialog. This lets you customize the defaults as shown.
  2. First, select a relevant Location with relevant model quota - I used East US2
  3. Next,customize the resource group name to be memorable - I used ninarasi-ragchat-rg
  4. Next, click \"Create new AI Search\" (blue lettering) in the dialog to trigger a new pop-up
"},{"location":"0-Workshop/2-Setup/01/#5-create-new-ai-search","title":"5. Create new AI Search","text":"
  1. Customize resource name - I used ninarasi-ragchat-aisearch - then hit Next.
  2. You return to the Create a project wizard - hit Next to get to review.
  3. Review the details one last time - hit Create to confirm AI project creation.
  4. Creation takes a few minutes - all elements will show green on success.
"},{"location":"0-Workshop/2-Setup/01/#6-review-created-ai-project","title":"6. Review Created AI Project","text":"
  1. You should automatically be taken to the AI Project overview page as shown. Note the Project connection string under Project details - we'll revisit it later.
  2. Click on the Open in management center link (highlighted in red) - it takes you to this Management Center view. Clicking Go to project will take you back to the AI project.
  3. However, for now click on the Connected Resources option in the sidebar. This lets us see which resources can be accessed via the Project connection we noted earlier. Verify that Azure AI Search is one of the listed resources.

CONGRATULATIONS! You created your Azure AI Hub & Project resources

"},{"location":"0-Workshop/2-Setup/02/","title":"2.2 Deploy AI Models","text":"

Let's revisit our Retrieval Augmented Generation design pattern. Note that we make use of two models to implement this design architecture.

  1. A Large Language Model (Embedding) for vectorizing the user query.
  2. A Large Language Model (Chat) for generating the response returned to user.

Let's find the right models to use and deploy them to our Azure AI project so we can use them in our RAG-based copilot implementation. Start by navigating to the Azure AI Project overview page. Then select the Models + endpoints link under My assets.

"},{"location":"0-Workshop/2-Setup/02/#1-deploy-chat-model","title":"1. Deploy Chat Model","text":""},{"location":"0-Workshop/2-Setup/02/#11-select-a-chat-model","title":"1.1. Select A Chat Model","text":"

  1. Click the blue \"Deploy model\" button and pick Deploy base model from the dropdown options.

  2. If you know the specific model to use, you can search for it here. In our case, let's look at what our options are. First, click the Collections filter and select Azure OpenAI. Next, click the Inference tasks filter and select Chat completion. We can see that this reduces our choices from 1800+ models in the Azure AI model catalog to 9 matching models.

"},{"location":"0-Workshop/2-Setup/02/#12-deploy-the-chat-model","title":"1.2. Deploy the Chat Model","text":"
  1. We can pick any of those options as shown above, to see a Model Card with more details. Let's pick gpt-4o-mini and click Confirm to get this deployment wizard. Note that it selects a Global Standard deployment type by default. The deployment has a default capacity of 10K tokens per minute (TPM) which can be useful when we begin the evaluation phase.
  1. Click on the dropdown to see other Deployment type options. Read the documentation to learn more about what each provides. Global Standard is the recommended starting place for customers so let's go with that.
"},{"location":"0-Workshop/2-Setup/02/#13-verify-deployment","title":"1.3. Verify Deployment","text":"

On successful deployment, you will be taken to the model deployment page where you can review the details and retrieve relevant information like the Endpoint and Key information, for use with code-first clients.

You can Open in playground and use the Azure AI Portal as an ideation tool to explore different prompt templates, model configurations and multi-turn conversational approaches to determine if this model is in fact suitable for your app scenario.

Task: Ask the model to Tell me about hiking boots for my trip to Spain - is response grounded?

Homework: Complete the Deploy an enterprise chat app tutorial with your data. Is response grounded now?

"},{"location":"0-Workshop/2-Setup/02/#14-view-metrics","title":"1.4. View Metrics","text":"

You can also select the Metrics tab of deployed models within an Azure AI project to get metrics about the cost (tokens) and performance (requests) of that model in a given time frame. The screenshot shows the data taken for this model after completing the entire project. The spike reflects the requests made during the evaluation stage of the workflow.

You can also click the Open in Azure Monitor link to open up the Azure Monitor dashboard in the Azure Portal as shown below, allowing you to drill down into various metrics or establish dashboards to monitor trends of interest.

"},{"location":"0-Workshop/2-Setup/02/#2-deploy-embedding-model","title":"2. Deploy Embedding Model","text":"

The previous steps focused on the chat completion model which has many choices for us to select from. Now, let's look at embeddings.

"},{"location":"0-Workshop/2-Setup/02/#21-select-deploy-model","title":"2.1. Select & Deploy Model","text":"

Start by setting the Inference Task to Embeddings.

You'll find there are only about 11 models that match this filter - setting the Collection to Azure OpenAI reduces this further to 5. As before, let's select a model and review the card to see if it matches our needs.

Then Confirm to get the Deployment wizard dialog.

And Deploy to complete the workflow.

"},{"location":"0-Workshop/2-Setup/02/#23-verify-deployment","title":"2.3. Verify Deployment","text":"

Similarly, we can use the deployment card to view deployment details and explore metrics. But note that we don't have a Playground for embeddings.

"},{"location":"0-Workshop/2-Setup/02/#3-deployment-complete","title":"3. Deployment Complete","text":"

The Azure AI Project overview page will now list both models under the Models + endpoints tab for easy lookup later (if you want to explore metrics or engage in Playground).

CONGRATULATIONS! You deployed both the AI models needed for RAG

"},{"location":"0-Workshop/2-Setup/03/","title":"2.3 Add Azure AI Search","text":"

The official tutorial recommends setting up an Azure AI Search resource at this stage. This was done under the assumption that the default Azure AI project setup did not create (new) or reuse (existing) Azure AI Search resources.

However, since we opted to add Azure AI Search during project setup, we have nothing further to do at this stage! Note that the search resource is not yet populated with our data (search indexes). We'll get there in the Ideate section.

CONGRATULATIONS! You completed setup of Azure AI Search indexes for your data

"},{"location":"0-Workshop/2-Setup/04/","title":"2.4 Add App Insights","text":""},{"location":"0-Workshop/2-Setup/04/#tracing-observability","title":"Tracing & Observability","text":"

While not covered explicitly in the tutorial, we will be working with code in the Ideate section that is instrumented for observability (tracing).

  • See: How to Trace your application with Azure AI Inference SDK

In order to visualize those traces in the Azure AI Foundry Portal, we need to attach an Application Insights resource to our Azure AI project ahead of time.

  • See: View your traces in Azure AI Foundry Portal
"},{"location":"0-Workshop/2-Setup/04/#create-new-app-insights","title":"Create New App Insights","text":"

Let's follow those steps (as illustrated in the animated gif below):

  1. Navigate to your Azure AI Project resource in the Azure AI Foundry portal
  2. Select the Tracing option in the menu sidebar
  3. Select Create New to attach a new Application Insights resource to the project
  4. Provide a name and select Create.

CONGRATULATIONS! You activated App Insights for tracing your Azure AI project

"},{"location":"0-Workshop/2-Setup/05/","title":"2.5 Setup Local Environment","text":"

The previous steps completed the setup of our Azure AI infrastructure (resources). Now it's time to setup our development environment to talk to our Azure backend.

"},{"location":"0-Workshop/2-Setup/05/#1-launch-github-codespaces","title":"1. Launch GitHub Codespaces","text":"

If you had not previously done so, complete the Getting Started steps now.

  1. Fork this repository to your personal GitHub profile.
  2. Open the fork in a new browser tab.
  3. Click on the blue \"Code\" button and select Codespaces
  4. Click on the Create Codespaces on Main button

You should see GitHub Codespaces launch in a new browser tab.

  • It will take a few minutes to complete loading.
  • You should see a Visual Studio Code IDE in the browser
  • When ready, you should see a VS Code terminal with active prompt.
"},{"location":"0-Workshop/2-Setup/05/#2-verify-azure-cli-installed","title":"2. Verify Azure CLI Installed","text":"

The repository is configured with a devcontainer that has all necessary dependencies pre-installed. Let's verify the az (Azure Developer CLI) was installed.

az version\n
"},{"location":"0-Workshop/2-Setup/05/#3-authenticate-with-azure-cli","title":"3. Authenticate with Azure CLI","text":"

Log into your Azure subscription from the VS Code terminal in GitHub Codespaces using the following command, and follow the prompts to complete the workflow.

az login --use-device-code\n

If you have a multi-tenancy account, you can set the default tenant when logging in as follows, where <TENANTID> is replaced with the relevant identifier.

az login --use-device-code --tenant <TENANTID>\n
"},{"location":"0-Workshop/2-Setup/05/#4-verify-python-packages","title":"4. Verify Python Packages","text":"

The codebase is set up with a requirements.txt file that has all the necesary Python package dependencies listed. These are auto-installed into the devcontainer at launch. Use pip list | grep <KEYWORD> to verify if specific packages were installed.

For instance use this command to list azure packages installed and verify they match the ones listed in requirements (e.g., look for azure-ai-projects, azure-ai-inference, azure-ai-identity, azure-search-documents, azure-core, azure-ai-evaluation)

pip list | grep azure\n
"},{"location":"0-Workshop/2-Setup/06/","title":"2.6 Setup Project Structure","text":"

This repository contains the following structure to start with. The *.sample folders or files are there for reference only, so you can check your work.

data/            # Contains application data (initial)\ndocs/            # Contains docs and guides (content)\nsrc.sample/      # Sample src/ folder\n.env.sample       # Sample .env file\n
"},{"location":"0-Workshop/2-Setup/06/#1-define-src-folder-for-code","title":"1. Define src/ folder for code","text":"

In this workshop, start by creating a new src/ folder and populating it from scratch to get a sense for the development workflow. Start by creating this folder structure:

mkdir src/\nmkdir src/api\nmkdir src/api/assets\n

Your directory structure should now look like this:

data/\ndocs/\nsrc.sample/\n.env.sample\nsrc/\nsrc/api\nsrc/api/assets\n
"},{"location":"0-Workshop/2-Setup/06/#2-add-srcconfigpy-helper-script","title":"2. Add src/config.py helper script","text":"

For convenience, let's copy this from the sample location - then review the code to see what it does. Use this command at the root of the repo:

cp src.sample/api/config.py src/api/.\n

Expand the code below to get a sense of what this helper does.

  1. Sets the ASSET_PATH to the assets/ folder in the same directory.
  2. Configures the app logger and enables telemetry logging (traces) for app.
Click to expand and view the helper script src/api/config.py
# ruff: noqa: ANN201, ANN001\nimport os\nimport sys\nimport pathlib\nimport logging\nfrom azure.identity import DefaultAzureCredential\nfrom azure.ai.projects import AIProjectClient\nfrom azure.ai.inference.tracing import AIInferenceInstrumentor\n\n# load environment variables from the .env file\nfrom dotenv import load_dotenv\n\nload_dotenv()\n\n# Set \"./assets\" as the path where assets are stored, resolving the absolute path:\nASSET_PATH = pathlib.Path(__file__).parent.resolve() / \"assets\"\n\n# Configure an root app logger that prints info level logs to stdout\nlogger = logging.getLogger(\"app\")\nlogger.setLevel(logging.INFO)\nlogger.addHandler(logging.StreamHandler(stream=sys.stdout))\n\n\n# Returns a module-specific logger, inheriting from the root app logger\ndef get_logger(module_name):\n    return logging.getLogger(f\"app.{module_name}\")\n\n\n# Enable instrumentation and logging of telemetry to the project\ndef enable_telemetry(log_to_project: bool = False):\n    AIInferenceInstrumentor().instrument()\n\n    # enable logging message contents\n    os.environ[\"AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED\"] = \"true\"\n\n    if log_to_project:\n        from azure.monitor.opentelemetry import configure_azure_monitor\n\n        project = AIProjectClient.from_connection_string(\n            conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n        )\n        tracing_link = f\"https://ai.azure.com/tracing?wsid=/subscriptions/{project.scope['subscription_id']}/resourceGroups/{project.scope['resource_group_name']}/providers/Microsoft.MachineLearningServices/workspaces/{project.scope['project_name']}\"\n        application_insights_connection_string = project.telemetry.get_connection_string()\n        if not application_insights_connection_string:\n            logger.warning(\n                \"No application insights configured, telemetry will not be logged to project. Add application insights at:\"\n            )\n            logger.warning(tracing_link)\n\n            return\n\n        configure_azure_monitor(connection_string=application_insights_connection_string)\n        logger.info(\"Enabled telemetry logging to project, view traces at:\")\n        logger.info(tracing_link)\n

CONGRATULATIONS! Your development environment is ready to use.

"},{"location":"0-Workshop/2-Setup/07/","title":"2.7 Configure Env Variables","text":"

We are now ready to start coding the chat AI application in our local development environment. But to do this, we need to configure a few environment variables.

"},{"location":"0-Workshop/2-Setup/07/#1-create-env-file","title":"1. Create .env file","text":"
  1. Start by copying the .env.sample file to .env
cp .env.sample .env\n
  1. Let's review what this contains
cat .env\n

You will see something like this:

AIPROJECT_CONNECTION_STRING=<your-connection-string>\nAISEARCH_INDEX_NAME=\"contoso-products\"\nEMBEDDINGS_MODEL=\"text-embedding-ada-002\"\nINTENT_MAPPING_MODEL=\"gpt-4o-mini\"\nCHAT_MODEL=\"gpt-4o-mini\"\nEVALUATION_MODEL=\"gpt-4o-mini\"\n
"},{"location":"0-Workshop/2-Setup/07/#2-update-connection-string","title":"2. Update Connection String","text":"

Note that defaults are provided for everything except the AIPROJECT_CONNECTION_STRING - let's fix that now!

  1. Open the Azure AI Project overview page. It should look like this:

  2. Look for the Project connection string under the Project details tab.

  3. Copy that into .env as the AIPROJECT_CONNECTION_STRING value.
  4. Save the changes to .env
"},{"location":"0-Workshop/2-Setup/07/#3-review-environment-variables","title":"3. Review Environment Variables","text":"

Let's review our environment variables:

  1. AIPROJECT_CONNECTION_STRING - is a single connection URI that allows access to all the Connected Resources in the Azure AI project (including Azure AI Search).
  2. AISEARCH_INDEX_NAME - is set to contoso-products and represents the index name that we will create and populate with product catalog data.
  3. EMBEDDINGS_MODEL - the deployed model we'll use for vectorizing queries (see: Ideate 3.2)
  4. INTENT_MAPPING_MODEL - the deployed model we'll use for intent mapping (see: Ideate 3.4)
  5. CHAT_MODEL - the deployed model we'll use for final chat response (see: Ideate 3.6)
  6. EVALUATION_MODEL - the deployed model we'll use for quality evaluation (see: Evaluate 4.3)

CONGRATULATIONS! Your local environment is configured for code!

"},{"location":"0-Workshop/3-Ideate/01/","title":"3.1 Add Application Data","text":"

This is Part 2 of the tutorial. This stage is completed USING THE AZURE AI FOUNDRY SDK.

At the end of this section, you should have created an Azure AI Search index based on the retailer's product data, written and tested scripts to retrieve relevant product documents based on a user query, and implemented a chat AI that uses this knowledge as grounding context for a chat completion model. You will also get your first look at observability support with tracing.

"},{"location":"0-Workshop/3-Ideate/01/#1-add-productscsv-to-assets","title":"1. Add products.csv to assets/","text":"

We want to ground chat AI responses in our application data. In this case, we want to respond to customer queries about our products by grounding the responses in items found in our catalog.

HOMEWORK: Think of other retail data sources that might be useful (and how to add them)

Recall that the src/config.py script identifies the assets/ folder as the source for all static assets. We already created the src/api/assets earlier. Let's copy in the product catalog data into this folder now.

cp src.sample/api/assets/products.csv  src/api/assets/\n
"},{"location":"0-Workshop/3-Ideate/01/#2-inspect-productscsv-data","title":"2. Inspect products.csv data","text":"

Let's take a quick look at what this file contains in terms of product catalog data.

  1. We have 20 product records listed in CSV format.
  2. Each product has an id, name, price, category, brand, and text description
  3. The product description provides the most content about the product
  4. The product name is a unique title for that product.
Click to expand and view the Product Catalog data products.csv
id,name,price,category,brand,description\n1,TrailMaster X4 Tent,250.0,Tents,OutdoorLiving,\"Unveiling the TrailMaster X4 Tent from OutdoorLiving, your home away from home for your next camping adventure. Crafted from durable polyester, this tent boasts a spacious interior perfect for four occupants. It ensures your dryness under drizzly skies thanks to its water-resistant construction, and the accompanying rainfly adds an extra layer of weather protection. It offers refreshing airflow and bug defence, courtesy of its mesh panels. Accessibility is not an issue with its multiple doors and interior pockets that keep small items tidy. Reflective guy lines grant better visibility at night, and the freestanding design simplifies setup and relocation. With the included carry bag, transporting this convenient abode becomes a breeze. Be it an overnight getaway or a week-long nature escapade, the TrailMaster X4 Tent provides comfort, convenience, and concord with the great outdoors. Comes with a two-year limited warranty to ensure customer satisfaction.\"\n2,Adventurer Pro Backpack,90.0,Backpacks,HikeMate,\"Venture into the wilderness with the HikeMate's Adventurer Pro Backpack! Uniquely designed with ergonomic comfort in mind, this backpack ensures a steadfast journey no matter the mileage. It boasts a generous 40L capacity wrapped up in durable nylon fabric ensuring its long-lasting performance on even the most rugged pursuits. It's meticulously fashioned with multiple compartments and pockets for organized storage, hydration system compatibility, and adjustable padded shoulder straps all in a lightweight construction. The added features of a sternum strap and hip belt enhance stability without compromising on comfort. The Adventurer Pro Backpack also prioritizes your safety with its reflective accents for when night falls. This buoyant beauty does more than carry your essentials; it carries the promise of a stress-free adventure!\"\n3,Summit Breeze Jacket,120.0,Hiking Clothing,MountainStyle,\"Discover the joy of hiking with MountainStyle's Summit Breeze Jacket. This lightweight jacket is your perfect companion for outdoor adventures. Sporting a trail-ready, windproof design and a water-resistant fabric, it's ready to withstand any weather. The breathable polyester material and adjustable cuffs keep you comfortable, whether you're ascending a mountain or strolling through a park. And its sleek black color adds style to function. The jacket features a full-zip front closure, adjustable hood, and secure zippered pockets. Experience the comfort of its inner lining and the convenience of its packable design. Crafted for night trekkers too, the jacket has reflective accents for enhanced visibility. Rugged yet chic, the Summit Breeze Jacket is more than a hiking essential, it's the gear that inspires you to reach new heights. Choose adventure, choose the Summit Breeze Jacket.\"\n4,TrekReady Hiking Boots,140.0,Hiking Footwear,TrekReady,\"Introducing the TrekReady Hiking Boots - stepping up your hiking game, one footprint at a time! Crafted from leather, these stylistic Trailmates are made to last. TrekReady infuses durability with its reinforced stitching and toe protection, making sure your journey is never stopped short. Comfort? They have that covered too! The boots are a haven with their breathable materials, cushioned insole, with padded collar and tongue; all nestled neatly within their lightweight design. As they say, it's what's inside that counts - so inside you'll find a moisture-wicking lining that quarantines stank and keeps your feet fresh as that mountaintop breeze. Remember the fear of slippery surfaces? With these boots, you can finally tell it to 'take a hike'! Their shock-absorbing midsoles and excellent traction capabilities promise stability at your every step. Beautifully finished in a traditional lace-up system, every adventurer deserves a pair of TrekReady Hiking Boots. Hike more, worry less!\"\n5,BaseCamp Folding Table,60.0,Camping Tables,CampBuddy,\"CampBuddy's BaseCamp Folding Table is an adventurer's best friend. Lightweight yet powerful, the table is a testament to fun-meets-function and will elevate any outing to new heights. Crafted from resilient, rust-resistant aluminum, the table boasts a generously sized 48 x 24 inches tabletop, perfect for meal times, games and more. The foldable design is a godsend for on-the-go explorers. Adjustable legs rise to the occasion to conquer uneven terrains and offer height versatility, while the built-in handle simplifies transportation. Additional features like non-slip feet, integrated cup holders and mesh pockets add a pinch of finesse. Quick to set up without the need for extra tools, this table is a silent yet indispensable sidekick during camping, picnics, and other outdoor events. Don't miss out on the opportunity to take your outdoor experiences to a new level with the BaseCamp Folding Table. Get yours today and embark on new adventures tomorrow! \"\n6,EcoFire Camping Stove,80.0,Camping Stoves,EcoFire,\"Introducing EcoFire's Camping Stove, your ultimate companion for every outdoor adventure! This portable wonder is precision-engineered with a lightweight and compact design, perfect for capturing that spirit of wanderlust. Made from high-quality stainless steel, it promises durability and steadfast performance. This stove is not only fuel-efficient but also offers an easy, intuitive operation that ensures hassle-free cooking. Plus, it's flexible, accommodating a variety of cooking methods whether you're boiling, grilling, or simmering under the starry sky. Its stable construction, quick setup, and adjustable flame control make cooking a breeze, while safety features protect you from any potential mishaps. And did we mention it also includes an effective wind protector and a carry case for easy transportation? But that's not all! The EcoFire Camping Stove is eco-friendly, designed to minimize environmental impact. So get ready to enhance your camping experience and enjoy delicious outdoor feasts with this unique, versatile stove!\"\n7,CozyNights Sleeping Bag,100.0,Sleeping Bags,CozyNights,\"Embrace the great outdoors in any season with the lightweight CozyNights Sleeping Bag! This durable three-season bag is superbly designed to give hikers, campers, and backpackers comfort and warmth during spring, summer, and fall. With a compact design that folds down into a convenient stuff sack, you can whisk it away on any adventure without a hitch. The sleeping bag takes comfort seriously, featuring a handy hood, ample room and padding, and a reliable temperature rating. Crafted from high-quality polyester, it ensures long-lasting use and can even be zipped together with another bag for shared comfort. Whether you're gazing at stars or catching a quick nap between trails, the CozyNights Sleeping Bag makes it a treat. Don't just sleep\u2014 dream with CozyNights.\"\n8,Alpine Explorer Tent,350.0,Tents,AlpineGear,\"Welcome to the joy of camping with the Alpine Explorer Tent! This robust, 8-person, 3-season marvel is from the responsible hands of the AlpineGear brand. Promising an enviable setup that is as straightforward as counting sheep, your camping experience is transformed into a breezy pastime. Looking for privacy? The detachable divider provides separate spaces at a moment's notice. Love a tent that breathes? The numerous mesh windows and adjustable vents fend off any condensation dragon trying to dampen your adventure fun. The waterproof assurance keeps you worry-free during unexpected rain dances. With a built-in gear loft to stash away your outdoor essentials, the Alpine Explorer Tent emerges as a smooth balance of privacy, comfort, and convenience. Simply put, this tent isn't just a shelter - it's your second home in the heart of nature! Whether you're a seasoned camper or a nature-loving novice, this tent makes exploring the outdoors a joyous journey.\"\n9,SummitClimber Backpack,120.0,Backpacks,HikeMate,\"Adventure waits for no one! Introducing the HikeMate SummitClimber Backpack, your reliable partner for every exhilarating journey. With a generous 60-liter capacity and multiple compartments and pockets, packing is a breeze. Every feature points to comfort and convenience; the ergonomic design and adjustable hip belt ensure a pleasantly personalized fit, while padded shoulder straps protect you from the burden of carrying. Venturing into wet weather? Fear not! The integrated rain cover has your back, literally. Stay hydrated thanks to the backpack's hydration system compatibility. Travelling during twilight? Reflective accents keep you visible in low-light conditions. The SummitClimber Backpack isn't merely a carrier; it's a wearable base camp constructed from ruggedly durable nylon and thoughtfully designed for the great outdoors adventurer, promising to withstand tough conditions and provide years of service. So, set off on that quest - the wild beckons! The SummitClimber Backpack - your hearty companion on every expedition!\"\n10,TrailBlaze Hiking Pants,75.0,Hiking Clothing,MountainStyle,\"Meet the TrailBlaze Hiking Pants from MountainStyle, the stylish khaki champions of the trails. These are not just pants; they're your passport to outdoor adventure. Crafted from high-quality nylon fabric, these dapper troopers are lightweight and fast-drying, with a water-resistant armor that laughs off light rain. Their breathable design whisks away sweat while their articulated knees grant you the flexibility of a mountain goat. Zippered pockets guard your essentials, making them a hiker's best ally. Designed with durability for all your trekking trials, these pants come with a comfortable, ergonomic fit that will make you forget you're wearing them. Sneak a peek, and you are sure to be tempted by the sleek allure that is the TrailBlaze Hiking Pants. Your outdoors wardrobe wouldn't be quite complete without them.\"\n11,TrailWalker Hiking Shoes,110.0,Hiking Footwear,TrekReady,\"Meet the TrekReady TrailWalker Hiking Shoes, the ideal companion for all your outdoor adventures. Constructed with synthetic leather and breathable mesh, these shoes are tough as nails yet surprisingly airy. Their cushioned insoles offer fabulous comfort for long hikes, while the supportive midsoles and traction outsoles with multidirectional lugs ensure stability and excellent grip. A quick-lace system, padded collar and tongue, and reflective accents make these shoes a dream to wear. From combating rough terrain with the reinforced toe cap and heel, to keeping off trail debris with the protective mudguard, the TrailWalker Hiking Shoes have you covered. These waterproof warriors are made to endure all weather conditions. But they're not just about being rugged, they're light as a feather too, minimizing fatigue during epic hikes. Each pair can be customized for a perfect fit with removable insoles and availability in multiple sizes and widths. Navigate hikes comfortably and confidently with the TrailWalker Hiking Shoes. Adventure, here you come!\"\n12,TrekMaster Camping Chair,50.0,Camping Tables,CampBuddy,\"Gravitate towards comfort with the TrekMaster Camping Chair from CampBuddy. This trusty outdoor companion boasts sturdy construction using high-quality materials that promise durability and enjoyment for seasons to come. Impeccably lightweight and portable, it's designed to be your go-to seat whether you're camping, at a picnic, cheering at a sporting event, or simply relishing in your backyard pleasures. Beyond its foldable design ensuring compact storage and easy transportation, its ergonomic magic is in the details. An adjustable recline, padded seat and backrest, integrated cup holder, and side pockets ensure the greatest outdoor comfort. Weather resistant, easy to clean, and capable of supporting diverse body types, this versatile chair also comes with a carry bag, ready for your next adventure.\"\n13,PowerBurner Camping Stove,100.0,Camping Stoves,PowerBurner,\"Unleash your inner explorer with the PowerBurner Dual Burner Camping Stove. It's designed for the adventurous heart, with sturdy construction and a high heat output that makes boiling and cooking a breeze. This stove isn't just about strength\u2014it's got finesse too. With adjustable flame control, you can simmer, saut\u00e9, or sizzle with absolute precision. Its compact design and integrated carrying handle make transportation effortless. Moreover, it's crafted to defy the elements, boasting a wind-resistant exterior and piezo ignition system for quick, reliable starts. And when the cooking's done, its removable grates make cleanup swift and easy. Rugged, versatile and reliable, the PowerBurner marks a perfect blend of practicality and performance. So, why wait? Let's turn up the heat on your outdoor culinary adventures today.\"\n14,MountainDream Sleeping Bag,130.0,Sleeping Bags,MountainDream,\"Meet the MountainDream Sleeping Bag: your new must-have companion for every outdoor adventure. Designed to handle 3-season camping with ease, it comes equipped with a premium synthetic insulation that will keep you cozy even when temperatures fall down to 15\u00b0F! Sporting a durable water-resistant nylon shell and soft breathable polyester lining, this bag doesn't sacrifice comfort for toughness. The star of the show is the contoured mummy shape that not only provides optimal heat retention but also cuts down on the weight. A smooth, snag-free YKK zipper with a unique anti-snag design allows for hassle-free operation, while the adjustable hood and full-length zipper baffle work together to ensure you stay warm all night long. Need to bring along some essentials? Not to worry! There's an interior pocket just for that. And when it's time to pack up? Just slip it into the included compression sack for easy storage and transport. Whether you're a backpacking pro or a camping novice, the MountainDream Sleeping Bag is the perfect blend of durability, warmth, and comfort that you've been looking for.\"\n15,SkyView 2-Person Tent,200.0,Tents,OutdoorLiving,\"Introducing the OutdoorLiving SkyView 2-Person Tent, a perfect companion for your camping and hiking adventures. This tent offers a spacious interior that houses two people comfortably, with room to spare. Crafted from durable waterproof materials to shield you from the elements, it is the fortress you need in the wild. Setup is a breeze thanks to its intuitive design and color-coded poles, while two large doors allow for easy access. Stay organized with interior pockets, and store additional gear in its two vestibules. The tent also features mesh panels for effective ventilation, and it comes with a rainfly for extra weather protection. Light enough for on-the-go adventurers, it packs compactly into a carrying bag for seamless transportation. Reflective guy lines ensure visibility at night for added safety, and the tent stands freely for versatile placement. Experience the reliability of double-stitched seams that guarantee increased durability, and rest easy under the stars with OutdoorLiving's SkyView 2-Person Tent. It's not just a tent; it's your home away from home.\"\n16,TrailLite Daypack,60.0,Backpacks,HikeMate,\"Step up your hiking game with HikeMate's TrailLite Daypack. Built for comfort and efficiency, this lightweight and durable backpack offers a spacious main compartment, multiple pockets, and organization-friendly features all in one sleek package. The adjustable shoulder straps and padded back panel ensure optimal comfort during those long exhilarating treks. Course through nature without worry as the daypack's water-resistant fabric protects your essentials from unexpected showers. Plus, never run dry with the integrated hydration system. And did we mention it comes in a plethora of colors and designs? So you can choose one that truly speaks to your outdoorsy soul! Keeping your visibility in mind, we've added reflective accents that light up in low-light conditions. Don't just carry a backpack, adorn a companion that takes you a step ahead in your adventures. Trust the TrailLite Daypack for a hassle-free, enjoyable hiking experience.\"\n17,RainGuard Hiking Jacket,110.0,Hiking Clothing,MountainStyle,\"Introducing the MountainStyle RainGuard Hiking Jacket - the ultimate solution for weatherproof comfort during your outdoor undertakings! Designed with waterproof, breathable fabric, this jacket promises an outdoor experience that's as dry as it is comfortable. The rugged construction assures durability, while the adjustable hood provides a customizable fit against wind and rain. Featuring multiple pockets for safe, convenient storage and adjustable cuffs and hem, you can tailor the jacket to suit your needs on-the-go. And, don't worry about overheating during intense activities - it's equipped with ventilation zippers for increased airflow. Reflective details ensure visibility even during low-light conditions, making it perfect for evening treks. With its lightweight, packable design, carrying it inside your backpack requires minimal effort. With options for men and women, the RainGuard Hiking Jacket is perfect for hiking, camping, trekking and countless other outdoor adventures. Don't let the weather stand in your way - embrace the outdoors with MountainStyle RainGuard Hiking Jacket!\"\n18,TrekStar Hiking Sandals,70.0,Hiking Footwear,TrekReady,\"Meet the TrekStar Hiking Sandals from TrekReady - the ultimate trail companion for your feet. Designed for comfort and durability, these lightweight sandals are perfect for those who prefer to see the world from a hiking trail. They feature adjustable straps for a snug, secure fit, perfect for adapting to the contours of your feet. With a breathable design, your feet will stay cool and dry, escaping the discomfort of sweaty hiking boots on long summer treks. The deep tread rubber outsole ensures excellent traction on any terrain, while the cushioned footbed promises enhanced comfort with every step. For those wild and unpredictable trails, the added toe protection and shock-absorbing midsole protect your feet from rocky surprises. Ingeniously, the removable insole makes for easy cleaning and maintenance, extending the lifespan of your sandals. Available in various sizes and a handsome brown color, the versatile TrekStar Hiking Sandals are just as comfortable on a casual walk in the park as they are navigating rocky slopes. Explore more with TrekReady!\"\n19,Adventure Dining Table,90.0,Camping Tables,CampBuddy,\"Discover the joy of outdoor adventures with the CampBuddy Adventure Dining Table. This feature-packed camping essential brings both comfort and convenience to your memorable trips. Made from high-quality aluminum, it promises long-lasting performance, weather resistance, and easy maintenance - all key for the great outdoors! It's light, portable, and comes with adjustable height settings to suit various seating arrangements and the spacious surface comfortably accommodates meals, drinks, and other essentials. The sturdy yet lightweight frame holds food, dishes, and utensils with ease. When it's time to pack up, it fold and stows away with no fuss, ready for the next adventure!  Perfect for camping, picnics, barbecues, and beach outings - its versatility shines as brightly as the summer sun! Durable, sturdy and a breeze to set up, the Adventure Dining Table will be a loyal companion on every trip. Embark on your next adventure and make lifetime memories with CampBuddy. As with all good experiences, it'll leave you wanting more! \"\n20,CompactCook Camping Stove,60.0,Camping Stoves,CompactCook,\"Step into the great outdoors with the CompactCook Camping Stove, a convenient, lightweight companion perfect for all your culinary camping needs. Boasting a robust design built for harsh environments, you can whip up meals anytime, anywhere. Its wind-resistant and fuel-versatile features coupled with an efficient cooking performance, ensures you won't have to worry about the elements or helpless taste buds while on adventures. The easy ignition technology and adjustable flame control make cooking as easy as a walk in the park, while its compact, foldable design makes packing a breeze. Whether you're camping with family or hiking solo, this reliable, portable stove is an essential addition to your gear. With its sturdy construction and safety-focused design, the CompactCook Camping Stove is a step above the rest, providing durability, quality, and peace of mind. Be wild, be free, be cooked for with the CompactCook Camping Stove!\"\n
"},{"location":"0-Workshop/3-Ideate/02/","title":"3.2 Create Search Index","text":""},{"location":"0-Workshop/3-Ideate/02/#1-create-search-index-script","title":"1. Create Search Index Script","text":"

Let's copy over the create-search-index.py script into our application source folder.

cp src.sample/api/create-search-index.py  src/api/.\n
"},{"location":"0-Workshop/3-Ideate/02/#2-understand-index-creation","title":"2. Understand Index Creation","text":"

Now, let's take a look at what this does.

Click to expand and view the Python Script to create the search index src/api/create-search-index.py
    import os\n    from azure.ai.projects import AIProjectClient\n    from azure.ai.projects.models import ConnectionType\n    from azure.identity import DefaultAzureCredential\n    from azure.core.credentials import AzureKeyCredential\n    from azure.search.documents import SearchClient\n    from azure.search.documents.indexes import SearchIndexClient\n    from config import get_logger\n\n    # initialize logging object\n    logger = get_logger(__name__)\n\n    # create a project client using environment variables loaded from the .env file\n    project = AIProjectClient.from_connection_string(\n        conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n    )\n\n    # create a vector embeddings client that will be used to generate vector embeddings\n    embeddings = project.inference.get_embeddings_client()\n\n    # use the project client to get the default search connection\n    search_connection = project.connections.get_default(\n        connection_type=ConnectionType.AZURE_AI_SEARCH, include_credentials=True\n    )\n\n    # Create a search index client using the search connection\n    # This client will be used to create and delete search indexes\n    index_client = SearchIndexClient(\n        endpoint=search_connection.endpoint_url, credential=AzureKeyCredential(key=search_connection.key)\n    )\n

First the script sets up a search index_client:

  1. Creates an Azure AI Project Client instance (configured with connection string)
  2. Retrieves an embeddings inference client from the AI project (maps to that model)
  3. Retrieves a search_connection object from the AI project instance
  4. Creates an index_client search index client using the search connection (key, endpoint)

First it defines the index based on a vector derived from product data fields.

  1. It maps product name to a title property
  2. It maps product description to a content property
  3. It uses HNSW algorithm (cosine distance) for similarity
  4. It prioritizes \"content\" for semantic ranking

It then creates the index from CSV and populates it using the index_client.

  1. It defines an index using the specified name and embeddings model
  2. It loads CSV and generates vector embeddings for each description
  3. It uploads each vectorized document into the pre-defined search index
"},{"location":"0-Workshop/3-Ideate/02/#3-run-index-creation-script","title":"3. Run Index Creation Script","text":"

To get the index created in Azure AI Search, run the script described above.

python create_search_index.py\n
"},{"location":"0-Workshop/3-Ideate/02/#4-verify-search-index","title":"4. Verify Search Index","text":"

Then verify that the index was created successfully:

  1. Visit the Azure Portal and look up your Resource Group
  2. Visit the Azure AI Search resource page from that RG
  3. Click on \"Search Explorer\" from the resource overview page
  4. Click \"Search\" - verify that you see indexed products
"},{"location":"0-Workshop/3-Ideate/03/","title":"3.3 Retrieve Related Products","text":""},{"location":"0-Workshop/3-Ideate/03/#1-add-docs-retrieval-script","title":"1. Add Docs Retrieval Script","text":"

Let's copy over the get_product_documents.py script into our application source folder.

cp src.sample/api/get_product_documents.py  src/api/.\n
"},{"location":"0-Workshop/3-Ideate/03/#2-understand-docs-retrieval","title":"2. Understand Docs Retrieval","text":"

Let's start with a sample user query like this:

 I need a new tent for 4 people, what would you recommend?\n

Different users could phrase the question in different ways, with different levels of information. But we need to map all these queries to a search query that works on the product database. How do we do that? We use a 3-step process:

  1. We teach an AI to extract user intent from an input text query
  2. We teach the AI to map user intent to a search query on products
  3. We use the search index to retrieve product documents matching query.

Let's see how we do this.

"},{"location":"0-Workshop/3-Ideate/03/#3-create-ai-project-client","title":"3. Create AI Project Client","text":"
  1. Create an Azure AI Project client using the connection string
  2. Use the client to retrieve a chat_completions inference client
  3. Use the client to retrieve an embeddings inference client
  4. Use the client to setup a search_client using the search connection
Click to expand and view the Python script src/api/get_product_documents.py - Part 1
import os\nfrom pathlib import Path\nfrom opentelemetry import trace\nfrom azure.ai.projects import AIProjectClient\nfrom azure.ai.projects.models import ConnectionType\nfrom azure.identity import DefaultAzureCredential\nfrom azure.core.credentials import AzureKeyCredential\nfrom azure.search.documents import SearchClient\nfrom config import ASSET_PATH, get_logger\n\n# initialize logging and tracing objects\nlogger = get_logger(__name__)\ntracer = trace.get_tracer(__name__)\n\n# create a project client using environment variables loaded from the .env file\nproject = AIProjectClient.from_connection_string(\n    conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n)\n\n# create a vector embeddings client that will be used to generate vector embeddings\nchat = project.inference.get_chat_completions_client()\nembeddings = project.inference.get_embeddings_client()\n\n# use the project client to get the default search connection\nsearch_connection = project.connections.get_default(\n    connection_type=ConnectionType.AZURE_AI_SEARCH, include_credentials=True\n)\n\n# Create a search index client using the search connection\n# This client will be used to create and delete search indexes\nsearch_client = SearchClient(\n    index_name=os.environ[\"AISEARCH_INDEX_NAME\"],\n    endpoint=search_connection.endpoint_url,\n    credential=AzureKeyCredential(key=search_connection.key),\n)\n
"},{"location":"0-Workshop/3-Ideate/03/#4-get-docs-for-user-intent","title":"4. Get Docs For User Intent","text":"
  1. First, receive input text string (user query)
  2. Then, map user query text into a clear intent (search query)
  3. Then, vectorize the search query (to support retrieval)
  4. Then, search the product index for matches (by cosine similarity)
  5. Then, for each matching product, retrieve its document (content)
  6. Return the collection of documents to the user.
Click to expand and view the Python script src/api/get_product_documents.py - Part 2
    from azure.ai.inference.prompts import PromptTemplate\n    from azure.search.documents.models import VectorizedQuery\n\n    @tracer.start_as_current_span(name=\"get_product_documents\")\n    def get_product_documents(messages: list, context: dict = None) -> dict:\n        if context is None:\n            context = {}\n\n        overrides = context.get(\"overrides\", {})\n        top = overrides.get(\"top\", 5)\n\n        # generate a search query from the chat messages\n        intent_prompty = PromptTemplate.from_prompty(Path(ASSET_PATH) / \"intent_mapping.prompty\")\n\n        intent_mapping_response = chat.complete(\n            model=os.environ[\"INTENT_MAPPING_MODEL\"],\n            messages=intent_prompty.create_messages(conversation=messages),\n            **intent_prompty.parameters,\n        )\n\n        search_query = intent_mapping_response.choices[0].message.content\n        logger.debug(f\"\ud83e\udde0 Intent mapping: {search_query}\")\n\n        # generate a vector representation of the search query\n        embedding = embeddings.embed(model=os.environ[\"EMBEDDINGS_MODEL\"], input=search_query)\n        search_vector = embedding.data[0].embedding\n\n        # search the index for products matching the search query\n        vector_query = VectorizedQuery(vector=search_vector, k_nearest_neighbors=top, fields=\"contentVector\")\n\n        search_results = search_client.search(\n            search_text=search_query, vector_queries=[vector_query], select=[\"id\", \"content\", \"filepath\", \"title\", \"url\"]\n        )\n\n        documents = [\n            {\n                \"id\": result[\"id\"],\n                \"content\": result[\"content\"],\n                \"filepath\": result[\"filepath\"],\n                \"title\": result[\"title\"],\n                \"url\": result[\"url\"],\n            }\n            for result in search_results\n        ]\n\n        # add results to the provided context\n        if \"thoughts\" not in context:\n            context[\"thoughts\"] = []\n\n        # add thoughts and documents to the context object so it can be returned to the caller\n        context[\"thoughts\"].append(\n            {\n                \"title\": \"Generated search query\",\n                \"description\": search_query,\n            }\n        )\n\n        if \"grounding_data\" not in context:\n            context[\"grounding_data\"] = []\n        context[\"grounding_data\"].append(documents)\n\n        logger.debug(f\"\ud83d\udcc4 {len(documents)} documents retrieved: {documents}\")\n        return documents\n
"},{"location":"0-Workshop/3-Ideate/03/#5-run-docs-retrieval-script","title":"5. Run Docs Retrieval Script","text":"

Before we can run this script, we need to create the Intent Mapper template for step 2. Let's do that next.

"},{"location":"0-Workshop/3-Ideate/04/","title":"3.4 Understand Intent Mapping","text":""},{"location":"0-Workshop/3-Ideate/04/#1-add-intent-mapping-prompty","title":"1. Add Intent Mapping Prompty","text":"

Let's copy over the intent_mapping.prompty prompt template into our assets folder.

cp src.sample/api/assets/intent_mapping.prompty src/api/assets/.\n
"},{"location":"0-Workshop/3-Ideate/04/#2-run-docs-retrieval-script","title":"2. Run Docs Retrieval Script","text":"

Before we dive into the details of that file, let's first run the document retrieval script and see what happens. Here's an example with the question we discussed earlier.

python get_product_documents.py \\\n    --query \"I need a new tent for 4 people, what would you recommend?\"\n\n\ud83e\udde0 Intent mapping: {\n    \"intent\": \"The user is looking for recommendations for a tent suitable for 4 people.\",\n    \"search_query\": \"best tents for 4 people\"\n}\n

The script output shows that it extracted the user intent and formulated a search query from it that related to a product (\"best tents for 4 people\") - and can be answered by the search index.

"},{"location":"0-Workshop/3-Ideate/04/#3-intent-mapping-in-action","title":"3. Intent Mapping In Action","text":"

The intent mapping is achieved using a Prompty asset - a YAML file that consists of frontmatter (prompt metdata) and content (prompt template)

  • Frontmatter defines model configuration and prompt inputs
  • Template defines prompt structure, context and instructions

Looking into the details of the prompt template, we can see it employs a few-shot learning technique where it attempts to teach the AI to execute a specific task (extract intent) by providing it with examples of inputs and expected responses.

Click to expand and view Intent Mapping Prompty src/api/assets/intent_mapping.prompty
 ---\nname: Chat Prompt\ndescription: A prompty that extract users query intent based on the current_query and chat_history of the conversation\nmodel:\n    api: chat\n    configuration:\n        azure_deployment: gpt-4o\ninputs:\n    conversation:\n        type: array\n---\nsystem:\n# Instructions\n- You are an AI assistant reading a current user query and chat_history.\n- Given the chat_history, and current user's query, infer the user's intent expressed in the current user query.\n- Once you infer the intent, respond with a search query that can be used to retrieve relevant documents for the current user's query based on the intent\n- Be specific in what the user is asking about, but disregard parts of the chat history that are not relevant to the user's intent.\n- Provide responses in json format\n\n# Examples\nExample 1:\nWith a conversation like below:\n    ```\n    - user: are the trailwalker shoes waterproof?\n    - assistant: Yes, the TrailWalker Hiking Shoes are waterproof. They are designed with a durable and waterproof construction to withstand various terrains and weather conditions.\n    - user: how much do they cost?\n    ```\nRespond with:\n{\n    \"intent\": \"The user wants to know how much the Trailwalker Hiking Shoes cost.\",\n    \"search_query\": \"price of Trailwalker Hiking Shoes\"\n}\n\nExample 2:\nWith a conversation like below:\n    ```\n    - user: are the trailwalker shoes waterproof?\n    - assistant: Yes, the TrailWalker Hiking Shoes are waterproof. They are designed with a durable and waterproof construction to withstand various terrains and weather conditions.\n    - user: how much do they cost?\n    - assistant: The TrailWalker Hiking Shoes are priced at $110.\n    - user: do you have waterproof tents?\n    - assistant: Yes, we have waterproof tents available. Can you please provide more information about the type or size of tent you are looking for?\n    - user: which is your most waterproof tent?\n    - assistant: Our most waterproof tent is the Alpine Explorer Tent. It is designed with a waterproof material and has a rainfly with a waterproof rating of 3000mm. This tent provides reliable protection against rain and moisture.\n    - user: how much does it cost?\n    ```\nRespond with:\n{\n    \"intent\": \"The user would like to know how much the Alpine Explorer Tent costs.\",\n    \"search_query\": \"price of Alpine Explorer Tent\"\n}\n\nuser:\nReturn the search query for the messages in the following conversation:\n{{#conversation}}\n- {{role}}: {{content}}\n{{/conversation}}\n
"},{"location":"0-Workshop/3-Ideate/05/","title":"3.5 Retrieve Related Knowledge","text":""},{"location":"0-Workshop/3-Ideate/05/#1-add-chat-with-products-script","title":"1. Add Chat With Products Script","text":"

Let's copy over the chat_with_products.py script into our application source.

cp src.sample/api/chat_with_products.py src/api/.\n
"},{"location":"0-Workshop/3-Ideate/05/#2-understand-rag-workflow","title":"2. Understand RAG Workflow","text":"

This script is the core orchestrator for our RAG workflow, executing the following steps:

  1. Create an Azure AI Project client (with connection string)
  2. Retrieve the inference client for chat completions model
  3. Use incoming user messages to retrieve related products
  4. Use this knowledge to populate a grounded chat template
  5. Call the chat completions client with this prompt template
Click to expand and view Chat With Products script (segment) src/api/chat_with_products.py
from azure.ai.inference.prompts import PromptTemplate\n\n\n@tracer.start_as_current_span(name=\"chat_with_products\")\ndef chat_with_products(messages: list, context: dict = None) -> dict:\n    if context is None:\n        context = {}\n\n    documents = get_product_documents(messages, context)\n\n    # do a grounded chat call using the search results\n    grounded_chat_prompt = PromptTemplate.from_prompty(Path(ASSET_PATH) / \"grounded_chat.prompty\")\n\n    system_message = grounded_chat_prompt.create_messages(documents=documents, context=context)\n    response = chat.complete(\n        model=os.environ[\"CHAT_MODEL\"],\n        messages=system_message + messages,\n        **grounded_chat_prompt.parameters,\n    )\n    logger.info(f\"\ud83d\udcac Response: {response.choices[0].message}\")\n\n    # Return a chat protocol compliant response\n    return {\"message\": response.choices[0].message, \"context\": context}\n

In the next section, we'll look at the prompt template and run a test with a sample query.

"},{"location":"0-Workshop/3-Ideate/05/#3-run-chat-with-products-script","title":"3. Run Chat With Products Script","text":"

Before we can run this script, we need to create the Grounded Chat Prompt template for step 4. Let's do that next.

"},{"location":"0-Workshop/3-Ideate/06/","title":"3.6 Design Grounded Prompt","text":""},{"location":"0-Workshop/3-Ideate/06/#1-add-grounded-chat-prompty","title":"1. Add Grounded Chat Prompty","text":"

Let's copy over the grounded_chat.prompty prompt template into our assets folder.

cp src.sample/api/assets/grounded_chat.prompty src/api/assets/.\n
"},{"location":"0-Workshop/3-Ideate/06/#2-understand-grounding-context","title":"2. Understand Grounding Context","text":"

Explore the contents of this template. Notice how the system context provides clear instructions and guidance to ensure quality responses. This includes grounding responses in context (when query is relevant) and declining to provide responses (when query is irrelevant).

Click to expand and view Grounded Chat Prompty src/api/assets/grounded_chat.prompty
    ---\n    name: Chat with documents\n    description: Uses a chat completions model to respond to queries grounded in relevant documents\n    model:\n        api: chat\n        configuration:\n            azure_deployment: gpt-4o\n    inputs:\n        conversation:\n            type: array\n    ---\n    system:\n    You are an AI assistant helping users with queries related to outdoor outdooor/camping gear and clothing.\n    If the question is not related to outdoor/camping gear and clothing, just say 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?'\n    Don't try to make up any answers.\n    If the question is related to outdoor/camping gear and clothing but vague, ask for clarifying questions instead of referencing documents. If the question is general, for example it uses \"it\" or \"they\", ask the user to specify what product they are asking about.\n    Use the following pieces of context to answer the questions about outdoor/camping gear and clothing as completely, correctly, and concisely as possible.\n    Do not add documentation reference in the response.\n\n    # Documents\n\n    {{#documents}}\n\n    ## Document {{id}}: {{title}}\n    {{content}}\n    {{/documents}}\n
"},{"location":"0-Workshop/3-Ideate/06/#3-chat-with-products-relevant","title":"3. Chat With Products - Relevant","text":"

Run the script with a test query (from the src/api folder). 

python chat_with_products.py --query \"I need a new tent for 4 people, what would you recommend?\"\n

Observe the response. It may look like this:

\ud83d\udcac Response: {'content': \"I recommend the TrailMaster X4 Tent. It is specifically designed to accommodate four occupants comfortably. The tent features durable water-resistant construction, multiple doors for easy access, and mesh panels for ventilation and bug protection. Additionally, it has a freestanding design for easy setup and relocation, as well as interior pockets for organization. It's a great choice for your camping adventures!\", 'role': 'assistant'}\n

Is the response grounded in product data from the catalog?

"},{"location":"0-Workshop/3-Ideate/06/#4-chat-with-products-irrelevant","title":"4. Chat With Products - Irrelevant","text":"

Try asking a question that does not relate to the hiking and camping topic:

python chat_with_products.py --query \"I am looking for a recipe for spicy bean burgers\"\n

Observe the response. It may look like this:

\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n

Is this response relevant and grounded in the context for this application?

"},{"location":"0-Workshop/3-Ideate/07/","title":"3.7 Test with Observability","text":"

Recall that we activated Application Insights during the setup phase of our projec. This now allows to ask questions with telemetry enabled and get observable traces using open telemetry, to help us analyze the cost or performance of our workflows.

Let's see this in action.

"},{"location":"0-Workshop/3-Ideate/07/#1-enable-telemetry-on-test","title":"1. Enable Telemetry on Test","text":"

Run the Chat with Products script with --enable-telemetry as shown.

python chat_with_products.py --query \"I need hiking gear for a trip to Andalusia - what tents and boots do you recommend?\" --enable-telemetry\n

You should see something like this in the console. Let's explore this next. 

Enabled telemetry logging to project, view traces at:\nhttps://ai.azure.com/tracing?wsid=/subscriptions/XXXX/resourceGroups/ninarasi-ragchat-rg/providers/Microsoft.MachineLearningServices/workspaces/ninarasi-ragchat-v1\n\ud83d\udcac Response: {'content': \"For your trip to Andalusia, I recommend the following tents and boots:\\n\\n**Tents:**\\n1. **Alpine Explorer Tent**: This robust, 8-person, 3-season tent is perfect for group camping. It has multiple mesh windows for ventilation and a detachable divider for privacy. Its waterproof feature ensures you stay dry during unexpected rain.\\n\\n2. **SkyView 2-Person Tent**: If you're looking for a smaller option, this tent comfortably houses two people and is made from durable waterproof materials. It also features an intuitive setup system, effective ventilation, and a rainfly for extra weather protection, making it great for hiking and camping.\\n\\n**Boots:**\\n1. **TrekReady Hiking Boots**: These boots are crafted from leather, ensuring durability and comfort on long hikes. They have a moisture-wicking lining, shock-absorbing midsoles, and excellent traction, making them suitable for various terrains.\\n\\n2. **TrekStar Hiking Sandals**: If you prefer something lighter and more breathable, consider these lightweight sandals. They offer adjustable straps, excellent traction, toe protection, and a cushioned footbed for comfort during summer treks.\\n\\nChoose based on your group size and hiking preferences, and you'll be well-prepared for your adventure in Andalusia!\", 'role': 'assistant'}\n

"},{"location":"0-Workshop/3-Ideate/07/#2-view-traces-detail","title":"2. View Traces Detail","text":"

Look for the output section with a link as shown below, and navigate to that URL in the browser.

Enabled telemetry logging to project, view traces at:\nhttps://ai.azure.com/tracing?....\n

You should see something like this reflecting the latest run of the Chat with Products script above. Click to expand the tree of nodes on the left and observe the details provided in the panel on the right, as you step through them.

  1. We can trace the flow of control from the user query to the returned response
  2. We can measure the time taken for each step to execute (in seconds)
  3. We can observe the cost for model invocations (in tokens) in the GenAI rows
  4. For a given model interaction, we can explore details (system, user, assistant) to debug

"},{"location":"0-Workshop/3-Ideate/07/#3-view-traces-dashboard","title":"3. View Traces Dashboard","text":"

Click on the Tracing menu option in the sidebar to get the historical logs from previous runs. This is an effective way to analyze issues in cost or performance, make changes, then compare trace runs to see if the iterations had any impact.

"},{"location":"0-Workshop/3-Ideate/07/#4-view-traces-insights","title":"4. View Traces Insights","text":"

You can also click on the Insights for Generative AI Applications Dashboard link (top of screen) to get a more actionable Generative AI Application Insights (Preview) dashboard for real-time insights and analysis of usage patterns.

"},{"location":"0-Workshop/4-Evaluate/01/","title":"4.1 Create Evaluation Dataset","text":"

This begins Part 3 of the tutorial. This stage is completed USING THE AZURE AI FOUNDRY SDK.

At the end of this section, you should have established an evaluation inputs dataset, created an evaluation script for AI-assisted evaluation, and run the script to assess the chat AI app for quality. You will then learn to explore the evaluation outcomes locally, and via the Azure AI Foundry portal, and understand how to customize and automate the process for rapid iteration and improvement of your application quality.

"},{"location":"0-Workshop/4-Evaluate/01/#1-evaluating-generative-ai-apps","title":"1. Evaluating Generative AI Apps","text":"

The evaluation phase is critical to assessing the quality and safety of your generative AI application. The Azure AI Foundry provides a comprehensive set of tools and guidance to support evaluation in three dimensions. Learn More Here.

  1. Risk and safety evaluators: Evaluating potential risks associated with AI-generated content. Ex: evaluating AI predisposition towards generating harmful or inappropriate content.
  2. Performance and quality evaluators: Assessing accuracy, groundedness, and relevance of generated content using robust AI-assisted and Natural Language Processing (NLP) metrics.
  3. Custom evaluators: Tailored evaluation metrics that allow for more detailed and specific analyses. Ex: addressing app-specific requirements not covered by standard metrics.

AI-Assisted Evaluators are available only in select regions (Recommended: East US 2)

"},{"location":"0-Workshop/4-Evaluate/01/#2-ai-assisted-evaluation-flow","title":"2. AI-Assisted Evaluation Flow","text":"

So far, we've tested the chat application interactively (command-line) using a single test prompt. Now, we want to evaluate the responses on a larger and more diverse set of test prompts including edge cases. We can then use those results to iterate on our application (e.g., prompt template, data sources) till evaluations pass our acceptance criteria.

To do this we use AI-Assisted Evaluation - also referred to as LLM-as-a-judge - where we ask a second AI model (\"judge\") to grade the responses of the first AI model (\"target\"). The workflow is as shown below:

  1. First, create an evaluation dataset that consists of diverse queries for testing.
  2. Next, have the target AI (app) generate responses for each query
  3. Next, have the judge AI (assessor) grade tge {query, response} pairs
  4. Finally, visualize results (individual vs. aggregate metrics) for review.

"},{"location":"0-Workshop/4-Evaluate/01/#3-create-evaluation-dataset","title":"3. Create Evaluation Dataset","text":"

Let's copy over the chat_eval_data.jsonl dataset into our assets folder.

cp src.sample/api/assets/chat_eval_data.jsonl src/api/assets/.\n
"},{"location":"0-Workshop/4-Evaluate/01/#4-review-evaluation-dataset","title":"4. Review Evaluation Dataset","text":"

The Azure AI Foundry supports different data formats for evaluation including:

  1. Query/Response - each result has the query, response, and ground truth.
  2. Conversation (single/multi-turn) - messages (with content, role, optional context)
Click to expand and view the evaluation dataset src/api/assets/chat_eval_data.jsonl
{\"query\": \"Which tent is the most waterproof?\", \"truth\": \"The Alpine Explorer Tent has the highest rainfly waterproof rating at 3000m\"}\n{\"query\": \"Which camping table holds the most weight?\", \"truth\": \"The Adventure Dining Table has a higher weight capacity than all of the other camping tables mentioned\"}\n{\"query\": \"How much do the TrailWalker Hiking Shoes cost? \", \"truth\": \"The Trailewalker Hiking Shoes are priced at $110\"}\n{\"query\": \"What is the proper care for trailwalker hiking shoes? \", \"truth\": \"After each use, remove any dirt or debris by brushing or wiping the shoes with a damp cloth.\"}\n{\"query\": \"What brand is TrailMaster tent? \", \"truth\": \"OutdoorLiving\"}\n{\"query\": \"How do I carry the TrailMaster tent around? \", \"truth\": \" Carry bag included for convenient storage and transportation\"}\n{\"query\": \"What is the floor area for Floor Area? \", \"truth\": \"80 square feet\"}\n{\"query\": \"What is the material for TrailBlaze Hiking Pants?\", \"truth\": \"Made of high-quality nylon fabric\"}\n{\"query\": \"What color does TrailBlaze Hiking Pants come in?\", \"truth\": \"Khaki\"}\n{\"query\": \"Can the warrenty for TrailBlaze pants be transfered? \", \"truth\": \"The warranty is non-transferable and applies only to the original purchaser of the TrailBlaze Hiking Pants. It is valid only when the product is purchased from an authorized retailer.\"}\n{\"query\": \"How long are the TrailBlaze pants under warranty for? \", \"truth\": \" The TrailBlaze Hiking Pants are backed by a 1-year limited warranty from the date of purchase.\"}\n{\"query\": \"What is the material for PowerBurner Camping Stove? \", \"truth\": \"Stainless Steel\"}\n{\"query\": \"Is France in Europe?\", \"truth\": \"Sorry, I can only queries related to outdoor/camping gear and equipment\"}\n

Our dataset reflects the first format, where the test prompts contain a query with the ground truth for evaluating responses. The chat AI will then generate a response (based on query) that gets added to this record, to create the evaluation dataset that is sent to the \"judge\" AI.

 {\n    \"query\": \"Which tent is the most waterproof?\", \n    \"truth\": \"The Alpine Explorer Tent has the highest rainfly waterproof rating at 3000m\"\n}\n
Let's look at the evaluation script that orchestrates this workflow, next

"},{"location":"0-Workshop/4-Evaluate/02/","title":"4.2 Create Evaluation Script","text":""},{"location":"0-Workshop/4-Evaluate/02/#1-create-evaluation-script","title":"1. Create Evaluation Script","text":"

Let's copy over the evaluate.py script into our application source folder.

cp src.sample/api/evaluate.py  src/api/.\n
"},{"location":"0-Workshop/4-Evaluate/02/#2-review-evaluation-workflow","title":"2. Review Evaluation Workflow","text":"

Let's review the workflow required for AI-Assisted evaluation:

  1. We have an evaluation test dataset containing query/truth pairs
  2. We have a target AI (applicationl) that will generate the responses
  3. We have a judge AI (evaluator) that will grade those responses
  4. The judge has scoring criteria they use to generate evaluation metrics
  5. The evaluation results are published locally, or to Azure AI Foundry

"},{"location":"0-Workshop/4-Evaluate/02/#3-unpack-evaluation-script","title":"3. Unpack Evaluation Script","text":""},{"location":"0-Workshop/4-Evaluate/02/#31-create-ai-project-client","title":"3.1 Create AI Project Client","text":"src/api/evaluation.py
    # ----------------------------------------------\n    # 1. Create AI Project Client \n    # ----------------------------------------------\n    import os\n    import pandas as pd\n    from azure.ai.projects import AIProjectClient\n    from azure.ai.projects.models import ConnectionType\n    from azure.ai.evaluation import evaluate, GroundednessEvaluator\n    from azure.identity import DefaultAzureCredential\n\n    from chat_with_products import chat_with_products\n\n    # load environment variables from the .env file at the root of this repo\n    from dotenv import load_dotenv\n    load_dotenv()\n\n    # create a project client using environment variables loaded from the .env file\n    project = AIProjectClient.from_connection_string(\n        conn_str=os.environ[\"AIPROJECT_CONNECTION_STRING\"], credential=DefaultAzureCredential()\n    )\n\n    connection = project.connections.get_default(connection_type=ConnectionType.AZURE_OPEN_AI, include_credentials=True)\n
"},{"location":"0-Workshop/4-Evaluate/02/#32-specify-model-evaluators","title":"3.2 Specify Model & Evaluators","text":"src/api/evaluation.py
    # ----------------------------------------------\n    # 2. Define Evaluator Model to use\n    # ----------------------------------------------\n    evaluator_model = {\n        \"azure_endpoint\": connection.endpoint_url,\n        \"azure_deployment\": os.environ[\"EVALUATION_MODEL\"],\n        \"api_version\": \"2024-06-01\",\n        \"api_key\": connection.key,\n    }\n\n    groundedness = GroundednessEvaluator(evaluator_model)\n
"},{"location":"0-Workshop/4-Evaluate/02/#33-create-evaluation-wrapper","title":"3.3 Create Evaluation Wrapper","text":"src/api/evaluation.py
    # ----------------------------------------------\n    # 3. Create Eval Wrapper Function for Query\n    # ----------------------------------------------\n    def evaluate_chat_with_products(query):\n        response = chat_with_products(messages=[{\"role\": \"user\", \"content\": query}])\n        return {\"response\": response[\"message\"].content, \"context\": response[\"context\"][\"grounding_data\"]}\n
"},{"location":"0-Workshop/4-Evaluate/02/#34-run-evaluation-print","title":"3.4 Run Evaluation & Print","text":"

This is the entry point for the evaluation script.

  • It uses the evaluate function from the azure.ai.evaluation package to run a built-in evaluator (GroundednessEvaluator) to score the responses from the target app.
  • If both the data (test) and target (function) are provided, it will first invoke the target with that data - and then run evaluations on the results.
  • If azure_ai_project is set, then the evaluation results are also logged to Azure AI Foundry
src/api/evaluation.py
    # ----------------------------------------------\n    # 4. Run the Evaluation\n    #    View Results Locally (Saved as JSON)\n    #    Get Link to View Results in Foundry Portal\n    #    NOTE:\n    #    Evaluation takes more tokens \n    #    Try to increase Rwate limit (Tokens per minute)\n    #    Script should handle limit errors if needed\n    # ----------------------------------------------\n    # Evaluate must be called inside of __main__, not on import\n\n    if __name__ == \"__main__\":\n        from config import ASSET_PATH\n\n        # workaround for multiprocessing issue on linux\n        from pprint import pprint\n        from pathlib import Path\n        import multiprocessing\n        import contextlib\n\n        with contextlib.suppress(RuntimeError):\n            multiprocessing.set_start_method(\"spawn\", force=True)\n\n        # run evaluation with a dataset and target function, \n        # log to the project\n        result = evaluate(\n            data=Path(ASSET_PATH) / \"chat_eval_data.jsonl\",\n            target=evaluate_chat_with_products,\n            evaluation_name=\"evaluate_chat_with_products\",\n            evaluators={\n                \"groundedness\": groundedness,\n            },\n            evaluator_config={\n                \"default\": {\n                    \"query\": {\"${data.query}\"},\n                    \"response\": {\"${target.response}\"},\n                    \"context\": {\"${target.context}\"},\n                }\n            },\n            azure_ai_project=project.scope,\n            output_path=\"./myevalresults.json\",\n        )\n\n        tabular_result = pd.DataFrame(result.get(\"rows\"))\n\n        pprint(\"-----Summarized Metrics-----\")\n        pprint(result[\"metrics\"])\n        pprint(\"-----Tabular Result-----\")\n        pprint(tabular_result)\n        pprint(f\"View evaluation results in AI Studio: {result['studio_url']}\")\n
"},{"location":"0-Workshop/4-Evaluate/03/","title":"4.3 Configure Evaluation Model","text":"

Recall that in the last section, the evaluation script identified an evaluator_model that will serve as the judge AI for this assessment.

"},{"location":"0-Workshop/4-Evaluate/03/#1-specifying-evaluator-model","title":"1. Specifying Evaluator Model","text":"

In this workshop, we are reusing the same model for both chat_completion and evaluation roles, but you can choose to separate the two by:

  • Deploying a new model to the same Azure AI Project
  • Updating the EVALUATION_MODEL environment variable to this one
  • Restarting the evaluation script

HOMEWORK: Try deploying a gpt-4 model for evaluations. How do results differ?

src/api/evaluation.py
    # ----------------------------------------------\n    # 2. Define Evaluator Model to use\n    # ----------------------------------------------\n    evaluator_model = {\n        \"azure_endpoint\": connection.endpoint_url,\n        \"azure_deployment\": os.environ[\"EVALUATION_MODEL\"],\n        \"api_version\": \"2024-06-01\",\n        \"api_key\": connection.key,\n    }\n\n    groundedness = GroundednessEvaluator(evaluator_model)\n
"},{"location":"0-Workshop/4-Evaluate/03/#2-configuring-evaluator-model","title":"2. Configuring Evaluator Model","text":"

Let's take a look at the core evaluate function that executes the workflow. This function will run an assessment once for each record (in data), for each evaluator (in evaluators). This requires a lot of calls to the identified evaluation model - which will require a higher token capacity for efficient completion.

Note: The current script uses a single evaluator (for Groundedness). Adding additional evaluators will increase the number of calls made to the default model, so make sure you configure quota to adjust for that accordingly.=

Update the model quota in Azure AI Foundry if execution has rate limit issues

Take these steps to view and update your model quota.

  • Visit your Azure AI project page in Azure AI Foundry
  • Click \"Models + Endpoints\" and select the evaluation model
  • Click Edit and increase the Tokens per minute rate limit* (e.g., to 30)
  • Click Save and close
Click to expand and see a screenshot of the update dialog

"},{"location":"0-Workshop/4-Evaluate/04/","title":"4.4 Run Evaluation Script","text":""},{"location":"0-Workshop/4-Evaluate/05/","title":"4.5 View Results Locally","text":""},{"location":"0-Workshop/4-Evaluate/06/","title":"4.6 View Results In Portal","text":""},{"location":"0-Workshop/4-Evaluate/07/","title":"4.7 Validate The Evaluation","text":"Text Only
# ----------------------------------------------\n# Run the Evaluation using this command\n#    python evaluate.py \n#\n# You should see something like this:\n\n'''\nStarting prompt flow service...\nStart prompt flow service on 127.0.0.1:23333, version: 1.16.2.\nYou can stop the prompt flow service with the following command:'pf service stop'.\n\nYou can view the traces in local from http://127.0.0.1:23333/v1.0/ui/traces/?#run=main_evaluate_chat_with_products_rxna_3r9_20241216_163719_733780\n[2024-12-16 16:37:42 +0000][promptflow._sdk._orchestrator.run_submitter][INFO] - Submitting run main_evaluate_chat_with_products_rxna_3r9_20241216_163719_733780, log path: /home/vscode/.promptflow/.runs/main_evaluate_chat_with_products_rxna_3r9_20241216_163719_733780/logs.txt\n\ud83d\udcac Response: {'content': 'Could you please specify which camping table you are referring to? There are multiple options available, and I can provide information on them.', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': \"Could you specify what aspect of care you're asking about? Are you looking for cleaning instructions, storage tips, or something else for the TrailWalker Hiking Shoes?\", 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': \"Could you please specify which tents you are comparing, or do you want information about a specific tent's waterproof features?\", 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'The TrailBlaze Hiking Pants are crafted from high-quality nylon fabric.', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'The TrailMaster X4 Tent comes with an included carry bag, which makes transporting the tent easy and convenient. You can simply pack the tent into the carry bag and carry it as needed for your camping adventure. If you have any more specific questions about the tent or its features, feel free to ask!', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n\ud83d\udcac Response: {'content': 'Sorry, I only can answer queries related to outdoor/camping gear and clothing. So, how can I help?', 'role': 'assistant'}\n...\n...\n...\n'''\n\n#This is what a rate limit error looks like:\n\n'''\n[2024-12-16 16:44:07 +0000][promptflow.core._prompty_utils][ERROR] - Exception occurs: RateLimitError: Error code: 429 - {'error': {'code': '429', 'message': 'Requests to the ChatCompletions_Create Operation under Azure OpenAI API version 2024-06-01 have exceeded token rate limit of your current AIServices S0 pricing tier. Please retry after 60 seconds. Please contact Azure support service if you would like to further increase the default rate limit.'}}\n[2024-12-16 16:44:07 +0000][promptflow.core._prompty_utils][WARNING] - RateLimitError #2, Retry-After=60, Back off 60.0 seconds for retry.\n'''\n\n# This is what the final result looks like\n\n'''\n======= Run Summary =======\n\nRun name: \"azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_rvrjml8t_20241216_164205_696721\"\nRun status: \"Completed\"\nStart time: \"2024-12-16 16:42:05.695977+00:00\"\nDuration: \"0:03:06.490785\"\nOutput path: \"/home/vscode/.promptflow/.runs/azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_rvrjml8t_20241216_164205_696721\"\n\n======= Combined Run Summary (Per Evaluator) =======\n\n{\n    \"groundedness\": {\n        \"status\": \"Completed\",\n        \"duration\": \"0:03:06.490785\",\n        \"completed_lines\": 13,\n        \"failed_lines\": 0,\n        \"log_path\": \"/home/vscode/.promptflow/.runs/azure_ai_evaluation_evaluators_common_base_eval_asyncevaluatorbase_rvrjml8t_20241216_164205_696721\"\n    }\n}\n\n====================================================\n\nEvaluation results saved to \"/workspaces/learns-azure-ai-foundry/src/api/myevalresults.json\".\n\n'-----Summarized Metrics-----'\n{'groundedness.gpt_groundedness': 1.4615384615384615,\n'groundedness.groundedness': 1.4615384615384615}\n'-----Tabular Result-----'\n                                    outputs.response  ... line_number\n0   Could you please specify which tents you are c...  ...           0\n1   Could you please specify which camping table y...  ...           1\n2   Sorry, I only can answer queries related to ou...  ...           2\n3   Could you specify what aspect of care you're a...  ...           3\n4   Sorry, I only can answer queries related to ou...  ...           4\n5   The TrailMaster X4 Tent comes with an included...  ...           5\n6   Sorry, I only can answer queries related to ou...  ...           6\n7   The TrailBlaze Hiking Pants are crafted from h...  ...           7\n8   Sorry, I only can answer queries related to ou...  ...           8\n9   Sorry, I only can answer queries related to ou...  ...           9\n10  Sorry, I only can answer queries related to ou...  ...          10\n11  Sorry, I only can answer queries related to ou...  ...          11\n12  Sorry, I only can answer queries related to ou...  ...          12\n\n[13 rows x 8 columns]\n('View evaluation results in AI Studio: '\n'https://ai.azure.com/build/evaluation/XXXXXXX?wsid=/subscriptions/XXXXXXXX/resourceGroups/ninarasi-ragchat-rg/providers/Microsoft.MachineLearningServices/workspaces/ninarasi-ragchat-v1')\n'''\n\n# ----------------------------------------------\n\n```\n
"},{"location":"0-Workshop/5-Evolve/01/","title":"5.1 Recap: Build A Copilot","text":"

PLACEHOLDER \ud83d\udc49\ud83c\udffd EXPLAIN WHAT WE COVERED (AND WHAT WE DIDN'T)

"},{"location":"0-Workshop/5-Evolve/02/","title":"5.2 Refactor: Make it Better","text":"

PLACEHOLDER \ud83d\udc49\ud83c\udffd EXPLAIN HOW CONTOSO CHAT STREAMLINES E2E

"},{"location":"0-Workshop/5-Evolve/03/","title":"5.3 Resources: Learn More","text":"

PLACEHOLDER \ud83d\udc49\ud83c\udffd EXPLAIN WHERE TO GO FOR ADVANCED TUTORIALS

"}]} \ No newline at end of file