From 2510eeea68b1d012ae685305a72af78102d3f2fa Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:05:02 -0700 Subject: [PATCH] Add Tree of Thoughts --- docs/docs/tutorials/index.md | 3 +- docs/mkdocs.yml | 1 + examples/tutorials/tot/img/tot.png | Bin 0 -> 37886 bytes examples/tutorials/tot/tot.ipynb | 527 +++++++++++++++++++++++++++++ 4 files changed, 530 insertions(+), 1 deletion(-) create mode 100644 examples/tutorials/tot/img/tot.png create mode 100644 examples/tutorials/tot/tot.ipynb diff --git a/docs/docs/tutorials/index.md b/docs/docs/tutorials/index.md index 3e414d592..d829827e4 100644 --- a/docs/docs/tutorials/index.md +++ b/docs/docs/tutorials/index.md @@ -50,7 +50,8 @@ Learn from example implementations of graphs designed for specific scenarios and - [Basic Reflection](reflection/reflection.ipynb): Prompt the agent to reflect on and revise its outputs - [Reflexion](reflexion/reflexion.ipynb): Critique missing and superfluous details to guide next steps -- [Language Agent Tree Search](lats/lats.ipynb): Use reflection and rewards to drive a tree search over agents +- [Tree of Thoughts](tot/tot.ipynb): Search over candidate solutions to a problem using a scored tree +- [Language Agent Tree Search](lats/lats.ipynb): Use reflection and rewards to drive a monte-carlo tree search over agents - [Self-Discover Agent](self-discover/self-discover.ipynb): Analyze an agent that learns about its own capabilities #### Evaluation diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 083661975..0dcecd07f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -110,6 +110,7 @@ nav: - Reflection & Critique: - Basic Reflection: tutorials/reflection/reflection.ipynb - Reflexion: tutorials/reflexion/reflexion.ipynb + - Tree of Thoughts: tutorials/tot/tot.ipynb - Language Agent Tree Search: tutorials/lats/lats.ipynb - Self-Discover Agent: tutorials/self-discover/self-discover.ipynb - Evaluation & Analysis: diff --git a/examples/tutorials/tot/img/tot.png b/examples/tutorials/tot/img/tot.png new file mode 100644 index 0000000000000000000000000000000000000000..519937d11d1f49fca9cad960439f52ace4a99c92 GIT binary patch literal 37886 zcmZ^~1yo(lvM!242oNN=LvVL@cNP-d-C0O*5591B_XKwh?jGFT-8FAz|9hXa$Gi8! zfZ0}E-Bmq%R)1X;uB<49jEIK_0Re$5BQ3510RhpH(Yi`}2F7X$fUEQ9> ztr9d>H8)G^KEgM1H}e8UMB?$?mS6=0CqrgQT6ezhjU z%FHl8c+S2=%gut+R{v=#nJ`R4vo(ruum=}&vLJT;V^AFU{3vR0zVhKg@#4MCwD4KGLC##|M+MYpBMP-PxW{9P=K zxaB=4GFbi08Q#syZ$f%9%q`LrhpIdAmBi|^q(p@2>nujueV?+J3M4Zkl&J!#ix zj5W(@gy0w+qHp*SI_=z<_$E#QcG`(YUirAW1R?UHjO#ic@Rz{iHa63gF;`H4parK9 zAfO=8A)vu2NboO2s1*dvztRv8l;9@>1ay1|1U&eO4*pZlh5An|WJfOaf6@>~e;bOZ zipj`;pQGnpVF2?+_m zlc_ncin!#z$-y}RGD{a1M_wi-H#av%H#SBGCkrMP9v&VhW>zLvRt9hj24@d@7bABD zduQ^0I{8mO;%3e!PF9XCRu1+gfBQ8uc5rnOAS3(B=zo6x8K;@M)&Fv`cmDUZz!PNp zTf@Y{$jtPABXhAb|9_DEt@(%SU-SBh9sl3Xc$KZ(&1|*At?a;31rJS-m5q)6Uu^!5 zn*SZ>Ka}duW=>)bc3?sm!T(juzls05^8ZixFP57B%aWa){+$A^l9Lsf zo6%nx3bODs{U2rjZqLv3SHS-z@PD-VuTro&1rhm~{%2_jBKlb3KR`eTL&%7WsJTO) z>cP9KNi6uzek4WpJXC1)_xGRq@mT^AkPa*HS>n|;tkip?1nNEnGm(szns$c%kz86e zaoy{J0uo*H`TZcvjOF|Id|O6a1`q$Jx!Xn7^?DkQ^F$`PFYr z(MVxpS8NU_Jjj1aQK#gUKEeFE1vM*zWwC}SQNwxZzbPz>O9uak8W)CrmX0+>+fD6X zy;*{@vnK!aYzY&A1eJb-nI`p_V3WlJw1{B(t*>$aU#NO9b`A9u2TOQHk*I|+&}FkbB9QB z)AZv!ezz9?ArS z+LPGq3&-GgJ12<1W{Q4%bT_rIAZK7e&dbX?X7d{z8WMJQ=ic7l{%rCkC?Y!gYn5t= zV%c^CCSCD4>CTrVY5-JPT3WfSjuw5&hJ>Uf87C)}#YCovy*;C`vGH%W5~ZB4548-h z*=9KcURZM#8X6m3HwMjITwY#GkbK^c%!=9k;k&!0UnnT_+-rhP>8rN9G&NJ9Sjosl z1^iWtWV`Qa^;+3Sk{Keue~0=0{k!`L{p`kMfZOg6T5u9FAHFX^Y9ELR9Uixa+@3-% zSq%q`$+-XFdT*4DUt!}&)Y{P6`Z$;+&Ch#zpxDkE-GULJkB4cX5je8J0m})cTiRj{ zq<3e1ur&$TDbQ_kiXcseMZ1qeirh&N59v#Z8j5|aMZ;CmuU)fvVZ1mfEv3FRBWLC9 zv=yh!NV#4g_2}Jb8Ne0sHSyHNxj$P8wOeU48jL1fY_Lgrmfz$?U6f($du@|VqP1@7 z4o;8cG|)9{z8M-BQOTl$pbx$92B!4DmY9#Hm#SA_F}9tu{{E?>kz@s<9IOua(&qJG zZ@bZ{*Aa#<;j4&;Z{8Qhbe{zs-n@jRxvjW5L>-PnbyBR5@q1!81RtLF;-_9)V^7q@ z6TFep-A2xx6PHZpO*G(KI-H?qf3(q3y;4`pC==VlSRvDHZ*hQ9sE%OWq&Ll^&U&u= zSnH`Srj>C4ZF6;{&SFx+`C_v>$Qo&J0A_=Sl3-kb6=Q-g8ovCPo9KE{U?siZwW4}| z{q|^fZh5J}Mx%`cD5k*iWdWmrT0nqMEmGfgP8}t37jEZn!C=rL$7lbNSHt&~W_Aav z+G$UgUZ)YM;_FWpvKqS?1E0G&>$sv+TNr?x*XbflgCTG$UBIhRWz;bbJS@!RM!O0) za}w#mM8;C3KR1&?HNJ12i8QMFJ;qCAs#JGw>v{=XkHEZr@kWN**^$ssUnF>X{ z{hIdH*BWC9hjQk#j#EDDydt@jz;ga_6YJlld6e%B3;K~FL2xuOi(xFi!yV2CiVglw zv)xz2QQ94zcfY?McyDTQU@ZOCG`CKuTIp$dc-Zi(XaT>EybUe&Kef2l-3##WB_{D`I;ltzRe zG=&DK4u~j5dcF{qArT7_ICMT08;w6+4H9D(bUdtkA=AxjiG;8Su;Q?`68k=iFc1*( zeN*GZWK1;!DIZkIdv>Y$bf+|QQA^l)@lSHDYhw#?$8%X zy2kT-wWUHOfH zC^&I*s`Buzr+s`ghHoo&O*A1Ee35J>D^}E3Yi=7fm^y9W&ZbI~h{s=%yp=P(*eoX0 zm}D0+y^!%a*{k7pBDZl#4u}?(_s26TS}f}KPrb-ij1{@@8zMI#1oADL=)wJQOqs4~CMPv+* zKHZ)g$x30K?FBGqB?mg+3{&M$f5>?kU8Q+Z^*LgoX>8^O>cZhyWI07=t zXL6${3BA>K6t~=E4a&#lGvg9r$G^Y57?>^}fBh)$$U;an{GGZ?@cnT|a7A5}wwQM> zlBpAd0);maZiX!L58&!`C(CUe&}Pco)xJOWRCV`Y!PYdcM?YVw_wl@^rI(o7aZ{pr z<2-q9;;wtYUyf;@8d|mI)HWccew=;4PN3hq08sc?m9_Ce?4cH}Vv z<Zas#%Rx4qsXTKnVlXZn`dr)ZWgHQEHUVu@Ru?4rPj1=WqPKQ!7o9J^ zJtqIWKM0Az0rYLb>Mxn-(f57YgZdP4qfbojH_H@@AKcd`-T=(qk~2bQVB_JD1mPHI zH&jqPPUR#eJF5wG+%MZ;2Yzx_`3h(YMJGBS?lp%^=bfFd+crN3t6DAjP!jY3?-}rB zBe^XZ6JjYDt0NyHi1#~G=E}Op~FIxc6A_o_SwC)shwm63>fyBpNW^ zZ=U0|VBpu`)K80pHrVCpteAqphR)_2TZ1pc$R z!ItJdWaEsgUVC5mhsWL2e99iIsP*xQ78f49cuQFPQgMUTeD(+GSmxH466cr@m5KAc zG=VpQC~pA%K1h=omWjM{Q`qK7)?!ord^@Uyp5E(>MSj^8>E+=IX@!~wDDCn~C6YnN zLa1VIH#uK51r2tRCukrv$Y$rD!~pL71Qob(?@w6&b}p6#4(klqU?IP-7KFi95ly;aC=hH49WI4egV#lw5)S(ol~30CT{`0VR* zjWvCc9*?p4&8wpjUPcce*HP6o-uuo*A6*}TGG=3%_jGvYIKh=-gZZ=A0QBV%xD^&# z?KWsG6*YDi@Oqgz=;KZwNO-%l!pckwluPh|dF?QVeNN4VZ6uHSJUNFWQO+DD8U~Q$ zWUSyC-t*z??yhESinS-i-w;FwPNPxjU_mt`}-}*IRGc^65hDC zlwXiEA^JllM>dg26h4xZw4_nXT0T3-ev$yv!^!MW|&9@-wNW@b&oeQ7w*TdoS=^qrZT_tG&+0GOj3Yt%*{X63mb68&yR2~n#Mk~;9miA)-{tI z3HU^x4UHq?5xRztA^7owqjE`BD9_UB+h<#ItNFIQNyYm>*^8 zTD9roJsv(fop2V=4t@@D;_?LvcBGJQJ)(TSO$)=K=mpB9tv5Gad{g7?Kx^yu;5@-! zwd5D3WA&3d%7YxQHtPL_lOegc=5+r95yK){Nz(ettThd6$)^jV))iRvwMcDf!kseG zI>&s&&fb49VO!mYZu3x)9*Z~+^>4C2{cluG-gw01uwB)rfBOvwg{A&4AB|- zryzlkRGTe3ZPRyG7C@b->PR96J6I$+t1roHj0t_7b<+FVDOC^0J~b?TsvecsuS#)h^yoWNWGtTqn7&MA)#B*%wK7j~ zb-vuJg1i{GP+nG?jxQNUI}6)!inLHJ7R`n~WyCehYy#yrI(-{OYwHYHK#s5D~cWxXR$_Gc#=gPI!)3Zg$+jG;hpJuUWBS|eH_#Gno8g_*B8li zK6RRST$V{Bs|q32-g^0UQBWPk?!7Sz&=XT#G_=n*H1%uuk&Jd+j5qRgY$8Xl8mEXS zIiPK*wRN?;6pbD$)apsn8R?1*tl#nk_vP6>&4h(l_~WUg$7)DMlqrAY7}?TItOTdNG1HWpr*+^yL=YKwXPicOi++cKrTFZTc%nPDAMMm5X?d8 z-Z*+0;-Zx+zDqNGF>6#>g$mfwo~U5&rOmgF!eI%a3)qf3;?5$Bnz_nDV9-JYq#3_r z$I}6f3n>m*V@Vy7ZwdMmX-Gp@DFB`FNTsMK6`GjyWD%oRk&8CcORzNZY4fMOpM-Af zZLrr4ux59R2iVC$9Y;4e38na#&xuBsE;Wf>fSss!5MW*O-yYE)wbCu0#|()5j5@?X+1_12Y&QO znLL6*Z?~#unccp49D*N@nbw5ck?qu#m)@V(IVnr=z-yYRjClYis3V35{8lO!DsLmR zbXJhd-Gq%U=Z}?Rv+V8>S3l&}H96Lw=t6A>@FMjRJz@3th1LT{ap-Y*2qr=4kpex# z`IQ}z09E3F*3iA4B)10FQfdYFyV&KEE-EdDPH4>UPZlPx z!3R?v(}N=!wBj}?6n_S?Ev+xnSrYWQ-ZO&R6frtcE8h z7(tY}DhE|v@78fH8dyuIde=I+z9M(grrOnEF^Ll1MXxi+(W*r8>J9 zvyIB4<)$QcI z|4xmM)doftN#JIT4aJRb{kM`3hV}{|z3>C$T*=lJ;|jWmYJ<|bUcF_U>{c(T4M@W} z*>=}utOyo=)&RgDnmInhsYC?xGmJ;!NU-;@cs1@N=x|qF9hkzX_X@~TI@wGu1r)&H zl)iv*2FulEpzx#s5nB`~n(tYPx1X&w360h#CSj;H02ddR zHkEZeu-{*#Xpd&`In*sR^4h%ETn(aN+l2Rof|vy3(g~0!yIo#UdEHVzqLs1wlvO;< zPqk$%zky2*m(O?C=E=}&$?}O1mv~kZ=)7-}lv`ovyvLiKGmEQP_<7aqqN8~$1Yfsh zyxJy^A)G#7YjvkM0hh<}XrNBv?(fL<%uQ_>T zQhgt$m~2(Xeq5nNMUo2QEI`>auk4v?!@B?r)0eXjrZ*FSRLSf6>aO-vTfHc^L@A~# zOa%x&P^uMwGahJ;b15{_8EdDGbWP#i*(xmH;ls`Ddb?WnkaW>IoXQJYhGo-vBldgE zV=nn@X`VU!qopjjP*!TEB*#}U&BwTR18&<+SwwS+@m31T*NbCn5xsWR! z(`<0K%;ZH!Z+w!wTOQ7uHfWF1&Fa?V=f_Q#e}6@FE|ws>$mI+ohsp~~jRxmdX>F#qhTy0Td_+^XcEiB-9dzmw7 z2txc9Cdu#Ic}T;7AV54h^)BKX~qs#HJ%BIEOmIz_yGN=H|g>)D8QK zSe%_cy|z5!e&ash>otDiX5FYhPM^SGzWeotDp(%fj{()(TAE%N}oE%!YZ>|lHyS{6h;5CXYH)W}B6 zpv3QDAoA9JRE-Et4UnN2G0A9rXiSC^PH~Ry308gDo6s>T6#jXY zZIt`!!)%>vbdc4<`I_~^NN@$skI@4Oy}pSL~Kb*5(k12lbM^ zG)ip^fZPikVvT!FZD*3e?F_MyJ64Dm7z{x+ zx2}1xs~H!t7w3N97J6lQaCiBIO;O(NJh`Bf^$-TD|3T2q^Jyy=y3Vd=1xqucN>>vM z>>7u$b(Y=~9m<1c>V_i860RX=#D7yAzS-E-;vW~VZ-ykv^85Ae^1&kF=?NDDo4ojv z^egA;qap_sOI=pkvLm@*;$|2=OWoE$)CUTg6=S6k>w7jPgH5H(Qthqtbd!XSsB_%9 zF4x9mU2-XmK9CmMgV-z0qZKh)wj;}uQ zy8=T_wzV&~hB+Jq+pcrPac92=A$)qh4*B+?nHsF+r~_EyIL1h zCe+2?Qhe~EF457lUlPMlp_2TUGLS#V{eH?@2p!5aq+Y)O0OIB^(5f?c2Sciu zH(tkOC4u|=ifjTYAwjK{UNdm*O|~$#&a6PQJrr(tCwTp^PW=4d5Y0hhm%>Wy;;LI3 zsBfzfKH51_!8S5|K8w^omaD6S<4c5zKX4;rECkuR8ga9x0I3!%E<3rTfyMa&!k!a6 zAOjw}$yXOHZH*Qz6Ne`>vmNv}3@oU6hfg)>K}ZOO@7`fc#?uh}yB+mUqZy59y6qe- zF%UpH*<<|&G^al(cv#aKr*9!(3nHYNp>HTvUSR{r6NP=-CpnAH5{- zhvp};an>6wxX{!;wT4eE=NW;Lg!`!`?sTjIybm*0Zs5(nxNgJ+ z6BExob!lo~&KJjLEe)iK&i;f+ztM&8!0GjwYpw+&kMs}ixD$nru;x(LR&yU}_sD6V zp@g>?NwUhsKY}!W7S2?nv}c?@6w5J^&CbTJQ&cyRB}=QlP2<5(+@FI}tT!(Rl3xvn4{JikQ!!km3~ZP=E*Zg0Qknn zg^cv{C5`1*rmYLgx6m*jFbbdOe5K>iQS>`22>ufab&k|C+DvK@)ts|LW^^&pFL=L> zFg=+S=H@$u`igI=0b$_#C31T9n8K6?*&1Q&LHUXZ)y%Dac0%+7R5F!=4%t6vVleVd zM>Q9oSKs_#4%q}ASnAfgk#NQuyHD}!M_f|2mPWSte6H3#4hZiscPM7?c$B4QB+G_4 z0%r6L>e&|JBXKjRHsjLP0IixH-ZMVYVj+W^yPDsR_S7AgZBoDQ#&d_oF6yGP85+(~ zGTbfo4#RJx>Cem_D@p~S*ak?cFoMCX>pKAt*RMAvRBW_`P>ly+fE7KT*yMGSf4!w@a zBDIex`$~k*6TW>6vRI2~2u<1fnb9@iqrrcM)l2@hejUlJczx_sd2_q8xqTSoFY7fP z8>}y14c{wnEuc{yP56w3?TE1Q=7i6orHV^{&LI3c0{Jvni7Xv~*UMho%eoyt&{I;U zR}dVsO5WWN#L=c=mk_yH*+Bi2HX`{eep=!0p+;Q2ud@4A-ksMR^h+F{+&J+;_IKwU zeYCT4yh)yQBw|<=!OYa}Kbu5R5(CoMNW(vqLQAMbWatY5=hzec%<+%YV@unFo5f^A&AkWX!I zVb{QI<#R|o-bQpC#mer9gbXysg^U)(ns;&%>oo#?7wXok?O<>Pf+?4pCht(&e5bXUtlDBxM1 z-=${?6F_TEp#AhASQxM~;Ff=IhaB;Eoz1gTv_Ez3+a)$6{tvgQ`CeOiVaP zAs;g8ToH-VJ?Md_RsB?0N z1rd)-1z)7|Oa8i*pA3`eH*tY_l>E8g{Th{urRHx{Jegm9 zce&N;{(Mvd1cTSm+J_!!nO^c2m`L?ohfdxd&nC9Kt#F?vTX8oWQombNd3;oA^1-0U z2LbG6xvK2$qPGjMD+@a^j&JYN}-{OkV;fhR4ZZ(dZH)w1%A&%WUd6Wu#(!n>*!+BP|}SL-(}r1Z?rtcj2Rv zh-Q@GMOSYR{xB-P=6|Bj`Nwgc>;d0f{Oo|~< z8c6?M{o#V$#ULBE;rsG40lIh_oDlV4g z-?&h_O&~%)4j0Ms*@985kTchJ-%HZ)M%M?u2dcTn*Ie7$mGoH^rYGmTiEvo|R<8W~ zUOKAbDzAIT+eoxh>U}_)QW~P@>d-rTHn^Ehsq|PVJJCWEbv|IC@n@21&p@q|pZLSR z^_y%Ut>IjGr0+cODY>ENiNeeH{06UBF10<@L?%yx{GTIV;aR`uUlf#-yg?&78)=f3 zHK~bG(#NUG4K`l7(FORpJs>#7oBeUsG<@o-yQ5jD4uUm{K8L<3TgK1B!^76ln-Q`+ zT>Kxc_M-FDWW~gbv9KFVsOPt1Jt6CugkA1WwSs}pAu!IutNbiPWp%A}1(REoptDp{ zw7UXm;|})XR*E?$)BqfXathBVR95m1d*dnz- z`7O$q3zhS_9rv9aeLqXfxN`j7Jzxp2B`F>6D0)pKz|IBgd83tG@18kwb7vGE9}aT- zDxam?J~B>qf(Qy|S-1SW8)=R#9G}jTbZxUz-}v%6J(Tli*LcRnq4;NBpDn-H@P2h`l+1leSrk1D-NH{NZq_M!Ta-;cC_soef z*DP+JsMdGh*5tc&dO?lo>#Hj~R?cjZeJVkv^^C;C?*d-;hPy*?$Do;|Rl~Hb{^J;x zvNjia`JI-jgst>&vV1W#Jr(piJaIt39vvMQJ3bEpx91iFU^);l;!DC)$EInKkkT=b zyww7=?IQgOi-)QO`5SsdhTD>I@#3TsVLuw|<-UVKGhl={tjyF>t(ja2ej%*Wrc%gZ z*bzWsgx|CB_b5&}2yg_#lKJd2e-d_7R8&MW=wWde|G_LGA_6I{c*=2#`P5k@7>+n&{LVTJr}+y<*;V$YZesz}_PPObU1b zhN8c2^K<-O<#_GCWT0FnQ6n_a(#Td^8Bxt%_bF0*T z8FCgx91RzqD^|bpu{j`Llbo*%7SE`@DS$U%d9v;BBwEEGXiCjKf`uP@zX(pgeU0&` zo*VLCBTQ==7w=mvXY+lWwvYU44DBS#q;{S=vord^!m$&Flv7wQb<8I)`8`Y45`Gp( ze^57LOR09`*&MJ<9r~n?6sB-3oUQBG z0$#kQP$y=ktOX}#&v;6uwe&OIX<5yZuQe7}ls`Q`0RdO;@5^q}^?^A9?R}yB27Zv} zRqBCVQ1LsA^4o3Se*DHPJ0t#8e@DDkq(Gc+ag{N}zz12=Xp?q5G61u)GmZjOZeS9J z1C@=FYgFongTGk%I&m#5r#QGdEnSDl-R$h31DsNEv54DS7qJWju$xUuKzf>yKV)uH zVNZFJaeu@n92t()P7Nkb*S2~lzZa4l*UKmSQV)DlF(egeUMP@>Nk5|W$VchckJ4X& zLmq*5(Hnm7rXU$L=esA%V@P@3H_G>=CY&bDc8m_|^~h9mDu`kvUU$@M#~%x-h=~>y zL9>AIXD%i*=8nPxvp*5LXN^cu95(YvmLY?ogQp*YMfOIL=s?y0&Aa6jdT4dx94mO@W};(dvDWqTf*X@S(9Y;Rcw+zQYP`PClJ5`;VfR^Dt$%O_r#bA z(AL>{e=>&6Wg7FL$BOypL~+pc(Xu{55VNI>LJ`?^CgSS|HfTXoEdVX_z~>YCfvJO2j*EXDGv5JymI+*%GuouvKmk0R!c(~EkrTQVd?Itx9r(_H`SFZkdF`9e6`z5wYa-j~5 zjQ9OuFxbp+-2s%yi?z?X;+3b=mw#HL9;lNm3(@UdE$?jKf~z2nu%IY=T5QzDt}k~I zFn7obT)-0e`OH?`&o=+AJg3#sde_gJs+@}Cc%W!Fj4u2`gGoBTv{O>c#d9%7xP2UK z#_dW)`}*x~ZEac3l_%v{u$y{=cN%kX{~N#I&KwC=Px$GrY4_sZ49o1r2_c1~)yNU) zV9>z`9V8DUYNeU<8d_dl^UwFU<{EP)?~Zq9ewLCLpW&pLF9uO~kS-jk3^6LfvQCCc zZx$5eCW4X^X{`KqUKvrdo#C; zzSKp#B$(QO-+nSxejMkN3nf_ED8`Xa}6^G+u%3aKr6SlyI>vrj9f3A$Q zm|7Jb;zr`OjB<;wj%XqkYP_3Ejw9H?Q{1^etO+qW-aiG2e=|3W>b9rY1{1lDVGxS8 z7L6(sz~l$8-p7^4RNt7)C@D~gihjIp%u%M4ocQUjV2ZQSq!XoP$+Ohare1V-;(3<1 zQLX#2eb$WkeUT48;cI84Rz_FfFRb3o7sB|wxZ$h0cAmbiS!;yen;`?=nsP}i1QX{- z>K-_j;-+<7M))E2b@!*elryBb{D(O(o?H3j2z1@{{BqIN&DH_NDPWLNdCJO$&?|nU z0udU+OsXUVrNSFLwZ9Tz21$N)%LDNkH#}Y$m|VJEP{?L+8SUvHd_wM3wUgW$8ClS4o+$B){Sdn}tyv`T z(f?RJq3qA$jPRU&$#Tc#T>8W1;^_)d?@Y&0X?E~xZ~5W!C7LJIdl0Hmf;arI$M>(+ znrqaL$?Hl!H(SQt*~VG~G~V>w+zgjIVjFLP7iOO%e(mRNglY8*^Z-a`IK$J$n%d}L z(S34W3Qz#g9jFj9^!?Xk*M@(!&8vl?(iCQ%K|<(sk)px(V9m7FxH~=S$QcxDKfE&O zM5;X~Ve%w=2ljdC;7)HB&XTwfImmNS`YPe2Fi7?s=uN-7;B%_nk)b;;5M%>BBS^l# zljxst!y4t;m*!cany9+g+U3&#kYeFjT*j%tj#;CMQA(dE>lETLy%;60KnEVB!P2jx z+>l-NV;gLOeHsTbHFUQYB*A+rMt{HBYwOADm%PUYw}G#XbRKMyZ|^v@&UNbDGFDWV zjgKo}v_aJUDV}=nbj_y0zAz6_?-P!U#H*%{s0(m2GK3ONCFwC8$78Jo=Uz_8mZuT2 zCI5Qe!}#O6HYb5nlgF&{mVZ29!TDuByCi22j`nvIGIO;rvVrVjc`qFsD}Wb^L;Z&EGW{;4$&K?)gXEU49jXC~NLUG#HTII*Mww zcfw(Ewe6at`?!87)NJ|ZHes$cPUT~e!)*MthFSsKOY?)JtRc`a8Mxg~2)j-i{lI2Bw|$Yl2u6lHMx7F*)?EYX+Cj)ouf% zI9S_|-;3RZLGkS>M{Lm%OGMBz{Z88*d?`hA zk>s~2b`cj6vJfXnry2uJW()MVeZNB3)2wBw9*N|Rb48_MMcI+w*FWWL`_N_kh5{&& z%Qe!qvEV4*u$e300I~UQd1LNsjPm}hm0sQEy!YTt<*=}p?C2?v=sez%?!ikCZQnb8X_YI%ule?=b zuzF6rzB;_~A=cK_vDhv$GZUnd=Z`R`QI^!59K3y?$2{bPI8NBz4? z9m_7ra>bhPO=WIS$zRlT1p=9k;M&#P0m znLk5n`cHQ^rcL$D+nI!z-@GEyc z@VV?W3m>h+IPI2owWT{9ysaQpt7zY*e)(@QZ#>N#cKN**8$WCg_!X%u>lKzqjQ$Du(=$=E`#s0 zZHl|MIBs*zTs4r>bAamQE>1{b6eBV z2?tcMBne4sHm51j0m%o96=*ivrg_GpIZl5RH?B6(rf#%dnz6d^xF2;Eun;d6g`4^g z#QvBIHs3eFo+OF6FB23}^Kwm}GHx6pC1j<0_rUgNaYTa~A+oW)&z*Y8af4e|#|5YT zDz)HDThgD5jtxSe!Aomzs~H3s45+^*PHg|%mLz4ji~K4*xAy89(^8k)<3l?6vwDg_7S`X=z8MJi+eg8Xur(fFvA$Ps@k_fv)_NkV8vQX7T3f z<+w=g^5v=}(X+9N#n#RCpfc1&rYkIqu~egS&Z7ZKn05&;2pL28`&i+Di0$flx&Z#X z<$&nH{ISgB4>y8tcRj?Uf?CDnX?t z$x3S)G4cr7op2VH;iCw)%8AKx{uLvC7_1ruT^zCEp1* ztEv{T8lUAS*G*wuiXIiq$ zED7jGzR3a%fY1ah%|H>Mm-`neR>j9gl@rNd>*j{wTSM>r$%;iUYpQ1s9~>^?s_5!w z_dbM5%zh{>GpUlj#*!MXG{4v?67pHL4wBv#sx=-iH)=TB(@;RsfU(P=G!ic>!7ToI zz^WDBl!F=#h>$QME-2dY@peD;xpjw~QqU@wk(~t1@Rw6366rn&sZ<(^A%;pyh~u4MjCo`F9#3{qv9F2N8>NMDo4Np4oEAuZ<+sZ|G38sH?BthCA=&3gw^>#S(Czjnao zxa34S0b!+$(H)62^vo23`*hvngOtrgsn~9Xdr%M|Cf#~&F zt4wZ>ZR4i_Vy~(JuiTVvhxDH+v`Y@C#M5$(CRvXRPDr6Q)3=OrZYr%lpEY*2b}?rcd?-{%^Qu%{SK4*&8-n*>X|sk zmQcyWkAxUs+XHmv7(;aL$STXd3oJw-X6;;0;A+Aklj&=;qh^#T0<;z)vC^`Sc*YBL z>$6ivJ1AyN*nP)ubwlJ0Qa?dXbjSlgUGjtjKz;fz8kk`J72%ux*lKQxub5429to@} zOIhuNJ{kPqD_x&S>3@&1@yIT&!psT;S)0l97k#mJ0O@%Pa)YY=?hJbK#|k^d-Xw`b z2c5?Kh`>md))lA>c}(}OU$MyaVo|=GuO7MZ*R224X)@mK3x}PzdI^vZ5c!c61=wG6 zrbwnr=xtSDdXdv5fSoYMU#K#;Q0ZuV|9s4Mcgbmz(q7(7tB%;9_5R~kd>Kt%|BW;G z{2iHV?jq`->IniEcQ8o>vPL+-Fqw2xCEZcMZDWqWFp&7QNCDmf0F&A&7@7#W&f^^0 zqsOahNlQ4LaY4`EXbrwILz5cgx?n51Lu37$DX~v2iDm&v5%1O$IQGz2uyuPIbfxRu zJd=3f&vxMgGOx0sbbmWha6bAl0pg68+^-|`H;sE?qd5++vD+)XbAs~E+`AEkl6BOhks z4n&*4afE#x6&15|lbJk0U=N<|BAYBMN^E7Plhu|wx)kH<-QmNTqK}JT$%k-F>gQ_Q zuZ&qdZG*w?w9B2WuN|#88Q?&*!AmDS+So|@D7>%GDdQQWXup42^T_P&)@-E7VZJ<& z;g!9%zUg?k?2D;^sz87&%3OWi4FT1*j)EGw!D#>QS*in!oRP~U9N$<3X6s&JZ&Scw zrqK{wK@(~j>!N+1<+FFUZW`Ak&r4`0%ukI~`(eX3mV`pFwr~8_g}>Qj!8H*DaH{k2 zs3#bFkD4Z?RfA}Cga390WvW^P(zQIoH636CZ+)4o>8oP?v?6$vK;YI6H(eMmwrJ#CTNsynz$lt4~gG}5H?$E-_>wNaB+*dDeavie6 zJP37OHi(QcuX*=E-%e9y$5x6J@NfL3Ot3RHwDb?YVc@rUH(slcVTC%$Rl*)P%~YE% zR2}MGpzyD334D(EJeQ9fj?SrmfP_s|Nq5{H1JI*3gKw^hPOzU`G5JNUw@>X~f+u@;7P@>%m=!7{+ulXo1 zCg+Rc*Rhzr3?vIT-X9bKZc~V{Hfj9HeQp-}?~pSTYh&IBk3mO!6*F%!YDwXD)bl^< z@bYjNNUOawHgHWmW)_37vd567DpKUEyjCv~PuzZR-@*^Py`^U^nN?&wjydhA=eesd zd<5RfW&{bbRyh;pc_KO9Z?9NeEu6J{BKOzUjqRi z2?~5&7e+Z}@`Y8%08JgJ>+aBYwxPz&-evLLD4MV5WdE@3@QE%28w))e^$8o3sEDkh zFw!e<%XD;9>H(#I3)=&q4v?K#6UpElG#ziY-s*kZ1ZFjU%jkQ}9w+qWNGmhT+1$dS zfHZPVHL-nPSz9NDattVs$Ci*O6LRd(bsK2@gfPV>oO zeknqQ?-NXly=g_L!DEOmrF}c&3d7hdN|`E2h-C3I`GQ++PLs!7rlB_p`h+5X+kz5B zB!K4<1C%72+_GF6r^eJq?w^q*r!C+nW7|Qc{ZpvkgY>5)!q5=9byrs@)EnN#d`r|c z;``p1BaI5k#|bvamWuXBw>$I+ddD(zJ1jLjEIIS~l`+rxz}?q)P4o5Y{{tgI+`fm^ z=*1bc<@2G1C=HSdk?i!Q5L;6~)xMtG1ypOwuV>7d;gU17tBqeRF%Rd@pRbvotALs4 zI%kWZw@qy>p}_dEN7-_ImL=+~l1>(Eb+0V4WPHvjMI6zyWN~-M^tjj_JZLqja8{sA zOXQ5{9XMc>5S=i+eT4!sdq%96 zl6+yiq(q2HEvkSdv$nRn^M20N^g;|bM+wd-I(^!uk}R+wvqI)$8vp=607*naRCuaO z*Sd01SZ1V4Vsk#&pi`$#sh?a#k)2BfJt@rpYt^}nRk%X#$G2~HJaXWSZG%@G3{EfNM zOdOia?FXT6&{D#rz7-Xk#i79+tD{llx>ANP zygWrKNwHR^(s=j%cP0fZ4b7>>*cqQQo#l7vd9cJA7N*z(5ouON(K!~%2Ao;??-8h7vBEtZ)f1>IuE z_p~_OxqIiuWibyNLrl4Ic6V_J&Og!ell2%xp~6Tr#f9U@XP+5Vq7)+miDMcxa9kdP z05F)e#OXAQXaQTbs?*|_DWT$;E1h2F>II>>VJLZdN#~b~s`~iXUmF*~OK^^2N+xH- z^I@@?Hf&rH=K=zlJ8O!Ozx#G&F`%W|kmX_!Crz4!l$4Z^~_J#D|n~E<`*4gc0PS;=*`xBs`!mDJv}(@s;F!EiZ9#$){?~ z`?Dc`lTlFBX%I0ebon7etZowql_@G6ggA?AL#b85zA0A?qv~ZjPBxK@t2%x9bQN~a z$Z8q0)fts#cs&M_Hj(Ug;!i{$O+ z(5_Fri;HG-f2L<)Kwd8DMiY0~Cr?@vei~S|QwC*VT^AL_g|iSAPllTFXSg|ApF^kt zFmS_SE7$2e>-DAf!D$10bAjO2G~14No6rGvdOtQ7VP}_ zB)W9&B8H7(*_emumv;Si*Q?)oosq(6sRt;(8~(cX^2m8dLbtH_zxDJ?oJO zHp;QRTD82jI{mvzlO}lSrI(yur!UY51QG(KXLm z7y+JjygK?dTv?|L)=xcz1Aiui>OoJ4>dZY0lCX8gF?4Bv3!Z;=gslJehd#f5VZ@8L z@#>qf`TL{T`-db;J?GiEMz*wf|I%b^_~r<%YS;l|M!hOpsshf{80kf_H7KWYx(P6> z7^ACrLOh3IqSbZltWjCIb+bCHds3FZMGuF&c4+(V)|Lexgb=Rq8!+C|&B!#i_%oD)aaY z@89oQ3Lr9KS})eFZH>E0rbS~LMrEOOCtvmq5I9H9o`pC7{`=0TER3e&!ufChe%L5T3bjF_@kvK(ysYa_NC-G`{S z2xaUNDnX}nvvZLnuZ*)BnM%Ok{*%sm%k;F}+YLDtj#_MFg>o=~)#x?7) zZLZW?EYe!97}}885t$$T$80`iR9o_V zj_+i4WPW6V^nH^0HTfTbb90Acz{Y+URcb6ohmORcBf~Mh;T_1Gwgde}mBQH3p%^*n z2!>B@j;XJA@_z$$jh7%`dZxz~YeSYxo?FmrIJqsC!qEY!x+W1k8g#_3G7?)KpZ|Y6})DkkogVi?HUKkZayN zSxUH#nXN%n!3cy2!{eii5{Ae382KC;CJaw~DGX12DWCOHTQG?J{Y5_s|KhD{bAkY+ z`p%s@W9rnYN_FVu@__!C0i9>goQbZI1>KB9f5L$S2b!^Ix=k7Zj}b6E)8m<5LFVNi z_gdd*Y6QA;=^{Jic#Ik~N`jR5I4)QrA;C-%BSwrsixw@!UPTO6G-(7h0)9atAkOss zf`qQbGzc)%)39N~RD^#@8Z=&^nvF&xMxF1{@T8JBgXfRGnHL0ZxDFgRpi%+l7!ZSy z<((DRg1`U%TQdDDN2gAm(7%6w!}^rB?MD1fBOMy;cw+f#ME$x9jesu@&`i%4sOmDk zMu3d($tR!0;lqb9ckW#EqP3(EMCQY?jvqgctFF2VJ$m%GC`KuLL+43;yPtmgNijTH zgN7Lt>yc-7btS(YEjDTGX=dWN>Xa!{@c#Sno4J{$Yun^M-$T-g@gT#cb(D&|G3- zYV~?R4C>dK?Gd;f}?^=Zcai-^5SM*gH0rwD~-`_p=+=E9Rc|@Hb(ns*a z4?k2sc%w&;#+6rI=@r8}_syMX^TZ-p<YDR^zd);!-_{AbSQ z&+yum6;(N?YZjWBk3-IOgGbr9A6USFh!{oWUQMSt<++FW-v}saKLWr%!^zpN? zx9Tl;u%`sP{%aG`52hflO;faOa0F9Ee~g7U_P~|(;*r>{J!&6YhNY*PpmFNta>~)e|f%~TaRJ#=xL~O5BTpxeOu_3M=-r6c1O_u3eX z7M+S=$(K8Ec=~K)x9@<2kT67)ZGtX6ufncR-@;#q=Hc79JMqi^-p1}52I88n?_kWr z?fBuPSFxmQ15`aS2j4ABMOfKJxUpkP)H?D5KKbzk)=rp;<)IRdB_tYKza5X?Q+MOb zuYSWHujVOdaTdbWeT}gFjd3_}>w{?a=ckx-$hCO3JbN6%M6sG&Lg2c(R@al9+#FGp z=O8*F8VwS)Wv)x$`SBuCGd-TUoRs93^k+JH^r#% zvs^imx(uYShW$`u^B9a+oPgWgQNB8b-`*dM$@Big)~)++P-RU@$G(g*Xj!iUF2AxA z()Q*cqV9F**S;F!)eM!4;&Mh*t7bV=?%o42|2G2TjyJ)bmq~<{&_wiXorPb&n2H0H zZ$?;F-g0u!-xr5{ySHJ(cN4HUq&&*Is3)b>W}P~93`8Tii)vA=9ZJMn6Ff(RMWF9h z_nX!-H@KgKFtp8Dzx1D>#x7?=bwK%?uq{O0s_DN_M2ptS%+`F`9>LJT=+r% z;Sd3aKBHs2p%hjNceM{kPP6`KCOk^G6v9HXk$(0ZvY4k?E1-;vF|j$0R;^lvZp_E0$t47KpV*2!KI(u<36(MP(G@P;qc8F= z0;b;b(O0X6q)O2loZTA~RjO1`H_)M5yL-5Yoqw*Dj11)x#F*q9PK0#AkOw3kkWf^D z9frprm`y<_LKs(s>``0vAB;Bn8Kb(w`d|KX)P3?(RD$aak0l+Ev_|vi&$kw;|91(2 z8fEG#;k{Gi9xh$4FY-SEg(m4S(gQ8!c>csRElJ7pl6Qt@I(&;Yy7>-=y02*s){kyfZ23 zB$7`hE8I?v zf-`!%(x@Q3hW>h#bhz+yO!91*SSQKfJSSze5?({BWoIGI>!E^on6mn$(|KvZSB+1F zbtw?%vSrH#)F$MgfBw3PQX>C>k#ntLZX_s)+8{zlCd$3t@Q1Bgp0kBAV-xhQK^ ztjxU<61mt5naap9TS+Tj8kaBXkz*vx(o*{-UDL8W~1u62{e|!(N z?A(N;YnN*jnzawN*Kxb-;U>+V3rf?&6>1mSDTktw_>I_m(r()-UZAjg87AKCL z#ED}kkytTNF~7QX>Y`q~dTKHj6)BM#euf)UILkQQv2#22?%gNq;W#9eNkH-VxC>a< zu38<%OUB72Axb5Y4GC3;aw>r>E%wf(Wgz9Wj7@p%I=CI{x1=HEOe&5aIWD6(MB&P@ z$zaaJnTwz%^C7^9JxmGh?DiIctUa1tzJ{TbBx&Lp6r2S z4u(j|L#|o5GA7Ph*DDzloP*ocsoA}Uv}gtCcc zl;mb)A2KsCacKV$9N2vjhxQ%8zFh|-D>u-pWh=C8+ZGKPG*FDx$T|zVEN;ZO30u5) zF_!zAvT&_Mc)>ASTiSa=hn@9E^#r{19m4@Q#tFBfF zYAFqxiMZ&`nl(#BtT9u&eAN~vC1zaIB{KKc%i2{gv7B}8$^7HoD^r1U&$`Jmstp;k zi9fXOuyyXqnAo>-Kf=Sp(W({aUOTlhDk_^#qcC+=j^Rvi-@bkLVa5+?@RTf70`;2I zL8E33P`+YWXKnGx?@1?;v2o=lY+kz^M-LrC(<_>yhnRg+0yGnGgD`W}kJzw&gIav+ zH>rz;&FhNnH#V)>jLqw|iQ2vd+O%zhTW-EZ2YL405V)vhj`h{6 zS7X-fS=h936ROp$f_hiT+`FQllJGCO*bg6+)?v?KnR^@6+{?<8zynveQSo7^9@peO z0){a?p3hC2HWeF19d~8BW@vtOQlxhO!hfmQje zD5}(_3CZ~93+DOtp6z?FYUx^R-LM_kU3)Eh_PkxIKivh= z#p^lu=mz!e^lwFNnTo5gZ;8uWG%!@Ucd;o2UdA>aId}{!7O%pZzt=12uXnHBs93RL zf%oVioFm{h)8j$^#~*);ox66*qJ1S=UE5L)^BtI$Id7BCui%06%4KV@WZs|ZPInzw6keg`hUyu_o9_()Tj|2e&}I! zAfKmOru&m;b)S9ynHV5!N2_bDR2vJP7n_-=omlkCGOS;@5$)TxllY+hgWf>PpU-2> znl%_dew@TI%|*LTR|&IgX!wzV(f#4M@Y3J@!iuG<(d>#V@Zf`klxD!_dWu4MUNJqI zGf$W>K^R>NwCi}a9PS5}6%x+?S1ehH`Lq8(&)aSj6~b*kKfB^t=T~2SgLF^}tP+bBG1A7h$dJJ_iqeM5#V~U8lR{bC>+fFTY^Qms8N@`c|@&Z-q!P z@(CoOX5zOW7hv!9eR%50r_iurL%%kkK6=xpO~dQ2zwXz1eN`DZyOSqR#vhCRz|Flo zs$-WxJ1?VA%>T_?D91J1@YsJI(+P+@7%Uz!Js$Xu9XnPm3b*0*0g{VM4*dg}Xu&vr z!jFiLD}kX;4fUByiBhpzKq4&Yd2ctAOo$I;*sw#+qh?H=B@?eX2FcEoiCN7+v~Ydz z{r7O-;C>mO-Nkq+u+JfRwmR$UpV9Hgj=1Zty9~?i=j>Splgk3na$kSrb;K5rMvwlt z6v}y>GrZmTee3$|nDONtv}@B2_uY4&;YNY(v?b84b(`sNxAfotjzDBw1iIhdRr$>Y zTBq%`mqrEiewdG4oA)ZlRJ^#c1|;-B96EF;u4vO39lEp&IO)zFX8bNq!fAac8-}=; zcs&2y^FEi93u=e$dv3^{d1e?YR4s>_dUpyq?Sri!`QJ|^pTYFcen91lRh87|4+$k$ zm>xHq920GBY^@}Ie_)sY`T62=%9*qHR>nkyvWa+l=+k~T#PUU-{86S`he%Wzz5Vvv z5*osycF=Sb4;Bg!4?~wbZbX$@74gcbSCsRyQGKHu!iytcMB`Qsaox@BgNES|pV(qC z==;#^Vu*8GolzQvnJ49NLpEykD3q;K2HozuIcOLjv7lDq?nnD#*PflqVcZjZisC(# z`bWy#Yc2K$ZEk2CRE(jx_&47*xEJ>B+aqe*Nk#GTderBzkwUKC`PJ87Vbhk4xa*NV zB2N3ouie$A8FC$WfR{C@t;~fttltRwPZ>d6h>In}qN_+A!LrNDiwmPp)Nfh`bH4u> zbYrR`4(^3PV?P*+Qz=R4HL$0OkWv_J@MWFh;o+#)xDLMiY6gnM7DKICwR~BfQkl_3 zrE1lxKCQ=Bg>#cmhF76lIdtrPov*4X)N(l{>WW*~4^w9%u2`Jk4e|=L^Bz|2vZP0~ z(VRJR(EEYgf<~oj4>N}By5s&H%9Vg>5pRhO(7Sf;LU)lof|23b@S!93?f3N%6Y%d; zI40Zo{(7gC=HKFE-si!drniFi1`|dr>Lufo#!xhh@+C3N5`z;2Dy?0+7AGWB*iF3+ z&YWQ~K;lcpp~pR4l}iqTE@*O$fXm|Aym|9bgiyKvXdf&fQ(B-1Zm6t1r)dwGV@ZTRi_=`@j^joC* z)HBM#p}FtlrBBu)BRU*e8O721sb}$U2Z;`0ufXgKWJoCED3_n?`9eIsFHajPOVpTz zG9|HW*)nwK&>_z+zWwjL~Yfs;E$C#JQcgG-$Gq!0b7* z(6W7VFOL)aG$y>9e>Lk=M?_>eR<2x$E3drLJ|OyAg@9dlV-K5ozs(arCoj_8=N`k1 z=O*H0-`6nd`;Rek#!G0w=>xp^(;0i!0ZG&G*nQKHZ2#ASzxDLKd?j8bX8;QpEKos( z^Zn%V$JEqRB{6d1c}h8lW@60PB-}RsGfbR54wK$*i)~}x!~6nr!zND}ipM6ojT$_? zFMq$-*Pnm+Isex_{meLtixw?X=QiF{o!-kN#DSW}l7XN% zN`#f>ZJRpnr=tt}H72~Bf7BvfDPv-;kH>@~tODgXmzf@;`d6vxMQp&-rTF7m3*6bM ztW`n@uZ&yz*TddlRwC)A_wd$FXBB1{e@(>eU+x7HN@s7MiDCU7!+$4k#_A8>#F+nz zgwuC09(rOrHc4u+6sPxX_xL%KDPLORPg+#gc25go=J(?9#l4EKa`yMdIB`{PTwB5V zsKTq?gkBBzV!<*AY;33hIfC4-ub;<5cl{63)~&{b|Gt4S4?Tv5di_V?D8jSz`!;#5 zHmh8{qTjRJ(=wJZRlU(d@EN;{bzzRt>X%i;LCH|!OWLz%52j9?>dPv8RyL0&7|h<= zKpFlT6Bo@}59c3a#vVO-)MuCmOTjKF!FV8?BPRVkH1(8nB%1h{;$@MT$A3MnqNwOj zC*$0q9oUXcDY_UeCwF7(o>ay=M)+mjFy`w%NDzhH{2_l}RpRUTa%wH?eeZF+J)t$m zL^-?9)7Dz=3zvY{%yi%>@m|l>nyfV!$tO|}C&mQ-!WJ$;io<%76YjKB>DOP=cOYjTW$OOhT%=a|?SNs*G` zxHJd9hLwy*?Ac_XZrhpv3PV5B zZ;#Oryz$qIkB>)ciZF25GjV;yihMa#=12K{S(oME-f}Nt@R#-=NZ`d2(-2y2oXGWw%?q zVf2{MXy2)=s101^mH%QHUJZAS?1B-)hu|TJ5+0j$04FN<#ml{9!8*_m2@k!5XLglE z>b64&ZzH$GwMEyOBk+8m&8XXL0In8==g!H`W9W)_B<%@7mv`zTHq6<558=Oe*8aB* zZv6H4A1iTtk2~DBY*HCM;XH6m=Tby7U4Uz6r0@8&CKAA71YBD9Trh!Rg&e zNF4A2ZWZPg(*Dx^$KH4osy#9i53RinogZ%{8q%Hk_MvC7HX#}N&=D^;jzDZFC-+J8 zik|gclO*$Yfw{pL4gT%7B!Gb*$%gat1%IJ-y&4|Z>@L=YHAbn0Sg~X^M!qo88FuH+T_mH*Kd36!uAY*=#>7SQ)`R)SVDKAPZ^9!FJnCszi_Cp)g&}+L z$tO^+SsjUNAd9J|l$(yEeMgaAA`ul6@xzIb<}J`|rOWcS_`h z>b0x7*n&$h%Q=^fLx<8)oSB-t?-U!G81U)2A_L&>#$W;_d($6X)lx2lH>vv|nJ!2*vPch6np&^}2s< ziENKQ{sb2Nx>O~+^tk3wVMNudUCi)USWFpI&ByTgb(q9-B;$%LT^c1t)mWxl9){<_ zeZ0gSIt=;LDG{L2^<7{v0kg z%ss6AJ4Ux{+=&Hq7GcOELtN}l(Y=i4dJjJMAin%)8j?>2!k9QeZ{3`K+yrgkunP|j zdZ_3=Y&UxDmZZmWnFo$xweA_xFW|E)lE>Q?Mcc9x-uXrS@gM-wDkN~BU7@JuDpiRoLu{$x6Cmu%IYI(0I< zJ9X;R!I{%3`2Y5<1g@&0eLw6Vi=qg!iL3%zxaFQp?xyC7ST3b$Xj%R2a41#9Em2LJq%g5UT30Vh`{ zG`6nW(=zHVr8SdunGA&XZ$5;#r@j5u&M!<%Ohi~%7={cPQnSR?v$wfkrr%2!_$eJ) zwm_3+P3l<+Yn~hl(T)8{=YaCFKo`;JCQq7&wdN_TzaeVe=<%#@0$RSnePGWoxO3w! z+I9#W+PQWw zO7csEZ?9FWR?qBvYlsooczRPglYH~_<}I5Mb|wtnU+IjleO{u`UkxXul?6V<>#{Nl zBF|n%=!tXi_VUKN@4hRTy+4g4C5E&XTlx^(tiLX3@EpJ=bXjj!s$yw z#qH$a2!H?aPi1{Xj+OkEZ&*Kn{yd#Zun)PpIp{I48#;7rN5{E5zVVDaj=KEK&nrOK z@pHID9fkHCI$%8E#`i+icgI=hLqbAu@ZbR$nHo_iqdR=tdDq*&coi` z9s%P6o{4_)a^GYiyom{QTRCjEuzIy?c;;Kb^ikZ$U3w(sa1NGoJe7Y*5@t zg@1eTq#M0n?j?-uxJ`xV9uI*!ZS>?x;Zq2F0sQQ_v*4t)o<8pIrh^VSNv<_5B0r5O zCf++ric7-nt9ONUM>|rMZgeOtKZ?bHHpkaT$2$$7p{H>!JRCVW+3oRr3uHAu^RbzDR(p5N;sUeJ@bbI*Jb+*2@oPb#`@pEW&tQbI&_I57HkH>r^5&bkRtRO+st$*21^fy?b}k4=Y_o ziSIoEfhW@FDaXl=_~7S$UATAwaj~(GNF{J=>3|l_cHoTsHZ5$>m=4A?H8ZVMW5_Fy zd}`nx(Bket2`NZRPDg4|8p_Md$$Y*QI(2+WI0&<$^^o|XsAtZcLFDC2)Nx3p^&C^7 zQWkI{HuYwqckSBsx&GMr z=r1ZNO6Z_nzH%A7v)01i4)&y*4led&nN0d=*YvU0spVLwrxMQbgmcn8q$Z~cdkLML zozeNF&VqjO>-y&M^GQGGcp*18xB8Z}Cz_t5q@;>(FA|CL?U`FNQu+3plI1l&%$JjX z*VJ)+cvw(`e6r-emqK+_LYnaHr6i{cyqVkzvs>40=+dRjQ?oIsX&xH-`%k>lQwE3^ zIImdv8envEH15XS#eF&oBZrt181-~48(Y#IHad#O%3RnEp{J)0eFLJ~A|Voqln8~T zbm~Dl3TS=m7c)*LMF1aOP3p$zLX^be9^Y^5#dF7mvk|M+H zZZu-4^T2V;&&wlx30{&g@s0N! zH;$91hZjxxdZKmf*08p2_**GzTMnE(hwG;BN4m*-|5-%#&9PM0O>@geFrrf(xNh=i z+;N=wN#>Y<~Jf6WTa$?_e){d%V8l zI`KTrq|_AOXYu}1ZS<4UyGmZ=Ijmt6PqiSwZ%Q52>Fvo>s#w%Tp6K~KsIWxzOkLX1AXatlo z<~Z{|jx#^HMj2Edupsdh@wO zSj?D+Km$d9PvP>@EB=?)c^hcaiOH@80o_M>#b>epyI@UGWtXsi#Ub36YCig(@X?A( z87{3~fkXeZpgh@*b>AV=IZp-@(_ASN&UBpXOcJgMZJP zgO9ge6`o5UoWa%=%dvd9;%~*z5xS*AYYbXEC3g_?>t#G5S6Tw<;#Gv8jYp7eufA_l zJLb~+XRvYU9K8S0=h*mn5=u0pD8r3kf5i0F3ZX#y8{N#y+j1Dk<*&x&rGi z)+8l02~#yLO=X}!feKJ`Y5_)l5CV(d{s?%*8u1Y^D2>>TEq~=`G>$4b6~^Pz`Mbzb zZ$VXF(sEOB8lTQPgr=hcF|D7g@PfXH4Uq)Uwhp@(95{TBm|w_Tx}~*l>Ch5|Rq_rkpn+E36RQxdQBZPbUW7@b5U}>pZ^r-}Aw`{># zY7I+He~NjBnqpMoH1u<=*iKM%3JXSkfDnt`@DF$e*6|UD(QG-ZO0BAY73ElG*r>)P z-iZi^2xvh-N9)aeTum?_S|lSB$DhQ`J%1ur;)#)glhMUSA5C9$LRm-*iEpnOg(Mas zCt^_;9ft4%4~(8Q8J( zQlE&uWfA3G*^3SPZ@{R_M9duFSQ)!CIRy{(ZE+#^3QVVXg2R<}a4o{StU}0)j>4() zv}ai6)Uj?Fuc_9l6o1a?%> zu)H!@cOQo_9%r!N!-K+9kD*;NAS**5Ren_(jt%d9hSTzQA**;qV{3Kp#-|34c6v%aP@FS^ZYRE*}MsxHf=%B)hOJ} zX@M6jd-_zV`*}hVcL!1pIq@Wh_Ndvp!o$@L7G=7`trmvwjIYX7;zFat9kVd!ge``R z>kUW!a_AY_p&7D}QC<;Pl7#4N2eg;f)=Cm-dhK*19EJ>oYv@AE`>j|L}8> zPNfLn@E$%pZI8jPIw1I?c?c@55JjV!O@VW9+Ia*E8nG;OVaN{+Qg0;ARRC_XpQ$i?SnH@24cdJ-MGq^0q_5e zIiFmF&zKRg`(ptc7Z%5iFpN{>Xufq-6 zApg&fudqG7QDsua3lRYk0aXNa9O+4j-GkUCsIDq|cn2poY(tSrHZo(cBS|t2-kgsp)>g@|TkPR|6!tQduSrd1`WM)32WhcR89P!j8i1F;WKVyA|2 zxze<{)0N6S7LEsq zL54=W^72&4zCx64R}uo0^i7YtNRPyvBwD|`0DDi)z}24@MUe#J|aXFaAJW4TxoMD$m(J42$uB4P-c(`y1N zX*RNW5+o0hEitcnp!<(XI;caUDbEOok|J4$qm-sjsi5?%tsu#HAmmAs@c?;p2$aiR zj~ag?MS1F;BF#4qY-u`Ih))O1<-zr>;Wz1Xj5a=tGiByzN)21U>#t+dqV3r9ZcF)F zT7LD>vL~Io{l@|Pqui&f3Tx#&b6s!L6$|zp#(&=^!S^%1#8py5VT`NpucyF9kF$5E zq|b%YrB>!tHWsB>P^gKtP(b3DaMji~aHP53)AUkip>br5HYX8ERw}QJbiqVi+AqdX3S>u?(TH zMi|rG40HZ(Hj>P*AX?H}NNWDgMCf~X>XvMMV;E!|#nOoda4C()!$E7%!>J5C3f{-$ z>8D_D`#Pju=`(kH6@&CS@m z5|s8=lVbo!8Y$|{8jEqE!!UPcQyt<~3$u1MDx=oOp&S=OjGfvS>laSOd%jjEvGvF5 z>35N+?*UJFvT1zUYtdpHnc|P3!M2@VXXR&mOGPPnA&R{ z=1mBsHOLqwb^4TEO49P8j$VeH@m8<>SfDIVzXkgDD8T!Zr{k2tZCs~Ct-x-11 zztjoq@=I&*=Bg+(w$4C-?MO5;c86=>PHfvN#qHmsEB%{0c2em2`&04Cy-$_Ww@^!> zdSYtdAFy!pd+@a?fvx{)EbKcGJ&k|I!)ZN8(;AMG#_z(WeHY>QN|TDtF;9CDF7I87 zvsS~(0t4c404tCbRQ=vdAPXqM?^FqgBgQd zm9Wu8rWEnJHeugINAxZZ#ZTXa%LWyPV6?;}ThF=sY%P+7VV%jr6L^vzkCEk8;?yfDvqELItvq#Jrs4 zrQBvWVl$aS+lpCcLJUjJk(U6Kb-q?5tBkocH-*K~QAPrbNzN7YNRqOF`Hx%6k~9@gB}{$0$=W!CS=)?Y zTXU3Cmc~}~vSbbu4zj$^_gQPp4(yW~a^Oo@#J~D7ch^?T!P=A=TQ+CTu5PTu>>!Q0 zLs?>NuJiW#GkY^z*0$d;HgWX@nf{bUuz&S+XRfUrn6;@fvuy6nTwQ(Go8N!Rx>z`| z$w#Ww{u-o063do#v0zS+Hvwt$0DWQo&g5S#`Ij&0XMD-3v8*rYVn39I$VxC=+QS4lSOKeF=Q3+XYj;8I0dpQc>yV`}39LEY#&J0cZ6Z+-~AyxVYG8KQ0_ zpuFW+bmk_+l7H~$4?o~ox)m}DWb?^JFZ#jVa1U->r-jcpd^Okz9hd%zn59M7Gu8#u z;@&~f#V^p7iZ4HMKh{OtqN8ODQXD6skAz#n(#(<1SOiXb2{vR{c4^lJ?7uFB&)7gr zddWudebT5d;^!Z+|7s!ZUzviyq3+0tNkX2vCq@q$i2eiH$;!>BBbNRdgQZ1#FxF)n z;@%0u#V^`IFYZfpUsQ&a6W?G#m?=6>o`o?!7LY!Oz|TMK$JIi6yfP&agDo#&=fQvA zHGV7{uI|9$oDP^W=0)U(Zo*H&*=Xrwi@b-97&ci}vSleJ@XdlSn0B6wS;NwB@z@cZ zOl=L@doN+|^j?tswh?=;6vOFde@q_aDKi%!BePrdJ+>U%8P*-A;Ln()DB3d?F4N-h zPS8bs!NKDB-JzUc?fEhw3+c`44YB9eby`JlqiT-Oa1>27rS-j47SkspAR_R*B2dJB zdEJ3o4p`6lc=gWLJ(;y-E7onGA8TP^B8>EkijJ@;&X%n2cbRG+6tdl8nzKgjm$2)? zAW*_$mvv!{%*>c&GglU{;!D=o#)kP0e3f-@XwLe7cZWrZ1 z@?kN>y|NE$X64Gd_4i|T7OmKn-?C-$%itezLeVQ>V*KVx8pB=MkHH95GI-7Y}yuju~kgJ|!1~G?4m*S3sswI-R`tgR%r<<(y3{unbQJQm0thT7#8E~`tP8@ zj#edDb@-qGaVg{^{yY|e0+U8C=9bjOmk@HAZ?5645T}A8A?>*sM}xNFEw={t>_2bkTfP{dm|LHfw;1yvwxu6RSQtlus=K;PCLGToUBc*4~|JhK~$t9{0weNx?=txLD)CzMcIf_kgShpgI6Hz@hN*A%hLb?2_HUw7qwN%RHAj>kj_p7FjWnqt z8aZ^rt7H4X?Xm3jAC=(}#QuF64?7G%O`(#}sm%r7=s=TYxO!j<_MJVnxZRg`T~Z%IT?Lgk~IgX zMcc9u=aQvx>OBQ-kK|N<&lYzfxi$Vz4>nEB!4e*jp@jjR`Dg-3ax#)~6Yx(~MHO3? zPE(!40oqJAYh-;(H!?UffH58HYC=}K?E_|GMA=9@Y}X&sWlO0MBSMVXBTzQ-A=(X< z?`JebV=FV1C#E1Z*AA(942>)rE2chm&X*BP8&P9ZKxbG|vjc<1ure!0VoEA!O&d|CfDWFe294>N!`wuQ zdyz-+ccc+Iz0eKwem;sZyWYjyvpzuUfyXeN&)MtInVIGc33NWGlolc?+rr9x)Fi)U zNY5;QiFGsRRsxlg)Y}V9JRX@;3DS>m#qm53__&xO{=aV^U|rlJL)GjBRVuK;_)9o{ z_YqWG+9If=tkOf`VPeL($$Lr#_;dbPEVy8YAps-co%{{PzrBw(-)qvTX%kr!KS}y= zY(1U_4<8qp$Nv{^1gt|`9e0;hDXa<+f9X7J-mhsS%5-W)IJICDJ_xZuZ~p+iVjYhN z4c;w%nAo&}CsCN$#T>xld#>=JQ`fy5oi!-wYJyi?yerq0GG!V&EwpHgxOB-c_4qaD z+ghWw7nyTbxU$P6$rS{u}jJT6w6`U9qY6b?`SX&BdmAuW~kS1T=A-RrJAUr+Ye z1m0fG&`*lO^-?nod@BHMiL24A^X5YpN=hOQJ)Q9<9Er-e@ZWX%=%8)$zno+*_EghY|H+O zBr=Od?)r)?{CFAL8h)QFjmlZf$s_FW?yuR>W!u<=xCpj;`BL`9=5s8A=Ap}DLfFw0 zhuODFm$FrdZjluh&Ch>1koBCgl6|{Ah((dypsN&Lc?=5{irmG% zTKF+rwl$pH7qB?VjvU_2zFxYFZMzW1B6crlOTXC6&dGpcAxBTJ!{07tOIICYw+IqR z?3Zj{&lzmxx9eF@bTNxQ5yY+*65hc7Q}8; znlomybz}V5mORQx?hZDt*L)UH`K%&yS&?klR}0z4%eJy`o>0L_SkgBG*lQaV>m7BC zVMRo0Ka@Yxmk4h0SCq&uU+30r5|(`G2n!28Cg_R^F4woQ@k8dat%Tdc&o;8qbb0zI zd{z0?&|z5;BuU?}0k3UjiX6-QGuMq_{v5zMlEu!bEC;n4@WfAkUbZ3DlW%jM*Sqbb;QZAJycjfHLLO*8dGL;>xG=Hs0k_uUR zOcYDVR8r=`bQTkpz?8izCDOC#s-9K8RP&3%^k^29pkzH-)VcJ*cVQAYJsha{Z_p~!cjP}O0 zCDZYJY$3wey^j?)ywTS&7!$kA!L_#i;d%aJ%-)-a((rYd@o^Ar2K0mV@9$#9?hHQB zUr8Tkd~p*tgwRM&3)NF$6*O2pLkh|&MfM~~MBLosMe31P_z~BlpQi+Icgj#Doth$e zjjyB+(-3tn5`S)AhcNRl@VmSov!;E8^Mfy9%*P+L^xq%*6J6(IljPcZJy zl{i<1B^HzA(B$CF1i zVxfDmc@t6YZ3)6{$M^7QpEH;;Z~~U>zKUF$_|i&?M(KH4xNBtnF+B|0PQd7nCdjyx zAWZ8S5YMcavz2U0PMU?7Tw}Bu`38E|YV)BPY~8)#(zJ53EU_Wk!rj{i%3hUOpl6;{ zJ*#}F<`-tRp73^Qsu(#MkfpkFbA^?25s`GH-oo*N`|)>SXDs;kBM7v@RYhe)1O_%% z!hTGBixzNmZvhLHn%n?wCyYi%lMLKRAl4{q_iFbi!TkfXVBBUT-l$yxRF#8g>R+oh zJ+Klao!W}6sa9|t^dD>==Z>JWE*QRWAtrj4;ozoX^j|VTpyYBNz6~=&i!sX32gBYO zhCVh$=&MIX+8hM*Btj^wo;b8N0>!NqRA8J8S?<)So>IbZ zX{Esu-mj#rl82_Y)QrBK5*ho8%$I1Uili_#BdwtY^{VL96cTE3#ZZ!T3R}0P!pd?T+n}g<}^_BG#}wzA!n9BNag^ zaUliLphcq@U?(bWt@))%p>wCzb+`y~+^aTbkg1y9W6Lu%BZeYNN|+Hb>@>1Qn+1Du za>iBsy!Zo*y15MhG1Wqg#`F7BaIXNiE`LZDm;QBAyF~rw7_;^Q#%O=~{P7(*IV%AAx@zI{%-|F7HCC@&hPmU~5k(m}y+*KX%&mp= z(8NlLY#ARG9%M^kZpGVH0~iQRtRDT+wP0Ap|D%z&u`!A(sI1Df^B}dbgsBRB7`-NU z^l{4uX);rvXJ*|LRe$%AZ2%w>i;1~Qxm1|C%Mg;A(VIyH_@Pxz+)fdk6CTUF6+sYWQ=RZB^y<^9@v z8MW&NhxiTH{jfb|jB9~}RR>YxHxRaRi!~)0J0T8bz2UTI6%lLAZX}%$~|}b)_$1u`DhrQJpQISx*c1st^$UMFbiO0<|0I zRhv7a+nU-7>*h^BAf3q*lhg^H&NW5SzN!Nm7#kq#=u%89a6xJOLkwEe11@?B`mpzU zRvp%)8_{5`tbzuslCnxEyis>7Tyz9)28=|oS2GmcO@>Y2k02H^WY>gFO-W>JFHL$) zMXz=9Fd@(r2DA~j)2DNZIjkC>v{AUZ=qV_y=B`xQaIKaqlWv&QVbWDZIA5uxRlSOz|IzU>WZgoC7y$q1{rGUvXO_N;>T#bn`XsL9|rwV1w6c zxU@?`<2#38Gv6Ji`mQM4xf3^MDbb!)t3cZFyB%L@(c$V6*0_x$%0goyAv!os{Aw!L!z^bT3mCDI$=T7g;a4Mh@+^_jL(1(NELP3( zLX{S+e&8-oviK_^P@fTa0wcYOHr}*3T-!3!p|IP0V!hLVNUfeNfz8Y~56 zm0l}#*_vtlYW`;3>C`lX*V4xn&2v6KM(=qoSutNC0wMw; z0wMw$2#AfI1|*_~h=7QIh=7Q|^NN7j=sm9`E9Of?Ktw=9KtwD+2!yCZI2N4Bbm~ P00000NkvXXu0mjf@4XE( literal 0 HcmV?d00001 diff --git a/examples/tutorials/tot/tot.ipynb b/examples/tutorials/tot/tot.ipynb new file mode 100644 index 000000000..7c2b6c294 --- /dev/null +++ b/examples/tutorials/tot/tot.ipynb @@ -0,0 +1,527 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tree of Thoughts\n", + "\n", + "[Tree of Thoughts](https://arxiv.org/abs/2305.10601) (ToT), by Yao, et. al, is a general LLM agent search algorithm that combines reflection/evaluation and simple search (in this case BFS, though you can apply DFS or other algorithms if you'd like).\n", + "\n", + "![LATS diagram](./img/tot.png)\n", + "\n", + "It has three main steps:\n", + "\n", + "1. Expand: generate 1 or more candidate solutions to the problem.\n", + "2. Score: measure the quality of the responses.\n", + "3. Prune: retain the top K best candidates\n", + "\n", + "Then return to \"Expand\" if no solution is found (or if the solution is of insufficient quality).\n", + "\n", + "\n", + "## Prerequisites\n", + "\n", + "We'll install the tutorial's dependent packages and set our API key for the LLM provider of choice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture --no-stderr\n", + "%pip install -U langgraph langchain-openai " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "\n", + "def _set_env(var: str):\n", + " if not os.environ.get(var):\n", + " os.environ[var] = getpass.getpass(f\"{var}: \")\n", + "\n", + "\n", + "_set_env(\"OPENAI_API_KEY\")\n", + "# To visualize the algorithm\n", + "trace = True\n", + "if trace:\n", + " _set_env(\"LANGSMITH_API_KEY\")\n", + " os.environ[\"LANGSMITH_PROJECT\"] = \"ToT Tutorial\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Task Definition\n", + "\n", + "Our agent will try to play the \"Game of 24\". Given 4 numbers, it must generate a math equation that uses each of these numbers exactly one time to evaluate to a value of `24`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import operator\n", + "from typing import List, Literal, Union, NamedTuple, Optional\n", + "from langchain_core.pydantic_v1 import BaseModel, Field\n", + "\n", + "OperatorType = Literal[\"+\", \"-\", \"*\", \"/\"]\n", + "TokenType = Union[float, OperatorType]\n", + "\n", + "## We use these schemas to prompt the LLM to generate equations that evaluate to 24.\n", + "\n", + "\n", + "class Equation(BaseModel):\n", + " \"\"\"The formula combining the provided numbers to reach the target of 24.\"\"\"\n", + "\n", + " tokens: List[TokenType] = Field(\n", + " description=\"The stack of tokens and operators in reverse-polish notation. Example: [3, 4, '+', -1, '*'] would evaluate to (3 + 4) * -1 = -7.\",\n", + " )\n", + "\n", + " def compute(self) -> float:\n", + " op_funcs = {\n", + " \"+\": operator.add,\n", + " \"-\": operator.sub,\n", + " \"*\": operator.mul,\n", + " \"/\": operator.truediv,\n", + " }\n", + " stack = []\n", + " for token in self.tokens:\n", + " if isinstance(token, float):\n", + " stack.append(token)\n", + " else:\n", + " b, a = stack.pop(), stack.pop()\n", + " stack.append(op_funcs[token](a, b))\n", + "\n", + " return stack[0]\n", + "\n", + "\n", + "class GuessEquations(BaseModel):\n", + " \"\"\"Submit multiple equations as guesses.\"\"\"\n", + "\n", + " reasoning: str = Field(\n", + " description=\"The reasoning behind the submitted guesses. Explain how you arrived at these equations.\"\n", + " )\n", + "\n", + " equations: List[Equation] = Field(\n", + " description=\"The list of equations to submit as guesses.\"\n", + " )\n", + "\n", + "\n", + "## These objects will represent a single \"candidate\" (or scored candidate) within our agent's state.\n", + "# You can update the candidate object to match your own task.\n", + "\n", + "\n", + "class Candidate(NamedTuple):\n", + " candidate: Equation\n", + " score: Optional[float] = None\n", + " feedback: Optional[str] = None\n", + "\n", + " def __str__(self):\n", + " try:\n", + " computed = self.candidate.compute()\n", + " except Exception as e:\n", + " computed = f\"Invalid equation: {self.candidate.tokens}; Error: {repr(e)}\"\n", + "\n", + " return f\"Equation({self.candidate.tokens}) = {computed} (Reward: {self.score})\"\n", + "\n", + "\n", + "class ScoredCandidate(Candidate):\n", + " candidate: Equation\n", + " score: float\n", + " feedback: str" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Fetch data\n", + "\n", + "We'll use an example from the [Game of 24](https://github.com/princeton-nlp/tree-of-thought-llm) dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Example puzzles: ['1 1 4 6', '1 1 11 11', '1 1 3 8']\n" + ] + } + ], + "source": [ + "import requests\n", + "import csv\n", + "\n", + "csv_data = requests.get(\n", + " \"https://storage.googleapis.com/benchmarks-artifacts/game-of-24/24.csv\"\n", + ").content.decode(\"utf-8\")\n", + "# Get just the Puzzles column (column index 1)\n", + "puzzles = [row[1].strip() for row in csv.reader(csv_data.splitlines()[1:])]\n", + "\n", + "print(f\"Example puzzles: {puzzles[:3]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Expander\n", + "\n", + "The \"tree of thoughts\" algorithm is relatively generic. The primary two task-specific components are the **expander** and the **scorer**.\n", + "The expander (the augmented LLM) tries to generate 1 or more solutions to the problem. On subsequent attempts, it is given a seed/candidate value from \n", + "the previous search.\n", + "\n", + "You can update this section to match your own task requirements. The expander can be arbitrarily complex. All that's required is that it accepts the problem and an optional previous attempt (or attempts) and returns a new result." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "\n", + "prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " (\n", + " \"system\",\n", + " \"You are playing the Game of 24. Using the provide numbers, create an equation that evaluates to 24.\\n\"\n", + " \"Submit exactly {k} guesses for this round.\",\n", + " ),\n", + " (\"user\", \"Solve the 24 game for these numbers: {problem}.{candidate}\"),\n", + " ],\n", + ").partial(candidate=\"\")\n", + "llm = ChatOpenAI(model=\"gpt-4o-mini\")\n", + "\n", + "bound_llm = llm.with_structured_output(GuessEquations)\n", + "solver = prompt | bound_llm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scorer\n", + "\n", + "In this game, the scorer is easy. We need to assert two things:\n", + "\n", + "1. The LLM has generated a valid equation using each number exactly one time.\n", + "2. The equation evaluates to 24.\n", + "\n", + "You can update this function to match your own task requirements." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def compute_score(problem: str, candidate: Candidate) -> ScoredCandidate:\n", + " numbers = list(map(int, problem.split()))\n", + " # Check that the candidate equation uses all 4 numbers exactly once\n", + " used_numbers = [\n", + " token for token in candidate.candidate.tokens if isinstance(token, float)\n", + " ]\n", + " if sorted(used_numbers) != sorted(numbers):\n", + " score = 0\n", + " feedback = \"The equation must use all 4 numbers exactly once.\"\n", + " return ScoredCandidate(\n", + " candidate=candidate.candidate, score=score, feedback=feedback\n", + " )\n", + " try:\n", + " result = candidate.candidate.compute()\n", + " score = 1 / (1 + abs(24 - result))\n", + " feedback = f\"Result: {result}\"\n", + " except Exception as e:\n", + " score = 0\n", + " feedback = f\"Invalid equation. Error: {repr(e)}\"\n", + " return ScoredCandidate(\n", + " candidate=candidate.candidate, score=score, feedback=feedback\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Graph\n", + "\n", + "Now it's time to create our graph." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "import operator\n", + "from typing import Optional, Dict, Any\n", + "from typing_extensions import Annotated, TypedDict\n", + "from langgraph.graph import StateGraph\n", + "\n", + "from langchain_core.runnables import RunnableConfig\n", + "from langgraph.constants import Send\n", + "from langgraph.checkpoint import MemorySaver\n", + "\n", + "\n", + "def update_candidates(\n", + " existing: Optional[list] = None,\n", + " updates: Optional[Union[list, Literal[\"clear\"]]] = None,\n", + ") -> List[str]:\n", + " if existing is None:\n", + " existing = []\n", + " if updates is None:\n", + " return existing\n", + " if updates == \"clear\":\n", + " return []\n", + " # Concatenate the lists\n", + " return existing + updates\n", + "\n", + "\n", + "class ToTState(TypedDict):\n", + " problem: str\n", + " candidates: Annotated[List[Candidate], update_candidates]\n", + " scored_candidates: Annotated[List[ScoredCandidate], update_candidates]\n", + " depth: Annotated[int, operator.add]\n", + "\n", + "\n", + "class Configuration(TypedDict, total=False):\n", + " max_depth: int\n", + " threshold: float\n", + " k: int\n", + " beam_size: int\n", + "\n", + "\n", + "def _ensure_configurable(config: RunnableConfig) -> Configuration:\n", + " \"\"\"Get params that configure the search algorithm.\"\"\"\n", + " configurable = config.get(\"configurable\", {})\n", + " return {\n", + " **configurable,\n", + " \"max_depth\": configurable.get(\"max_depth\", 10),\n", + " \"threshold\": config.get(\"threshold\", 0.9),\n", + " \"k\": configurable.get(\"k\", 5),\n", + " \"beam_size\": configurable.get(\"beam_size\", 3),\n", + " }\n", + "\n", + "\n", + "class ExpansionState(ToTState):\n", + " seed: Optional[Candidate]\n", + "\n", + "\n", + "def expand(state: ExpansionState, *, config: RunnableConfig) -> Dict[str, List[str]]:\n", + " \"\"\"Generate the next state.\"\"\"\n", + " configurable = _ensure_configurable(config)\n", + " if not state.get(\"seed\"):\n", + " candidate_str = \"\"\n", + " else:\n", + " candidate_str = \"\\n\\n\" + str(state[\"seed\"])\n", + " try:\n", + " equation_submission = solver.invoke(\n", + " {\n", + " \"problem\": state[\"problem\"],\n", + " \"candidate\": candidate_str,\n", + " \"k\": configurable[\"k\"],\n", + " },\n", + " config=config,\n", + " )\n", + " except Exception:\n", + " return {\"candidates\": []}\n", + " new_candidates = [\n", + " Candidate(candidate=equation) for equation in equation_submission.equations\n", + " ]\n", + " return {\"candidates\": new_candidates}\n", + "\n", + "\n", + "def score(state: ToTState) -> Dict[str, List[float]]:\n", + " \"\"\"Evaluate the candidate generations.\"\"\"\n", + " candidates = state[\"candidates\"]\n", + " scored = []\n", + " for candidate in candidates:\n", + " scored.append(compute_score(state[\"problem\"], candidate))\n", + " return {\"scored_candidates\": scored, \"candidates\": \"clear\"}\n", + "\n", + "\n", + "def prune(\n", + " state: ToTState, *, config: RunnableConfig\n", + ") -> Dict[str, List[Dict[str, Any]]]:\n", + " scored_candidates = state[\"scored_candidates\"]\n", + " beam_size = _ensure_configurable(config)[\"beam_size\"]\n", + " organized = sorted(\n", + " scored_candidates, key=lambda candidate: candidate[1], reverse=True\n", + " )\n", + " pruned = organized[:beam_size]\n", + " return {\n", + " # Update the starting point for the next iteration\n", + " \"candidates\": pruned,\n", + " # Clear the old memory\n", + " \"scored_candidates\": \"clear\",\n", + " # Increment the depth by 1\n", + " \"depth\": 1,\n", + " }\n", + "\n", + "\n", + "def should_terminate(\n", + " state: ToTState, config: RunnableConfig\n", + ") -> Union[Literal[\"__end__\"], Send]:\n", + " configurable = _ensure_configurable(config)\n", + " solved = state[\"candidates\"][0].score >= configurable[\"threshold\"]\n", + " if solved or state[\"depth\"] >= configurable[\"max_depth\"]:\n", + " return \"__end__\"\n", + " return [\n", + " Send(\"expand\", {**state, \"somevalseed\": candidate})\n", + " for candidate in state[\"candidates\"]\n", + " ]\n", + "\n", + "\n", + "# Create the graph\n", + "builder = StateGraph(state_schema=ToTState, config_schema=Configuration)\n", + "\n", + "# Add nodes\n", + "builder.add_node(expand)\n", + "builder.add_node(score)\n", + "builder.add_node(prune)\n", + "\n", + "# Add edges\n", + "builder.add_edge(\"expand\", \"score\")\n", + "builder.add_edge(\"score\", \"prune\")\n", + "builder.add_conditional_edges(\"prune\", should_terminate, path_map=[\"expand\", \"__end__\"])\n", + "\n", + "# Set entry point\n", + "builder.add_edge(\"__start__\", \"expand\")\n", + "\n", + "# Compile the graph\n", + "graph = builder.compile(checkpointer=MemorySaver())" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAGDAHcDASIAAhEBAxEB/8QAHQABAAICAwEBAAAAAAAAAAAAAAUGBwgDBAkCAf/EAFgQAAEDBAADAgYKCwsJCQAAAAECAwQABQYRBxIhEzEIFBUWQVEXIjJVVmGBkZTTIzZCVHF1k5W00dIJOFJyc3SSobKz1Bg1N1NiY3aCsSUzQ0RGV4Okwf/EABoBAQEAAwEBAAAAAAAAAAAAAAABAgMEBQb/xAA2EQEAAQIBCAYKAgMBAAAAAAAAAQIRAwQSITFBUWGhBRMUUpHRIzNTcYGiscHh8BUiMpLi8f/aAAwDAQACEQMRAD8A9U6UpQKV1bpco9nt782UsoYZTzK5UlSj6kpSOqlE6ASNkkgDZNQQx+Xk47e+uOsRVbLdnjulCEpPd260nbi/WAeQb0ArXOdtNETGdVNo/dS2TMq+22E4USLhFYWOhS6+lJ+YmuHzqsvvxA+ko/XXFHwvH4iAhixW1pIAGkRGx3dB6K5fNWy+88D6Mj9VZ+h48l0HnVZffiB9JR+unnVZffiB9JR+unmrZfeeB9GR+qnmrZfeeB9GR+qnoePI0HnVZffiB9JR+unnVZffiB9JR+unmrZfeeB9GR+qnmrZfeeB9GR+qnoePI0HnVZffiB9JR+uv1GTWdxQSi7QVKPoElBP/WvzzVsvvPA+jI/VX4vE7G4gpVZrepJ6EGKgg/1U9Dx5GhKJUFpCkkKSRsEHYIr9qsLwGBBUp+wqVjsvfNuCAI6z/vGPcKB9J0Fd+lAndSNjvTk9b8Oaz4rdIuu2aB2hYPc42fShWjr0ggg9RWNVEWzqJvHhKW3JalKVpQpSlBV7vq7ZvaLcvSo0Jhy5OIP3ToUG2fwgbdV19KUHvGxaKrDw8T4kxnF7CJ9rWyhWunO06Fa36yHSR/FPqqz10YuqiI1W+835rJSlK50UCFx4we5ZRcsdh3hyZdrcp9EhqNAkuIDjKSp1tLqWyhbiQDtCVFWxrW+lVzhJ4TOPcSeGs7LprUqws25Lrs5EiDKDTLYecQgodUykPEpbBIb5iknRAPSqdhwvGOeEAYOF2TLbZityudwkZNBvluKLU25yqUmZCkK9LroSezQpQIWSUoIqCxG5Z3hvg333CrHjmRWzOLA9I3KTaypt5hy5KU45CcUOzfc8XcUtCRs8w1r1hmm1eERw+vWLZFkMW/nyZjzYeuvbQpDL8RBSVBS2Fth3RAJBCDvR1vVVfO/CxxTGLTY7ja2598h3G9xrUqSza5vZBtw7W80oMEP6T1SGyecn2pOtVgfJMNu9yjcZFWPHM/mwb3gaI0GTkseU/LnSmnnedADm3EHTyOVpQSTpZQnlG6z5x+sNxTw9webabLMuicayG03WTbrawXJPizCwHA00Oq1JB3yjr0NBl+z3aPfbTDuUTtvFZbKX2vGGFsOcqhsczbgStB0eqVAEdxAruVG45fG8lskS5tRJsBuSnnTHuUZcaQgbI0ttYCknpvRHpFSVAqsZfq13GxXpGkramNwXj19uzIUlsJ/KllX/ACn11Z6rGeJ8bh2m3pBLsy6xOUAb6NOpkL/B7RlXX4xXRgesiJ1bfdt5LGtZ6UpXOhSlKCKyKym8w2uxcSxPiuiTDfUCQ26AQCQCCUlKlJUARtK1DY3uuO132Neu2t8toRbihJTJt7x2eXuKk7A7Rs76LA0e46UCkTNR15x+3ZAy23cIjcnsyVNLOw40rWuZCxpSDrptJBrdTVTMZter6fv7xvvUgeDZwnSQRw3xYEdxFoY/Zr8/ya+E/wD7bYr+aGP2asJwYt9I+RX2OjoAjxwO6H4XEqUflO6eZMj4VX78sz9VWWZh9/lJaN6yR47USO0wy2lplpIQhtA0lKQNAAegAVyVV/MmR8Kr9+WZ+qp5kyPhVfvyzP1VOrw+/wApLRvWila++C3esh4xcHLdlF+yi6ouUiXMZWIamm2+VqQ42nQLZO+VI3176y15kyPhVfvyzP1VOrw+/wApLRvdDIuB3DzLrzIu17wiwXe6SeXtpk23NOuucqQlPMpSSTpKQPwAVHq8G/hStKArhxi6ggcqQbSweUbJ0Pa+sk/LU/5kyPhVfvyzP1VBhD5BCsnvy0n0du0P6w2DTq8Pv8pLRvc1stGL8Lcd8Wt0K3Y1Zm3CpMeI0lhrtFH7lCQNqUfQBsn11+2iFIu12F9nsGNyNqZgxV+7abUQVLWPQtXKnp9yka7yoVyWvC7XapiZobemT0ggTJ765Dqd9/KVk8g+JOhU7UmqmiLYe3b5GrUUpStCFKUoFKUoFKUoFKUoNd/AG/e0WX8YXL9MerYitd/AG/e0WX8YXL9MerYigUpSgUpSgUpSgUpSgUpSgUpSgUpSg138Ab97RZfxhcv0x6tiK138Ab97RZfxhcv0x6tiKBSlKBSlKBSlKBSlKBSlfLjiWkKWtQQhIJUpR0APWaD6pVJOX327JEmz2uELcsBTL1wkrbceT6F9mls8oPeNnej1CT0r88u5h94WP6W99XXZ2XE22j4wtl3rR790+4GHK8GtvEe2Ry5csfAiXDkGyuEtRKVev7G4o93odUT0TW1Xl3MPvCx/S3vq66F/84sosVxs10s9hl224R3IsmOuW9yuNLSUqSfsfpBIp2WvfHjBZ5ifuenBRzinx4gXqQ2oWXElN3V9wbAVISrcZvY7iVp5/UQ0oemvYGtdfBz4L3Twb8Gfx2zMWm4Kky3JkmfJkOJdeUdBAOm+gSgJAHdvmPTmNZT8u5h94WP6W99XTste+PGCy70qkeXcw+8LH9Le+rr6RfsuSra7bZXEjvSma8kn5eyOvmp2WvfHjBZdaVG2C+M3+B4w22thxCy0/Hd1zsuD3SFa6eogjYIIIJBBqSrlqpmmc2daFKUrEKUpQKhc2UUYbflA6IgSCD/8aqmqhM4+0u//AIvkf3aq24XrKffCxrRVkAFmgAAACO30H8UV3a6FrcDNihuK3yojIUdd/RIrV/AuLHGHPIlgzG12m9y7bdZbTqrQuFbEWtEFTvKrkkeM+NdolvauZSdFSdcgB6d1c2qlG19K12j8RM3d46u8Jjf4vasyDf1X3lj+MKtJ0UwQzy8vbc55Cvl32Wl+6INQdxz7iNDwXiVn7Gac7OJZDdGWLBItsbxWRDjSCOxW4EB3m5NgLCgd62FHZrDOG0tK1X448bcpx2XkN8w2+XefHx2NHlTrMzY4q7dG2hLimpUpxaXStSFb0zso5hsVbL/kOc5TxL4hWix5gvG7dYbJb7jEbat0d9annkSCQtTiT9jPYjY1zd3KpOjtnDPldO1Xm336H43bJ0a4xedbXbxHkuo50KKFp5kkjaVJUkj0EEHqK1+xrihmfGq5YZZrJe28MVIxGJlF2nx4TUl11x9RbQwyl4KSlAUhxSlEE+5A11NWfwRWn2OCURuU+JMlF3u6XX0o5A4sXGRzKCdnWzs630pFV5GWMBP/AGtmI9Aurfd/Mo1XGqdgP+eMy/Grf6FFq41pyn1nwj6QslKUrlQpSlAqEzj7S7/+L5H92qpuurc4Dd1tsuE6SGpLK2VEepSSD/1rZh1RTXFU7JWNatWX/M8H+Qb/ALIrHmKcALVhF6Zk2TIslt9lYlLmM4yzcALY0tZJUEo5OfkKlKV2fPybPuatse7ycdis2+6Wu5LkR0Ja8YgwXZTTwA0FpLaTret6UAQenXoT9+ecb3sv35kl/VV6tWFVVN4i8LaVMHg744ltMgT7qMgTfDkAyIPNeP8AjJ9qU83Z8nZFrTPZ8nLyADW+tUfEfBmcvruW+eVyvzFouGWXC6DG2bg15OnMKklxlbqEJK9KASSjnT3DmTusqp4w40rJlY4l6ccgTH8bVahbZHjQZ3rtC1yc3Jsgc2tVLeecb3sv35kl/VVh2evuyZs7lFzDwa8dzObkqpF4yC323JAFXWz26almJJdDaWw8RyFYVyoRsBQSrkHMlXXdlsnCq22S8366ifcJc29WyHa5bklbZ2iOh1KFgJQNLV2yyo929aA7qlfPON72X78yS/qqism4wY1hVrVc8hdnWK3JWlszLlbZEdkKV3J51oA2fQN1eor7smbO5WT4NlhjQsTTab7kNguON2xNmj3a1y225MiGnWmX9tlC07HN7gaJJGquPDPh3buFeIR8dtcmbMhsvyJAeuDodeUp55by+ZQSN+2cVo63rWyT1r7hcQbbcobMuHDvMqK8gLafYs8paHEnuKVBvRB9Yrsoy5l08rdpvi1+hJs8lG+vrUgD5zTqK405qWl38B/zxmX41b/QotXGq9hlnlW2LPlTm0sTblJ8bcjpIV2I7NDaUFQ6EhLadkbGyQCQATYa4coqirEm3CPCIgnWUpSuZClKUClKUCsd8d+Mtv4H4BJv0phdxuTziYdqtLOy9cJi+jTKAOvU9ToHQB0CdA3i8XiFj9pmXS5SmoVvhsrkSJLyuVDTaQVKUo+gAAmtauC1nm+EjxOHGvJYrrGK2wuRcGs8pOtN705cFo/hrI9rvuA9PKhRC8eDVwauPD6z3PKcweTceJWVuCdfJp0ex6fY4jfoDbSdJ0Omx06BOs00pQKwP4cuLnLPBYzyOhO3osVueg/wexeQ4o/0ErH4CazxUTluMw80xS849cO08Qu0J6BI7IgL7N1BQrlJBAOlHWwaDxe8HPj/AMWuGOQR7Rw6kXC9GUtTgxlEZc5qSQkqWUsJ2oHlSVKU3yq0nqdCvZvCJ9+umJWqXk9qj2O/vsJXMtsWV4y3HcP3HacoCiBretgHYClgBRwhwo8Crh9wLsdsulosSskzqztGS3eJMtyO5MlBtwaACihltXaFITogDkKytSeY5f4VZhcs94f2W/XnHpmK3WYzzSbRPTp2O4FFJHXR5SRtJIBKSCQO6gtlKUoFKUoFKUoFKUoNXOLMmb4UHFxzhHalvR+H+OONS80uLRKPG3d87NuQoevQUsju13gpAVs3AgRrVAjQoUduLDjNpZZYZSEobQkAJSkDoAAAAB6q1+8GH/Sx4QX/ABWn+4TWxNApSlApSlArF/ESJF4d5NM4tXPJ73FsNnsjse4WGMlUiK+AsKS8GuvKtOyCpIGxylSglKt5Qqv8Qpc6BgeRybXaEX+5M26Q5GtTo2mY6G1FDJHpCzpPy0EnZLzCyOzQLtbZCZdunx25UaQjfK60tIUhQ36Ckg/LXdqDwWVNnYRj0m5WtFjuD1ujuSbW2NJhulpJWyB6kHafkqcoFKUoFKV8rcQ2NrUEj/aOqD6rCnhUeELc/Bswy3ZPGw7zstj0rxSWpNxMUxFKTttR+wucyVEKBJ5dHlHXm6Zm8aZ/1zf9IVV+J+D2bitw/v2JXdxBgXaKqOtYIKmlHqhxIPTmQoJUPjSKtpHm/wAIP3QaThueZxNh8N13qXmt6ROZhNXnkUwspDaWgfF1doSdddJ79ar1Nry/8BHwXpzHhEX645ZFQ3GwGSplKXB7R+fshpSCQOZKUguhQ9JaPca9O/Gmf9c3/SFLSOWlcXjTP+ub/pCuWlgpSlQKr/EKJOn4Hkca13dFguT1ukNxrq6dJhultQQ8T6Ag6V8lWCqjxd8h+xTmPnP2/m35Hl+U/Ft9r4t2Ku15NfdcnNr46CSwWLNg4Rj0a5XRF8uDNujtybo2dpmOhpIW8D6lnavlqcqr8LPI3sY4h5udt5veR4fk3xjfaeLdgjsuff3XJy7+OrRQKUpQdW6TfJtsly+Xm7BlbvL6+VJP/wCVjy14lar9bolyvNviXi5SmUPPSZzCXlbUASlPMPaoHcEjQ0PXs1ecq+1i8fzN7+war2Nfa5av5o1/YFelk8zRhzVTNpuy1Q6XsfYt8GrP9Aa/Zp7H2LfBqz/QGv2ahsU424Vm9xnwrLfEzHITTr7zxjvNR+zbWEOLQ8tAbcSlRAJQogVw4nx2wbOJcpizXwSfFo65i33Yr7EdTCCAt1DziEtrQNjakqI61t6/E78+KXnen/Y+xb4NWf6A1+zT2PsW+DVn+gNfs1X8T49YJnEuRGs1+El9mKqbyOxX2O1jp90612iE9sgdPbN8w6j1iuOx+EDgOR2GZfIN+57JEiomPXN6HIZjBtRAADq2wlS9kJLYJWFe1KQelOvxO/PiXnesnsfYsP8A01aPoDX7NctiZZxXKrfbLc2mLbLgy8TCbGmmnG+QhTadaTsFQIGgfanW9kxeDcWMV4jvzGLBdDJlxEpW/EkxnoshtCt8qy08hC+U6Ola0dd9Sr/2/Yz/ACcv+wmsorqxIqiqbxaeUSsTM618pSleMxKr/EKXOgYHkcm12hF/uTNukORrU6NpmOhtRQyR6Qs6T8tWCq/xCiTp+B5HGtd3RYLk9bpDca6unSYbpbUEPE+gIOlfJQcmCyps7CMek3K1osdwet0dyTa2xpMN0tJK2QPUg7T8lTlQeCxZsHCMejXK6IvlwZt0duTdGztMx0NJC3gfUs7V8tTlApSlBF5V9rF4/mb39g1Xsa+1y1fzRr+wKsmRsrkY9dGm0lTi4rqUpHpJQQKrWLrS5jVpUk7SqIyQfWOQV6GD6mff9l2NUDiOU5BZczwHAbTlNlwy5WC4DyblUHxVq3TlLBbZiPK6rad5nAU8y0pB2CN6q/32+XHjHwXyLArVheS4reX8ddjJTdLaYkNp5KEoEZLxPKsK6pCkbTygkkdAdg6UzUazzXrvxVy3A5Fuwq/Y3FxS13JVwVdreqKkLehGOiIxv/vvbkKJRtOm09dkV1rxw0yC4+CFw1tkOz3HylYk2e5T7HGWuFNeSwUqfZQdpU291Kh1CuZI111W0NKZowzwVx/G5uUXHJLdYs6gXNmEm3+PZrImqU40tfaKaaRKdUr2qm0kkJA9sNE7NZNf+37Gf5OX/YTUzUQ42XM+xzlG+RiWtXTuTytp386kj5a24cWv7qvpKwvVKUrykKqPF3yH7FOY+c/b+bfkeX5T8W32vi3Yq7Xk191yc2vjq3VX+IUudAwPI5NrtCL/AHJm3SHI1qdG0zHQ2ooZI9IWdJ+Wg6/CzyN7GOIebnbeb3keH5N8Y32ni3YI7Ln391ycu/jq0VB4LKmzsIx6TcrWix3B63R3JNrbGkw3S0krZA9SDtPyVOUClKUCqnK4fJ7dxdsvdysbK1FZiwwwtkKPUlKXWl8uz10kgbJOutWylbKMSrD/AMZW9lN8wLh8M73+Qhf4enmBcPhne/yEL/D1cqVu7TicPCPIu154PXfJeIWb8ULNcMqnsRsWvYtsNcaNEC3Gy2F7cJZIKtn0AD4qyp5gXD4Z3v8AIQv8PWKPBh/0seEF/wAVp/uE1sTTtOJw8I8i6nDAbhv7c71+Qhf4epiwYvGsKnXu3kT5zoCXJswpU6pI7k+1SlKUjqeVIA2SdbNTNKxqx8SuM2Z0cIiPoXKUpXOhVf4hRJ0/A8jjWu7osFyet0huNdXTpMN0tqCHifQEHSvkqwVUeLvkP2Kcx85+382/I8vyn4tvtfFuxV2vJr7rk5tfHQSWCxZsHCMejXK6IvlwZt0duTdGztMx0NJC3gfUs7V8tTlVfhZ5G9jHEPNztvN7yPD8m+Mb7TxbsEdlz7+65OXfx1aKBSlKBSlKBSlKDXbwYf8ASx4QX/Faf7hNbE1rHw8vCuCvhS5xi2StCND4iSxeseu29MyHUNhDsRW+50dCBvqNelSQdnKBSlKBSlKBVf4hS50DA8jk2u0Iv9yZt0hyNanRtMx0NqKGSPSFnSflqwVSeJt/jLtzuHwcpiY3meRwZbNiW8v7L2yWie0QnvPJsK+T00E1gsqbOwjHpNytaLHcHrdHck2tsaTDdLSStkD1IO0/JU5UNhltudmw+xW+9TxdbxEgMMTZ4/8AMvpbSlxzr/CUCr5amaBSlKBSlKBSlKDHfHfg1buOGASbDKfXb7ky4mZarszsPW+Yjq08gjr0PQgEbBPUHRFc8GzjJceIFnueL5gym28SsUdEG+QiNB46+xy2/W26nStjpsnXQpJzPXlz4dPhIy7dx4SMJtl7wXKrJDl2W5X19Pi0i4sLUUpDSQTtkJBcbeJCiXEqSEFCVEPTWz5FasiE02q5w7n4jKcgyvE5CHfF5CNc7LnKTyuJ2NpOiNjYqRrQf9yayoycQz/G1K0Ic6NcEJJ7y82pCiPyCN/hFb8UClKUHw652balBJcUASEJIBUddw2QN/hNY74Y2u5Zcxbs1zvCrZjmdMplRI6WnEyJEWGp0lCFOjY5ikAnlJHUka5ykQCU4v4SuRxJrEm/sReHuTODkAMeHcJjTeubfe4ltaiAQR1CgQUq65moFKUoFKUoFKUoFKUoI2/Xxmww0vONuSHnVhpiMyAXHnCCQlO9AdASSSAACSQAaxNxS4fRONFqEDL+HWP3llIIZekXdxuSxv8AgOojcyPRsJVo+ndXjNCfOrE0947SSrR9fY9/9Z+epCvRw6KKaKaqqb3333zGyY3MtTV7wbvBbyHwaeIeRXyySINzsd1ieLItMyesOskOJUlZfSxpegFDXZp9139Oux/nRlvwcs/56d/wtSVK2ei9nHzeaX4I3zoy34OWf89O/wCFqr8RpPFHJccEHFnbJiFyMhpxVzVKXNUGkqClIShUdKQVaA2rmGirpsgi9Up6L2cfN5l+CMTk2WpGvNyz/GfLTnX/AOrUrY8qcnTzbrlB8mz1JLjSUu9q08gHR5F8qeo2NpIB0djY3r5qFuiinLsP1rapzySdddeJvnXzgfNSaMOuJiKIjRM6L7IvtmV1r7SlK8piUpSgUpSgUpSgpeafbZif8eV/dVQPCEye/YviliVjt08j3C45HbLWqX4u2/ytPyUtr9osEHor4j6iO+r/AJp9tmJ/x5X91WP/AAhsDu3ETErHa7Oh/tm8itkt96K+hl2PHbkpU66hSiNKQkFQ1s7A0CelenPqqLbvvKzsY04jcS864UN8ScfdydV9mQcRVktovT8GO3IirDymVNOIQgNODYSpJKB90DvW6tk68Zrht/4Xx7llrl3Xkt5LVwZ8RjtMttiA+6WWuVHOEdohJBUor9rrm0SKlh4NmPyLDmEC53m/3yflMMW+fe7lLbcmpjjfK00Q2G20gqUdBHUnZ3Vrzvhlbs+tNpiSJk+2SrTKbm2+5W11LcmM8hKkBSSpKknaVqSUqSQQo9K12lGGeIHFjNLdlub2q03tuEImXY5ZoCnoTTqI7MxljtgRoFe1OKV1VsdwIFcWX8Zcw4PjiVaJlzVmE61R7Q/ZpkqIwy6Fzn1xyh1LXZtqCFoCh7ne+Uq9NZAieDXj8d6e+9eb/cJU6926/wAiTMlNuOLkw+Ts+vZ9EK5BzJHcOieQAATeS8EcZzC55XLvLUie3ktujWydEW4A0G2FuLbU3oBSVhTpPNzHRSkjWuq1QqHCG7cTzm6oeSxb7Lxp2A44udkEK2xXWJSVo5ENCG+vmQpKnNhadpKE+2OzWU7t9t2G/jB79DkVDYBwzVgb8h5zLcmyZTjSWEC/zkvpZQk7HKlCEDm9a1bUfSambt9t2G/jB79DkVvwotf3VfSVhfqUpXkoUpSgUpSgUpSgrGaWuS85a7pEYVLdtrq1rjI1zuNrQUq5N96h0IHTeiN9agznNrSSFIuSFDvSu1SgR+EFush0rrox4imKa4vbjb7St97Hnn3afVcPzXK+rp592n1XD81yvq6yHStnaMLuT4/g0MX2fitjGQwUTbVOeuUNalJTIhwZDrZKSQoBSUEbBBB+MV3fPu0+q4fmuV9XXR8Gu64veeEtvlYdj0rFrCqVLS1bZm+0QsPrDijtSuilhSh17j6KyjTtGF3J8fwaGPPPu0+q4fmuV9XXatTTuT5DbJ7UaRHttsU48HpbCmVPOqQpsJQhYCuUJWslZAB9ry82zy3mlSrKKbTmU2md831/CC8bClKVwoUpSgUpSgUpSgUpSgUpSgpnCGVm0zBorvEKHCg5QXnw8xbyC0Gw6oNEaUobLfKT17991XOsbeD1aoNl4XQYlvzRfECKmTKUm+uO9oXSX1ko5uZXuCSjv+59FZJoFKUoFKUoFKUoFKUoFKUoFKUoFKVrP4V3hjzPBeyOyQnsCcyG13aIp5m5i6CMO2Qshxnk7FfVKS0re+vaa10oMj+DXdcXvPCW3ysOx6Vi1hVKlpatszfaIWH1hxR2pXRSwpQ69x9FZRrQbgH+6TX/AIiZfiuF3PAW7jerxckRHLjb55abaaW71c7AtKJ7Nvale368hPtd9N+aBSlKBSlKBSlKBSlKBSlULiLxLGLq8m2xLUm9LSFq7UEtRkHuUvRGyfQgEE95IGt78DAxMorjDw4vMi+0rVy6S7hf3FOXa6TbipXUocfUhofxWk6QPm36yajTYbeokmK2SepJFfSU9Azb+2Jp4Rf7wXhtrWEPDE4Gp488EbxaIrAdyCAPKNpUPdF9AO2x/KJKkdem1JJ7qxz5At33o381PIFu+9G/mrP+Bj2vy/8AReGJP3LngQUqu/FO7RilSSu12dLida7vGHhv5GwR/vRXohWpXkC3fejfzU8gW770b+an8DHtfl/6Lw21pWpXkC3fejfzVzxYSbe52kJ6TAdHc5DkOMqHypIrGegdGjF5fkvDa6lYYwri5MtchuHkkgSres8qbmsBK2D6O10NFHo5+hT3q2NqTmevn8qyTFySvMxI907JUpSlcaFKUoOrdbi1Z7XMnvnTEVlb7h/2UpKj/UK1galSbhzzpqiubMWZD53v26upA+Ie5HxAVsbnVuevGEZDAjJKpEq3SGG0j0qU2pIHzmtcIchEyIw+2QW3W0rSR3aI2K+w6CppzMSrbePD9+hOpy0pSvqGCMyPJrZiVrXcbvMRCiIUEc6gSVKJ0lKUgEqUT3JAJNQLXGDEHLHKu5vTbMCK+1GkrfZcaXHccUEoDjakhaASodVADXXegTVf47Y5Pu7GL3KNFuNxh2i5+MzYdofW1LU0ppbfO0UKSoqQV75UkEgkVUrzicK5YhOuNhsOUifKu9qbeVfTJekvtMym184Q6pSwhAW5skDWlHu615+LjYtNdUUxFojjedHnoVlqxcR8dyNm5uw7iEptiQuYJbLkZTCCCoLUl1KSEkAkK1o6PWqtZ+Ndty3iLZLHj8hudbpcGVKkPORXmnAUFsNlsrCQpCuZfUBQPL0NVji7hN7yjJc4ZtcB50TMXhNtLKShqS63MdcUxz+55ij2ut9yxvQNSlqvUjMuLeI3KPjV9s8GFapzLy7nblx0NLWWOVvZ6fcHWuh9BOjrCcbFmqKZ0aY2Tp/tb4aNevWMwUpSvTQUkKSQQCD0IPprM/BW+O3TEFQ5Cy49a3zDC1HZU2EpU3v8CFBPXr7WsMVlPgJGULbf5Z32T08No9R5GkAn5yR/y14nTFNNWSzM64mLM6drKdKUr4IKUpQK194g4WvCbq6+2g+QpbpWy79zHWs7LKvUNn2p7uoT3gc2wVcUmKzNjux5DSH2HUlDjTqQpK0noQQehB9VehkWWVZHiZ0aYnXA1ByHA8by2Q0/e7Fbrs80nkbcmRkOqSne9AqB0N1FewxgWteZtj16vEGv2a2PufAmyyHlOW2dPswV/wCAw4lxkfgS4lRT+BJAHqqNPANWzrJ5YH81a/VX1UdJZBX/AGq0Txj/ANLcWG8dwyw4gJAsdmg2gSOXthCjpa7Tl3y83KBvWz85qZrJfsBq+E8v6K1T2A1fCeX9FardHSmRUxaKuU+Rm8WNK6t0tcO9wH4NwiszYb6eV2O+gLQseog9DWVfYDV8J5f0VqnsBq+E8v6K1VnpXI50TXynyM3iwD7C+A/Ayx/m9r9muaFwjwm3TGJcXErNHlMOJdaeagtpW2tJ2lQIHQggHdZ49gNXwnl/RWq54vAOJzgzMguchv0tspaZCvwkIKvmIrTPSHR8aYt/r+DN4saW22zb7c2bbbWfGJr3Xr7hpHpccPoSPnJ6DZIrYrFseYxTH4VqjKU4iO3yqdX7p1Z6rWr41KJUfw0x7FrVikRUa1Qm4iFnmcUkbW6r+EtZ2pR+Mk1K1870h0hOWTFNMWpjnxk4QUpSvGClKUClKUClKUClKUClKUClKUClKUClKUH/2Q==", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from IPython.display import Image, display\n", + "\n", + "display(Image(graph.get_graph().draw_mermaid_png()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run\n", + "\n", + "Now let's try it on one of the puzzles!" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'expand': {'candidates': [Candidate(candidate=Equation(tokens=[12.0, 5.0, '/', 7.0, '*']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[12.0, 1.0, '+', 5.0, '*']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[12.0, 7.0, '*', 1.0, '/']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[5.0, 7.0, '*', 1.0, '*']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[7.0, 5.0, '*', 12.0, '/']), score=None, feedback=None)]}}\n", + "{'score': {'candidates': 'clear', 'scored_candidates': [ScoredCandidate(candidate=Equation(tokens=[12.0, 5.0, '/', 7.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '+', 5.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 7.0, '*', 1.0, '/']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[5.0, 7.0, '*', 1.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[7.0, 5.0, '*', 12.0, '/']), score=0, feedback='The equation must use all 4 numbers exactly once.')]}}\n", + "{'prune': {'candidates': [ScoredCandidate(candidate=Equation(tokens=[12.0, 5.0, '/', 7.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '+', 5.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 7.0, '*', 1.0, '/']), score=0, feedback='The equation must use all 4 numbers exactly once.')], 'scored_candidates': 'clear', 'depth': 1}}\n", + "{'expand': {'candidates': [Candidate(candidate=Equation(tokens=[12.0, 5.0, '-', 1.0, '*']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[1.0, 7.0, '*', 5.0, '+']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[12.0, 1.0, '+', 5.0, '*']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[7.0, 5.0, '*', 1.0, '-']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[12.0, 1.0, '*', 5.0, '-']), score=None, feedback=None)]}}\n", + "{'expand': {'candidates': []}}\n", + "{'expand': {'candidates': [Candidate(candidate=Equation(tokens=[5.0, 7.0, '*', 12.0, '-']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[1.0, 5.0, 7.0, '*', 12.0, '-', '+']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[12.0, 1.0, '*', 5.0, 7.0, '/']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[12.0, 5.0, '*', 7.0, '/', 1.0, '-']), score=None, feedback=None), Candidate(candidate=Equation(tokens=[5.0, 7.0, '*', 1.0, '-', 12.0, '+']), score=None, feedback=None)]}}\n", + "{'score': {'candidates': 'clear', 'scored_candidates': [ScoredCandidate(candidate=Equation(tokens=[12.0, 5.0, '/', 7.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '+', 5.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 7.0, '*', 1.0, '/']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[5.0, 7.0, '*', 12.0, '-']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[1.0, 5.0, 7.0, '*', 12.0, '-', '+']), score=1.0, feedback='Result: 24.0'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '*', 5.0, 7.0, '/']), score=0.07692307692307693, feedback='Result: 12.0'), ScoredCandidate(candidate=Equation(tokens=[12.0, 5.0, '*', 7.0, '/', 1.0, '-']), score=0.05737704918032786, feedback='Result: 7.571428571428571'), ScoredCandidate(candidate=Equation(tokens=[5.0, 7.0, '*', 1.0, '-', 12.0, '+']), score=0.043478260869565216, feedback='Result: 46.0'), ScoredCandidate(candidate=Equation(tokens=[12.0, 5.0, '-', 1.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[1.0, 7.0, '*', 5.0, '+']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '+', 5.0, '*']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[7.0, 5.0, '*', 1.0, '-']), score=0, feedback='The equation must use all 4 numbers exactly once.'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '*', 5.0, '-']), score=0, feedback='The equation must use all 4 numbers exactly once.')]}}\n", + "{'prune': {'candidates': [ScoredCandidate(candidate=Equation(tokens=[1.0, 5.0, 7.0, '*', 12.0, '-', '+']), score=1.0, feedback='Result: 24.0'), ScoredCandidate(candidate=Equation(tokens=[12.0, 1.0, '*', 5.0, 7.0, '/']), score=0.07692307692307693, feedback='Result: 12.0'), ScoredCandidate(candidate=Equation(tokens=[12.0, 5.0, '*', 7.0, '/', 1.0, '-']), score=0.05737704918032786, feedback='Result: 7.571428571428571')], 'scored_candidates': 'clear', 'depth': 1}}\n" + ] + } + ], + "source": [ + "config = {\n", + " \"configurable\": {\n", + " \"thread_id\": \"test_1\",\n", + " \"depth\": 10,\n", + " }\n", + "}\n", + "for step in graph.stream({\"problem\": puzzles[42]}, config):\n", + " print(step)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found a winning solution in 2 steps: [Equation(tokens=[1.0, 5.0, 7.0, '*', 12.0, '-', '+']), 1.0, 'Result: 24.0']\n" + ] + } + ], + "source": [ + "final_state = graph.get_state(config)\n", + "winning_solution = final_state.values[\"candidates\"][0]\n", + "search_depth = final_state.values[\"depth\"]\n", + "if winning_solution[1] == 1:\n", + " print(f\"Found a winning solution in {search_depth} steps: {winning_solution}\")\n", + "else:\n", + " print(\n", + " f\"Failed to find a winning solution in {search_depth} steps. Best guess: {winning_solution}\"\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}