From 42c663e1548e38563f954f3522ea0a23dd4074bf Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Thu, 12 Jan 2023 15:29:10 +0100 Subject: [PATCH 1/8] tf logo Signed-off-by: Max Pumperla --- doc/source/_static/js/tags.js | 31 ++-- doc/source/_toc.yml | 27 +++- doc/source/conf.py | 14 +- doc/source/images/tf_logo.png | Bin 0 -> 29954 bytes doc/source/train/api.rst | 65 +++++++- doc/source/train/architecture.rst | 51 +++++-- doc/source/train/config_guide.rst | 24 +-- doc/source/train/dl_guide.rst | 32 ++-- doc/source/train/examples.rst | 140 ++++++++++++++---- .../examples/horovod/horovod_example.rst | 6 +- .../examples/mlflow_fashion_mnist_example.rst | 6 +- .../benchmark_example.rst | 6 +- .../pytorch/torch_fashion_mnist_example.rst | 6 +- .../pytorch/torch_regression_example.rst | 1 + .../pytorch/tune_cifar_torch_pbt_example.rst | 6 +- .../examples/tf/tensorflow_mnist_example.rst | 6 +- .../tf/tune_tensorflow_mnist_example.rst | 6 +- .../transformers/transformers_example.rst | 6 +- doc/source/train/faq.rst | 9 +- doc/source/train/gbdt.rst | 8 +- doc/source/train/getting-started.rst | 4 +- doc/source/train/key-concepts.rst | 20 ++- doc/source/train/train.rst | 11 +- doc/source/train/user-guides.rst | 4 +- 24 files changed, 355 insertions(+), 134 deletions(-) create mode 100644 doc/source/images/tf_logo.png diff --git a/doc/source/_static/js/tags.js b/doc/source/_static/js/tags.js index 986dd60ccfbc..943953ed8a4e 100644 --- a/doc/source/_static/js/tags.js +++ b/doc/source/_static/js/tags.js @@ -12,22 +12,27 @@ window.addEventListener('load', () => { .then(panelTagMap => { for (const panelId in panelTagMap) { - let element = document.getElementsByClassName(panelId)[0]; + try { // panel might not be in _this_ gallery + let element = document.getElementsByClassName(panelId)[0]; - // For each panel, attach data tags to the 4-th parent of the panel, - // which is the "div" element that we can later toggle. - // Sphinx Panels is too inflexible to allow us to attach data tags - // directly to the container. - for (let i = 0; i < 4; i++) { - if (element.parentNode) { - element = element.parentElement; - element.setAttribute('data-tags', panelTagMap[panelId]); - } - else { - console.log(panelId + ' has no parent element,' + - 'please check if the panel has been tagged correctly.'); + // For each panel, attach data tags to the 4-th parent of the panel, + // which is the "div" element that we can later toggle. + // Sphinx Panels is too inflexible to allow us to attach data tags + // directly to the container. + console.log(element); + for (let i = 0; i < 4; i++) { + if (element.parentNode) { + element = element.parentElement; + element.setAttribute('data-tags', panelTagMap[panelId]); + } + else { + console.log(panelId + ' has no parent element,' + + 'please check if the panel has been tagged correctly.'); + } } } + // simply skip panels not in this gallery + catch(err) {} } const allButton = document.getElementById('allButton') diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index a74e5efbc11b..72f8a424294a 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -74,15 +74,40 @@ parts: title: Ray Train sections: - file: train/getting-started + title: "Getting Started" - file: train/key-concepts + title: "Key Concepts" - file: train/user-guides + title: "User Guides" sections: - file: train/config_guide + title: "Configuring Ray Train" - file: train/dl_guide + title: "Deep Learning Guide" - file: train/gbdt + title: "Boosted Tree Guide" - file: train/architecture - - file: train/faq + title: "Ray Train Architecture" - file: train/examples + title: "Examples" + sections: + - file: train/examples/pytorch/torch_fashion_mnist_example + title: "PyTorch Fashion MNIST Example" + - file: train/examples/transformers/transformers_example + title: "HF Transformers Example" + - file: train/examples/tf/tensorflow_mnist_example + title: "TensorFlow MNIST Example" + - file: train/examples/horovod/horovod_example + title: "Horovod Example" + - file: train/examples/mlflow_fashion_mnist_example + title: "MLflow Callback Example" + - file: train/examples/tf/tune_tensorflow_mnist_example + title: "Tune & TensorFlow Example" + - file: train/examples/pytorch/tune_cifar_torch_pbt_example + title: "Tune & PyTorch Example" + - file: train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example + title: "Ray Train Benchmark" + - file: train/faq - file: train/api - file: tune/index diff --git a/doc/source/conf.py b/doc/source/conf.py index 20c8c89ddb7b..08bec54c7794 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -354,10 +354,20 @@ "xgboostExample": "Classification,XGBoost,Training,Preprocessing,Prediction", "timeSeriesAutoML": "Regression,Sklearn,Tuning", "AIRExamples": "Regression,Classification,Training,Tuning,Prediction," - "Preprocessing,Serving,PyTorch,TensorFlow,XGBoost,LightGBM,Sklearn" + "Preprocessing,Serving,PyTorch,TensorFlow,XGBoost,LightGBM,Sklearn", + # Tags for Ray Train examples gallery + "trainTorchFashionMnist": "PyTorch,Training", + "trainTransformers": "PyTorch,Training,HuggingFace", + "trainTensorflowMnist": "TensorFlow,Training", + "trainHorovod": "Horovod, PyTorch,Training", + "trainMlflow": "MLflow,Training", + "trainTuneTensorflow": "TensorFlow,Training,Tuning", + "trainTunePyTorch": "PyTorch,Training,Tuning", + "trainBenchmark": "PyTorch,Training" + # TODO add and integrate tags for other libraries. # Tune has a proper example library - # Train, Serve, RLlib and AIR could use one. + # Serve, RLlib and AIR could use one. } # Create file with tag mappings for tags.js to use. diff --git a/doc/source/images/tf_logo.png b/doc/source/images/tf_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..6bc2536aad0994650b93d9d799311f23df256c4e GIT binary patch literal 29954 zcmeFZcTiK``Y#+&RJtMxf(Qbk7wJluqJ-Xi?;YvAhytS2(4@D}rAcoI7Ni$L?;_0r z0i{Yy?oRxkbI&{T&b)uTGrzfa=HLv*ti9HLp3l=h>sfmnqoMYUjF^@f1Okyk6y>!* zpewZ?&}Go|tH3uxD!D?yUsvp(s6GLK>JmuKEeU~tGg>QZse(X(tRPVMD-h@u_$vG- z2;}n+1lqI!fyCZ{K-AB(+cYJBfvZ-^&*VW^+@Ji;(qs_G`z}QOiH`s5_Ci3giBBi? z_wj~GT3S^~T29xeuz!u&oq`OiM1?Tk?_w=0{hr-+#Jz-_W^su^ zgj~fCxg`24@*Hojvs2BqflQ%&*(19NPvvbKlfCSQ2Rm{Tu7%Az?@Mk34<+M1X?(o}0(`NC;cOj$1Q)u{pS zbxj<8h1cm@CTj<;g*L_Xi-Zl4k*8TZd%M~UfS}(~6)i4-K#F@c7>dM9--0z;Kgk%qLB z%$N||1>yY)_{mheI*nnb_LXhyxXE%M@7c-{fUM4Wi-;Qp(%?ULc0zg?vEJ^)-`TMX zp&o+!-Vi5O_=gSiZ=o0;vfgnn$0>nFu)EN-ljCyy^vn3^P3Lr(tL9@0#FVBRP7O{r zV#)#dd+*`RkV22Ux2`=GcR6135|Ty}zmx9q9s&?(Ed~b-I~PV+7-h(ljFN@i;@s$J z%#$Vi6Z!W7youOJFt^bg<`dx^E}JHClcX7GC;tq;#Lsuckg4v~{A4mToNT47qJ`LV zj5CMcJgfttK{3Ao^i)C)_*6$7zJ9;G1r2V5oaU@OxMG0&aYrs?Lz5{W{ zvs+30Ga3xafnWFvq;T4i;ca9UP+D;joKtnrr`qbFVKvF2-vR^&a-uhWnCaxi3Axm$ zQaG$Tc%+}7bxz6iOBoL`6pxW26(GRK&neGMliSmzYmI~0ESz&N2H;I3@83F{TIQuI zFYJjIdi4QL>o@&l6-Rt17c$h%7%}}Ai%f2mtYne+1p>eBn*=;|v{-1Skivk?c&k#` z1$bcP8K%8u^^cJ;`sG4;(YFiFmE0@B+ZjB~ z4uJ@kDU9dBs&paFZm+N6{NkY#h7gT1AC(1e@+;G>w_z|$9*4n05u6BJtbl$1@R6dr z%qVW%tp~d>+*BHRO29T2MV%%aoTJQu$t;t%+4W2HG;rFvi2WV>voQg4ZB$Xq;fn3U zK;8CQ@xajGb@!LU-8FAY7O>mvaZ*B~^&I@Ddpyo&{;Hw5tS13&iVu9MZ)A}j);ZzT zS){=U{EJVb$_bl2oQApyh^%o#TxoM$3GWnBPjQ%c3`*Cqs!j?3E3?13ux;zBH~PNc zK(Y)Zs=tY*SGq{@LV`y}F^7xg6KU)Hy@0=hB?9)$kHD#<0F0=`nbW0=E(cQ?fb`nAn*rv+5HCZKh=5-tyg6R+gyCbK$b!~!?xI2-Uu9x1}%mHHZ| z*6mNYJDc#Nl1~u;S3-JAZU> z%QN=jAWD=8jXye2N9hBB82-oo(U682#YrlDyc|Cc^Ed!vJ#Sn=33Ul@9GOe5WeLLX zZShy&Gf*OkZ>p6R$tBbgXI)r_lfoV`yU*NBAQ=(A#oaD4 zCHdwRz}qdxaMlHZL?V-4Uj^ll;(bL?m^AnW)PBAPSb-MjR>U!2V7CJVI=+s8Txco3 z0Ymsm!wtzi{w@Vf?)p2K`@xm|OUL{m5Fswq!ewdgz5@{la^b|;0Mu3ToeNlt>K=aG z^2`@}8hhDiG{ALu2Y9e{_YY2KeC^0jzm!6K*0M8NFD74|yc(GK8j!OEZ*LEOStR7N zf_A&IE$?{>-vE1qpm!?Fuipm_`?x3VV28^X4u3)4GnZ}AE7?y)wgVUUb2dNK>aG(! zfaVm$5}=vOE8kO0LU33AaRc_t?AaafRl=k^GeXNX@|tS4Jk-3`K0FRIK>iLbFX0@G z^hBG+NB3m_zEb-R4_9%&=l7rlmFPKrABjG$^Orf*-2!Mf{b<<6>+3)TA;F<<(U1!* zm(vYIgy=TkAn(ZZVYe^ks&qp0qO2e%dygLer?9o$SK*H#)Rjs>)B356v9fC1i?4?! ziN#gzxkg`jF6iuClPAOZb38511+>icez-4f*@*SPOP)Q-nAS;~41sP2S7)-AvN%!~ z0)8lR73U>gU;P!Ut@USZ-qf>hN%Zj;fSl|uq%weiixMbW=J-`mN8Ou4_mu{}hN+VJV;Y>6)))h0Z* zMq>m*WRR}%KyY^4aRQJH4_n*RPaUqS*}N3-Lbk49$8)M+bDW6S`-O=(i=TZ44#xui z%eZh*I->j}J>b(3q8-Olqd{utnN?J&Q;E0gA!I>!7#H{W3M8wvPWee#Ag9I%ZM~#L z%|_I1$L190!(JqF)=L?n3dPm2E(6)?fI}0;(Ee$Id&GOggUtzDs{~g_&W;c(0pu8GolG_BPwsdI}1mGUE zDJAEr8REtSSR?>Zc)twJoc*C+YPN*l*KfB6rdZ^=@NSVyvY-qQDyq;>M#rfL*ZKbh ziF2Fs(-{lB!Z1@bfvJsY|1C-#%B@jt-wH9R%?qeojJw<* z4$ILpytfnblLf231?N~(|D?EbH+6W{C!w8uYV8!rJxL8-FuHtPYE$_6^fD;?Q%oq) z@@G$acj}+)Q2j3Hj6T<Ot77KJQ}!&jPyZH_XvVX zT1us6^-^5pV_wpK*9q0Dv9{lXqD`Ww{d@{Tu?pW-s=s8SH48Ix4mR6yk4TXIc@`xq0L`f(lkKQN%l#F zEr3lK9bapcf@F~GKviKZpHym-ac(RPd8U82F-Rl$T5sbx-vj|dzCGpi<8)6$zf_6W zZ3hO(lzV@kCkK#F|B1dhb6Ni!lb{(tpK8;J^y~L1yMYY~<~z2^{bX(S!#G8c8ZvL& zI`hGpCh}>>P%^H({8OL;>uYYbg>s>upXFw52L)z_Klu1{;J~nIN-oqfW4uQU@>aO) z!ba>Fa#QrQG&}U)&>Wl!*5)RA$74#4=Z+8P2qGyz0G!CzR*Ruh?y)$8y04)U_7=SM zxErt)11>$k;bKEskvZz0uy*j{dcumn(hhd9F(w%L~1+tsma4>-tETsdi|F@WQ zdR|R)tds5Rf!EZdDLZAJYp~j}7fb({|4^RbQOL^XDaxN&^L>BYzx?<=6S&~XPBY9My0G1AD>m-cv~kDFM^JhF9j|HLFxfZP~in2!zCWTt;iFd%keX1it#Ka9G3L}r3b{3%o+}D zP`H+c98h!0@OwUx265~S0M%Cf$^M7b&kvx63ZDe>f32LVl*kD0djdS6DL84hF*dX)r zZ~c!AdAaqU^TxN6jqFRe2r`aRyc=2AfEvN*gxB$*oo{At!kAR}Ge(xBD3=WKlm_o2j01kR`VX3Kz9QR3 z#Jpi9f?&zhDD1ntxR~I^#{|1rRuQ35Q75h`__pRG0nhvXSyH}M-6?128@>SREANG! zT*FsOtG}L5IPDmSM2cy?AcuON4PfQ)&hyU=Df{fU)Ec*L;L1*e=YN2bXkh2*Jf%zA z*gpBwkcj{!V_U%lP$`h2*w(01Auiaha77mPLy6yy{`V6ejQ^tXZ;xPyYltctsWMV- z@D?9DvIM^G{<}xa2~m(vKOl!tEBD_@DhyI8A(t$D`Cd-_COMGBTyRb(!({N3i@T`) z_RkQg8!MASQZmY_^{6(1%lU5PE5Hp|p4`T5SRq#C$WCbdul)%anc#;-H63EBIurZ3 zjKD+YafP5Mc zqvm`>H9+;#LHwC2OAW9NzUKt$j{^3C=Hm(i057j3GoP=d;sU6ec0)o=v4ROeaq_eA z8(!NUshZMllFh_}OOWQYlVx({e$JW&w2P@t57!8=hXM7I3F8bn^MUWDn;(5i&6XS# z&W5V8^?iNenPZ(demUC}kZSm^RGAs`@O<702VmczY-E0Vt@xRZ*rHUyQ>Z?0LrVM& zZ-11k5{Zrew3Da)~I8163E|ump&1_7RK0vc+N1Kd*{q=W~wjQ3nif0P|?8f=GabE}Q zqr}w>*BaVkZ;RzaKJId~iX9X#x?^j$5io$({EEUAs_-}VcdKegP%f=N5^v~jPfqr( zt7wc4Q8FB0UnK>`m7d}DQ{gv{0v6a%irBQw)iwj<97zYs+uJ|6&t9c@hyYMwVsJu% zYODZ)=G2&u^H7~|Rg7lI%C@%Hn^geKE(D$e{i4Mc#0?a|T4S^ckl<-V>XoX!4xvYl z*i!SGKDgSAkF4)x)6@sBoYGsjH!UE3^5ZT*zoXT+S$&IOz~Qwbj{iyFYYE*)sZ#L< zvuH&z2vX46VUv|`c1f!07JAIu3Ft58TWNfu#~paZ1>gL|iG2|$4D+&`wYr5lCu)_d zQQvZ^H%Ma!f(Q45?MKx7_Bc2G`>+50{x1UmGYG^&ZxXn?zl=m}R^`=^#K`XvTJCI# z9+}l;D*~acD(8IGJou6)M&4V?Zi8<#_iT_g@9p~0Qm;!^*>TSTywN6fLpS>_sEk|p z6^PEcU%SFAd2b9@CX!eTjAKnY_nWm%;5;v0tMM|Ja62a7G7y+EMXu7Q?Xe6KgLHDk zZ~WEsZ^s+=neiK*F8?3*3D|FnOhypS_-JR1ow0^U%Z>SgRo|BSoLPOBl1A~E7Rv>?#HlJfT3C zrw7FapzoU(;8!>VfD$Q$PcQ#dpBgx2LEm-X*EL?47DV`EIrG*3GTmu<3FBZ@^>m!) z!k}wm)ZnUAm(hhh;S+1Sqq2Dnr4dGRDew=)Wqq^O+}f}1PTHrgW^2)}+;#;VMf?1c zx6#lF@Py)Q@M>NZR8N^UZ7V=j@}^=~>d8t$#98HDYrETar&U04PklA|fG33^usO0k z5xRaYPij${TeTqMmt>l5xR9`dQjQ>hKJ+wJy~+ z(dI~NJm;rPv(0tl<~A!=lCze{?Pv^12G4L|PQ)Nv&#HevfQ+=x71=EpxdjUK&K12p z84p3UVxDouxJb;-&N}I>v(i&?2G`i^CI#;%1=$tCs-Eu9LyS03MCtcQVeQW^slcNZgt<}?Zm4vo4bAquV z0m%=lP7@cVrlxZ8)H08OU7HLU56Va(p6l-zHMC0Q9dkA2L0FooH7#r{TQLXf@Y znFCb4dK07~4Y1v$nJV53R#7T$p2io}+S-Bj=~~OVnoTG@;YXvA$H&%6wb{XYkBm)& z-nXi!j?*Z$_n~hy2LyFWlRdo(g*qB1AN?W|2Pbne=#&?LFJ5IFM-x|@nm5iA&b5Ws zR?qLgbgc4fqVshZI?+jw*KqqouXRI_w>i$K(05fDGV3cx>9HEzJ}?NY&;Z0vg@$s? zglC-Yart#PTQCX^CjZRaw9jn{N+eG3Iu?UmYjrlB;mVc4M1@I(t|DTXvBz$9p|5QO z(j^j|B05AHVCk*rBKzOdDjrJLI()QENV9rh(p1d0>a{FhO3d>unkQX_^_Z-AVgDp%i!O6+mw%>8V8Lm^=tB!vyO`$1 z$Xzery^3Y}LN`BlQmnV~?rlES_2A|lc;s4?FM~4OL_n1yGKe#|?~3Tc@t4pf(R+G` z2H5eluvLy1Q@}BGnFd+C+Jd3wrcH)jUt^8CO@bqp6PTXqSWe|sN7W~^tNz{V^DnW?x<_E=Ic}U)vinlEA0tlQbC%r&MB{#l9v()H^FwvRS^#du!Y#! z0S3O*mbK_@jr#AWWpg3E7hhb#lE-$~opA5Az{*mz^iW!u8D<2Arct@65d}ny)(Dxa zq%N`AL*vlx-gZX#V!C6tO>~=g8B2iLOR_2yoeJjd;YqMNC8dNOy;|xu{ih2%k4V-P zH&yyuDLg;3DSj-rcC>y}d<~4wNwKDF!R`hT-aL*2_i3M~LA?B}z8?aFmGiA|Qxn5R z^%&JGKa?Qg<8i@Lt>jhJ&slNiW7AMf08I(p)DhA2M>uO?48kd z&yc~F->m_cmE!ng5;ig~uKXB7R#i{OO)7;dDH_tfy>i3K+QktWy@pxd@VcchB-D)k z)VcXdsfDdXUAo@)qUDf5SU-AT$$T|vPsWR$UEOqabUw5)EA;tlkl*I8jesQ>&h13y zRxS#$5N?2#qVD?ipQFF!66XhxH|D_Q5(jdA2M?=>j2XZ2kD2Nh9-py6 ziAXBFifcsX6W9o3SfD!9>Fw9eJ*Qp}%{C61igibv_mT0?BZ%d4q(|oh{35f{hPE~Z zbm{87-+Sntok@_wSx#b;d@{So*&~~QI1JOW{aAX(XC$QLm6aRW5Sm9IhpYLx9qh@_ zoHTQE2?m%!qLAEoyo&JKo3Y=}YCBoo4@$5IOL1)td zLd~|^B9Y6Wy)QhmOqqM%pE)@42D?|>P(20W1ip6M+cA||wmMOnt02XE{e1W|?I1O- z%3h`7m6vuqdTH_boyh}~z1zfSCj66JyWHO0n9WlqHqxL!?O!ucr&_NM$1}W+D%~M` zsWXkxPQ(@8$B9ACH?B4dH+XZuJN-Des7XE*Y;0_-DyMv!{jgNJ9knHZPW)vWnx3_O zwbJRno6F9h=ipB9z4GtEdFnnqUW;(nQAsy3<=-l%HdpT`&1wp&XCO!ClgRc0yNb0B zMrL0_gojq&(itHy1C)ZWMtWmUfnTV7f4t;ij`BD6g3GxkU7FX^(UU#-L&6nBi-iEW zsWmSDH{VsS#)0}ZR|aS1M1@Mj*6+-(B7>A|nulP?^IjB_qNUPWI~%=IbFZeP!%(Lm zXFo3W9rE7}2$`dsUcAiX}cdm^>xkhh3MXc`&Qz}6>>{JZBRKQlkH z$>wH>(poFNIgb|h1HN87&vzVmcesbj^s{af9+vEN42egNs_2xW~(w4Ek&2ljMslT5YRKekC5fA^Zlq zUh4dz4;bElju&^e0;EY8+Me?8_9yAr-CaKhB3RJX)wO(Yhw-X0W+2=ah#A>29_S*a zS1iP$V0FB)0U0o<+zx<$>|d6&EfKw-zt-*=jAh#G{wUx{Y_Q!H ztx4*Vbu4#BDPH>^&Vn`k?^lbCB- zm((4`LYxWAS3j&H>3^S=Zf4D?^R4r$qm^Li$-^P<~8 zr~xMJM(if+nC>Y-#9CQN|X@tV{;)NAG@}p}W4WA>Tmrv0Cgi&NK00soV&)%pJCIqwWqEU_4o?oL}FPz%%oOpTfVtOJ{r8MuF0 zWL&vJSyk5|5qOoNghf4u_6U&w8X}VddA1H#QDkO1d`GFTOk4 zjs_fzw4ML-lz^~`R|3o|gru%&dFbm1PxM;4bw`?=ioag^9p|lUu(64kD_6rTORPCh zxZiEl#qpr+)BC>Nt0bUu4U4#}5eHj_c!@=UUFx_Wh}V{L zbQ&h*7O3)zz(*QNW)#Z#?)o2G@Q5yS-MUH_LPklDe=t6h=~dQ#tXNSsdcy^Maz28C zWIO?pZ9lBbQCst+GwEwuyf*lPTAdJ5J##pON}#eHGnOn0@ElrYQRF=fmGcVXN}_2@ zyus)g?KId;0_Px~)&bG>rcZhP_hP~lRdGwNlia*h|e_Co{{B={Ly}7(`V|JbvdAaQzYZ^mRR666|{G zkfWlUJpH-0<%+l=Na?BR9LktoWhlLY{#&9+sF)xjTE?+M{uzf~FsPFc_o$P*WIw3* z&+?^pyih#PlJsgF=M%_KNm=RZMLc!TS|(||R>#Y5{*0ledX!->rU;YTy5NH=<1IP0 z-65$PN)ic7hmq~meARRXCh4dT$JPK2BLc^{cJX2O+(jo^%W#qDq~BpCf+#YK{KQn zYQZ12t=|r*X(XB{{aSp?m#`CE;hzSj*V0cf=hTq_)XdiyBdIN)HBwgP*&ciRI@r#G zMh;22R`uovsJNi6bwzFmQ%69-jM-VA%JNR$IGj6NY5FtZYAS7Tycs6qiVNr!k)=E{=^&8qY{c3*zsycTv_L9iR5c>_PKG@IC5QBtG1JS zzRxD8uOZJQh@*t>N5@AZ+H@`SHL&S@*LutZ0XnQ^&1ZS}OudyOxzEMhH8<&Ol@%gt zE0D48c8X3GjOc>@M7|)LajoBNXl&%!+1z{s&PeXFi6}niK`0?tnM}@JLfr()`MePb zqX`P~s>Pl17cT~XNn@VioXFu(aC_u3{LW9A*7($hAkC|O$5JN{$kG~56OsFU#!Dr_ zd6-E{vJtLNC32ACmbp)i*LOXeZIdhXrH(c?0)S0xMIchUeKoo*m31V5$e*K*Ed?)7k+MR8Ue7_4)$O56n zOwve~65%-FQwh*qNC|Ii8uR-#|J?r4GYX(TE^E%e7Nt*4Y(7$`pkI^Zz1tg>qpFG; znuMMEXM`wKcdjg1p2;s<8`CE@Ne(G{&v#{Pm*s+Ka3f4=H0UHx^l*eCFLkkwwI;)D zlK12iVSDbNbj9^s266SXOBEHDuZ|1+SLL@ze;0M4C0ZK~}bh5QGil(fAn2L}3pp@Mog-`!NV1af4Hua$IG@}%K z-vFZ{I+P|<_gvh~-2GkZ&Mrda%y`j`?u?k=jr+8x7dXqHiwnEUJD?X(z5jKQD-GdT zXd{Q+UQWW^Q9QN!9Q#<@P{e_zPR*EFOXt4H7|Q5R!awrZWoxCCuZ_URlJhQ+%{oQxzz;CLr}Nu?*^X;12hfi!PHd*>57WOB z@a@Z#XPCw5pC3Eh?q6pwC3zRauy^m)n&0qxR>9i4l?Cl@ygx|~g^w@g(2ogiWz`Hx zfy=eE>;dw?zk`v41l`V$_W5GjlOyt5bPDTW%Q6({tCtlO3lh*L0XsVco~bk_z;Lh`6atLzK_1fyJ9-vVuOUn$w9I1pWRLnNAvCT!8DL zaiEmPO7l(*4_B^^2uxA4xax*m`|CS7{85H}g=deRKW$}Fvp_wBP~TZ~x$4CEUE8G{ z&kU17t=?;K(68Sd*%1uDO3nm4&V4l9U`#4gC_Jn(m38vWYEgw9bv`kN3b9r->zoYl zY!gGIi=2zmlxAof+wU6Mm0}-zId1_OYkVm=uHmVSx!*?3l&*Vd{ycG=1e30Dj(8aNI&&OujFRrso%DW*INcecm?7Un0_}bpUkWbwpU*ZYj z&S?R}5MsLCW>zBYAQzHHmd>jo4^X4p);?bD(e*n9G&@Z&@v|KK!ponOucmw3BW?i; zyX~D22J~kP1HGwT+9!I*7quGCu~KXF9~T4iQNi`(j*2!jkC1>-?`3Fi0O3i5xxW6?=q1!aG@FDq zBL4T5UfPH8-%zG39z4C=Sq{6+D34Gr-j>GkoMm{fzsN%%8kfKq&*Yqg^5%J=LQ?Bb zzD`l(1oZQ{qdI>mq?WSHdxtP0hP}#zp4mmX%pOjpUCL$49pP-CMdncMK zit-APoY`gn3PbmLRX5gwFCgjg+xycsdUTr?agHLv*v6S4V81)UKFjCkJ~us!sfOse>CKOa99)3#r{2B72$UMQlbgaQ=;HltRb)Fr$~4!7;xCH zzXUzWF&9B;|8{-CP?aTo##0=+sM>%3_aevGv$XT>YWL8idXJH-P(v2M+Ie)#yQ5!8 z&p(ZB5C`P$_Stz^E=qHwP}6Qi_11-Gp+EXRcE$hHh>UFgqN_m#x7^q$Km;vzJiJcV zusmSKFO}P{(vpALArPb*2@PBgbpT4PE8vG-r)?XiqM56yEM(lhQu;@Yq)84qYHfo- zy%QC5abrMl^@SQoW_bL>VUjV<>d3zC+9=d8e19uJfH%OLl%n$UkAw!{+shw9)N!0C z-s>_DhlX5^*w@hsjf5h9UcW?X8JD6I00-xiD;e`AH@t z!yw0RZEO)dcRQP4s{i+9#qta^H_qb#KBaigL1pj<$D&(F4Y^6c+No^bHc;|52G6r; z)X^Vtq82)o!y8v5J>6p-vzS&k*QX_k1xAWfOquz%XHE13QYd3!-w?vpAq zxrsi2K>WIq?gAn5k(gKBI(%OHm8U@;N~x_GAAK0y=i+xObh>CMko=020a`MgB|$$T zGY=cF`tSD_rzCebpgRd%J0}mM*)xX`VVsje0~cwtA7^jv^x3`D8ufwrZCo%DwSA-! ztyfT0g%^D>SLbVaUxctsN|@(+&X z_m|KYoS&6Olx;k@*xy)~9Yrqk&gk;zq6Oz>XW>f|6T&dc7331Ze)+}1jC8qHs}FF_ zAU=Su9o(LV?+lykdm^0i?ayXnzt$Ub z2L4#Z75`$$@mJ5_on4xNrDh<=ZNxa5Hp0)}0Cte)0n=0u=akcIS+;VSTGbS%7DSw4 za<`c~3N+(4fgLjY6#RE5=KMX31qF6obuM%4PS>d13!ZofCq{d4^Y8no-CPXDYBMA@ zglL|*^Ybvj&kLA!d*g2h?@r8?UjrT(WvRO5v)8vWc)0v76Fab)nJTT)Uw33KdJHk- zCQ3%<4ygLOb7Z?G@8s%q3S^~R+%hx(5RaiyV{O;+5ilCw(Yn<$(zZ>)@Al(+xhkqTwb!|CEp$_6WWL8=uE-># zyE$78SrxLdkKuFdnO{maW!ZPOibcIU{b2k`D*v@JWhQsp6+Id+vWmE%sjWxdnxz85 z_v9`r6M^6o22vyNC|+fSz0#~$$>JA3sGcAAVK=mczxfnNb9ZweICMLKZI5)x*iEnb z^?3X23N<-*{|K;;%uIY8v~(!XJ!SAcE3PUJ#Xk&3QcwfkaMqaPb&|Q01aS9Zj62C? zX|v+xXdsAu_I;nWIypO6@@u;c1YSPoWAkJDX|AO*Fa<_SX;E71+~Mx6PUS!bmhhtK zCej1U4>4|>N3GtDJ#9cIQaE7pZN1FxsEm z&)z)=)%Ps>Tvzp zK5rA1=#Jt2Q#GyK!JrQ_*%^5Zb0;EawNUp|L!T&7!TuSbb?XCdiYY%arXI;_epKEA zZ#ny5Ylj(YnWfM^;g{T)m)y({dqFYxQB&lRK$dTw`~myRpY{>0RIUUA4KM?9x~Boa zt5Gr$_vVjce&-r{>8N}zuB7JvX_D3T**6j{V|O#`#Rz##<$w+usZCVt2a!hi<**|f zyMh7X5^(s7IpTy;lb!|^6E48Sp`57Fn6^7+WsWHgwkkvBg0+liCeYnT*9tu!Nok?q z?E>f<7wG)UTSn_0lHZi~Rj3bqu)hLw->`e})cHK0r^I&m2TQfETtENfPhi22;=Myd zYrrxiS~y><5!wRlg}2--5fXe-S`X1;t@yf`iA)Bg@BAJ}SbIrx3*cHr0&c&Gi&$Lm zJe#vD%VRlgS)l5vNj7$>EL|MX70!H8S@p*8p@bWE_u=Q@8&lNIP0Fgld!MuaOg*x_ zztqL6@jL*TfT*l=R$Nc-zH|9leuPR`cVvZErd4{KMYw1DXp=`4Ym2=65IBEnhP7Kc zVRFVY)x)+Nuzz}w;cI4)Bwj($gbs4OD+AZi_tP;!Zz!1P4-9gA zkLN#PxWvNle&dVomAw&W`V@Iu^6kk-vt$v!7EE-Dz8lf&kMVUjd|ofQnCojCk>qqV zy?1uIEyU~eK(P78Dt4~%##pR|B)r9bM-}`{Q%U#YQPZzXhKyBf^E`9WFRJ05dQ)Qd z?hx%C0l}wy)PUFW#K^oey{SJIFpzNiIK9tCw)K=*N^rME{AAT5$wb<<)cpaf_j+~e ziRa-{Pu9=lb>BKc%r#+${^rLW(!W|R273sZMp#r39{95?)bxK44wN1C5G z{T7x!eG576_FkaxOT9|v!AP~jt>|THF0e2X3^&%IQKlI8`SK1U^%}sks`DW))M_Jb zfS#w^cbm-6{l@h9>qRcrZ>FE`10EbVqfc|@l?9aG=i`IgC5nOi+&R%*Zi=VB+qXXU z&XEEQrlK2Hmq|PtO(m&fQJuG1+}@&Th3G|>?+&@UNLo|szYw|y+x$Vt6G@`XNwe+| z61j`U9rtEblCdMJs;2Fpkdp<5dX#@Ds5$;n3SPK5w(q6F-(<2G_0x|d*5Ph}LK1gi zFhDkT^<1|uLz(^<#EFUs7x$RN+W;1G2ERS3jQYmLtoZ`iUqT?XFs)}_CK{sUnpxnt z90S>-aybNRpYB^-Po1y%d}Weh|4zw(7llwmQ}fh>E$4+C+qIr?H@Eub=@IMk5$H6f z+x@Ey-^(vkMfV(Jd!1A@ADI2Nis@#m-k5iL)|?}@StEf@?i@cD9xwD{;}0qzJ+UW; zKQj6HaGZGOsPcNe+xaY~JB>5*OW`ewL`RcXnj`x9YiItO7K3t^087JGzWZ9JsA4_x zH~MwKHzIn97`!%lm=B3GSewd?Jgns-)t1U`A&TZde4HG>YAnrr{J{0A&67Tgpc%A3 zBe*i!9y05xHl49l#jy=GvH6K;_>UQFxRk!zG|!krgO7WgA2u4F!op>`z}XsI(kUZhix{<6y% z`BHQr(eus4>5*upa%Hf$kI$NFTcJ$W+|VG_KGe843DF!MiTwYYiMPjsQnX)Kj9(4D=KFvC!V8V9(>% z>E!48xk89`7Hg{~+-I&xv-t^`<;cB|A5QIJDyb#}&bW=PB7cfkqK%t%;IJ;D!A%Bf z0TWLp{@M^bx~9oeTfMuv8Q3Zr1!*D5JmqR&@N8Fx9jGoqOEXq*bqYQgCKB(_OT*!gVuO92|($Aq}uQ6_<=HRiDoG2n*bG}_jN;E zMh!nVg#{L9(aferY8G+9!7B^a2?NsU6qx!Y+93g1FNQ;bPeoGGzj-qXx1|v+f!7eK zGFN~nu{45DIMI}^em9_z8=J;RdloqJ-6-QA?oQx^VBL7W8lNqA+MLVzGu1%nn|{__ z>B0jO_t;uoT8V=fwHB{4}X$xDpAtp+_A|ArcI9iK554)MFT2ve&z zE^~u%$>R*Ij&%aUFyh-Id}NR5X4L>Elbr*X_$rIMSi3Lba%Z4Etet(b zcwO*~B%rs*QtB1Ted@@_9`^5Es2 zP5m9;1=#HEW?bCe5_>Zvl}4CU(xY@3D84EwF424dU;FHFkfYK>@8?}!o2AR7yf5nb zs9Fcisn9ePHfGRN0Z(B z8QiZwKzY|39&IA1Z0W){wdgi$yq*^-CF-F7MBkJhoUzgFA|i>Pjc7(Yk4hQBcU-D` zXfyyk_hGGkYMAaD8#<~;Q0e{j&+^2@!v6Zr*T~7qNxl7GHRG@Q`BQ?$wCNfcl^g1h zdn^ur!63A9EW{Yo4IE_=?9pf2?R`ZB9|4r48Ao#Aoj2&jlH6G7m#S#BJ>Y} z1yO&+FN_Yt;L)5%<}QLMk4lSYP#EZ3{Z>@-~xjQ*WQN#&7v#>g0EJPPj5-{F;$wX=3!O!sQM*kd7r+x>Gp1x?? z))Y9H##kxTv+Q{v@+=Ko40MH3+YIZMw7!&TaSPru34}M)hy*=I)8qM052DhD>)DS@ zhIP3rlz;7>+|cWp@J;39r0u_hu`m)2)P;3v)+!E+2hsjFa%K)Z-Y z9cwL=5I%my(to2}oQPZgEEar4hRNmW^&hm+f|n~PNxe-^p*6Y4)xCj%@6UaGQE#8n zNKB3*>+<1`FJ3Th%W;FR4hGvg=4Cz)kx1as^`gEU*^&Bky1&fdppEE#L4oNtYm$J) zuM#`=T8=)jvGBazYZSV-F50#@h78J+P`o+FLh{K1leO~>ae|oXm79m%wRXF2dr>*M zKRI&wnkMkh5=*!j-`BgQb{4v?afb;&uMWL)3NzUn_@1Z*9eFs02Kro>Q$y4W%k9mg zLVmKHh>u7qFh3e0E)01=AP~UWWELIJC$(|)s=#OH9+7m=x8YC3+rLz&eYLMpFD1VX z+E@f0+j@_%-!X;ezP5A?(Qz=M~^T=-O22NCmJynf!255wFzD(VgJcI z@p=06DhEZBl7vKzT9<;^wE@ura=m$i6s3wod1s1@6TKW2t9$yMD(@LQOEf8fPW- zcvmhh?#mAo$`=Tv-dksI+y-7#V}GxMPWBMaISwvotEIdsaynGrx=Q5=bwvP&-Fqd% z-)QY}+UiY$5>76hW`Dv`2AB<>I>0?*ni*@^(}$s-jT>JZl&Qbd)koe0I?5 zz&i!cTRYlK#lVWF#ZyBs@;FDjtsf5KDy>@N9Rp5397JP19^VQN^ZOIj6jycMTtgQ* z@;F&1d6nJ-SHXA@JfB!AzZ!3@ zfSJhj*y>pr9=n<(ZB29uco}42{&2h?V&235;5loN-axi@bU7$Jyqr+L9&n;)P}!`%m+hU(hF)ZUS#buvlxPuw9ku zI9>OXnqHEu;i3NN>M5>Y!Zz@|w!-w(&00CojWs!DJ7?#E&+lzF$FSSuYy3kh*8{GB zBB_-N7nS}1yL_j)X*+Ymoh0z){(X-Jew`i$oy}zA>VT!pZCqXi6 zxA%_!UofP8Aw!1A4BfO$TuYgPhk^6P(PGSXKll2LDAt|h(-BNu<%2_3_rzk{^T+?I zy|W67>Wkk!2BL%*fPsXPinM?-r2Y_xkPxJ$LAtv|L_xwKhLBQnkf9`o8U*QdXc)R{ zknWkYw$Hga7w7KWoagz^4crXuS$plZzU%#d-glDqeeBX#ameTXZyEEg+_=&t2yTUX z;b4wjKR0B?GIKRpJGB^yW%cQ;z~n>sisa^!wzf(*@efd046>-JhNmI=bFLj*rz$Rg zh~H^pd%KjSz1Jgxi7mVybw9}5o~M#3=$+nm`R~S4WAU~D8nU*Re#mI%f3xEs<*?=q z=rVr~sHgiLF!#0f(Aw`rJuSM9ixow21M+phj@P3v1|Qlm^M6zX{;+ET+>~Xasfa}H z=FC(|+6^JJfGtp|yaEtAlJLoVpzVfh*n6TB``DQ{t@t@kuFoRsq{~G7CjT|bdYfMr zQ%63;qcsY{^X!%{8S`Zqx{%)mG-IzH|KTgnG0Cm5ksoH)wSz*&s#^N)8qBV0$&I{` zYMs3D{!dqvK%X(9?s$bnayf)kBsK_=4iN1=8%z9yMe+>#D8ZLff~B_*_MW>WEP8&f zK>kSV4_>c(OZGQND2qOCVPj%_*i@HlOL6AHb4zHjf8M=>XS3s;OUlJBf=mm&g6~Ibu^bTXMJ${8tEF=b8KuKx;Dd?mO%@0NX4V2XfVHcq0nkx3A zS?3K~c4?L2&;^g|4F(|Pc*0fo@=qm4OrtzEV(lJMPtFo7E5}qgPDQTv&{?&_T1!32 z4lj-9QeYiDdMBirF1CqXzru`IqloO>I4J}|Ww-2uvi!ClC#&L2{o=i@EEOf26k${%*`WsUQ-Zrq zJH$(sJnAWLiLbwXc~H82{5)Mk)AnozygJTNW5&B%l0k*^WF^a*q)TD5LZRXldQV#y zNn7CY!!8r$YR5j5vq<8-yQs$Cxc?$1tR6eL2R$c6+$@tnB34w7uNbF0G8k@u$L{QQ zR=)+KmAkTXa@iRsN2s7P)+)dG+Fzrz=v@X1+f?>SL{>E^7`8udYV9@f7U_NVW}hU} zkW$~cq8O*(dUeb8Gdt_9BM!uc?_Ixf_`YQs)MQcmns!HC98|j;C!z@*Y=%j1#`s?Jx|iZqfwMLn*3nsQsDo z_#`}Jv<{P!&}JrjXfPQtj-N-oZozF9O^s)b7?+$gx&0q02u9A`Ah|B9&*C+*keisJ zbid5=4jB9InVx%NC%+yi89-YQ!ahofnzZFPIE<_YzTKzo}SddN{n7L4UlBE6T-c!?XL#c+2=1x(QJc9wN7u-5onxiadl* zw3+v!Cvx5SltR?{e^JI#MtU7G(LL88 zwZ}iy_iRfKya~n)@hdCg4Z%JFTA!X$vp+N3r%l_~NcFH+jT;@CKj-AY4}`|gTPhn7 z=0B#zbNjn>tlifA5vbW)z9SozJb?|XD=Y1a-mKXRwD&Qx##hFLO0*bJU8hp$@-7Kn zlH#FJqfo7lMq7A-Z^9Azk_TW$j?rSjc|c2Sl`id z5D#ZDvZiT<9;H4R_x~wa#-dMtD0Ry(() zUNycjd`h21*zYK@j16{c#8#FTG(r!cQNcMlj?){avDBOG2`eumN!<} zRQkR*D|rULuf%y>VUWc*CN%1YeggBaU;EGfY8cL|J!He*(<&wYg*eYIHaQ>N*FOc> zl+a%Kc#KiL7??>Ny;Xcl>41YjWIK=WIUYi@cv$PFxgyPGUukZP^h+mcmu|_*9pvxj zGb;!y&d=SM*ekZKB$G21YZ-8SU?~3SC?lg^5Mi>+bz!sVATDKz8;M>& z#s3u*W>;o5g&hhwP<@x`*BtXs%hWWrH_Zd!T*om>8-{_Z;aj>_XB#zkXt2BrCeb5(G&A zyS%_0arF0QVK1kbD$@YPwR5yLP199-c*rhI^BJ%ht{w%vznPUSs_A9`+#_U;m^RA&dv95rlG?Xm>i5IuFDE~B7jy$iar2)Ac@zX!@YqR`1BPONIr;Z6mgGM-5irpi@5tBhtu(s>))QMp+*@|I|y z>%DXVZmblF|FY9jX#Cnc0DjygQpC7ad(wlSlljvWms`;rL;6~(PcRL$(~P{q2Tv>F zpCS)p`0bvnS+0sxJ+CZRveKA2*(^9l?U}ME6qq^uuuu*$*t{>a>u}XPHX+XGk+{95 z-Ms*MkEU(8hueQRilB9?bb&$3uWrA;M}e#Q`R8LV!PM15{DjPA0d45|?QK0+r1e|n9+jESTfax{4j7uQ*fT}9B1={iJ&$%RKxCf1E !?t#h>kzPz-6@rn)sLR+Q~sbR>Evz5))OwnYr{8}L^`9}BuyDVkh1=Ty&Nt+MF@BK%4%zO!*Rm*C&q>1;q zv~)c7Wz!RwY?6LTKG9gcTh;(jVx9bvPS^Pde-zdxRWrVbM5EE*2zg5ntC%!8{PiW* zyeFfsrC6)4`{(4!;>Rp`MKePt&Z}KOMUoYbdvE^!5>ILrl*RjkwP#EMzG~t>>ze^o zxR>gty#lNKDf!RFGlvdiixnxV7VV{={(7ZJ`w(<0X+K%eCXn@cwIzp`(`-Jn3Aovj z#!pD}iBSspxLH4j_$vY0m5MqF(k@ml2S-Oo5bRY`KTgB0Sf-A=pKU@w%w>nK>WxJ` z+;b8wFW=y#t)iR#EI{k(Bp7L@zPr8u!n-bEvyAnj_ViJG@Heia&aY<)+}xn!dKN-m zxGK`H$wo7$+U7p_tATS^Y&5ZpJe3Y}fOTu@q?iB;W>;YsNL>z^C-YOWk9i}Wl zg%A#fBhx>TgFACGgpT;l9htjOE%sm@^Ep$*N%|aqcXGF2xsc~f*)5>UsY3X1$aJ7S z%x*4iUDUQw3>4-b^QikO@wEPT?`JO`>}`Hnh|FfcW64c20nLv?=yCp>{@phs)5nxr zHm;gK7xE0;+;n(rMMAYn-P*Pw7C!EC2H{0=}y%F1=cC>+Bt1`n(cxWMu^?3cOb0Hv&;6?72na`8= zd#RROB6yjo7(qo>r`@6_z5R0>IIw)bwAJ0sv%XI9wS+4&Cd2k-+V4xB4t4pdZ`D=u z1tgLj#yca=g?oLb=~^B^i2ZgAkl$6T>{#y*4E;8E_E-*d57|P({2*VGsbUK)$MMt7 z?9tEcF-kZ6S9%NmS#%R9tpU1-Uo4H9u+z+3u0IVk zyjI%{yeub+u)2?KMWFt@2QdcP1EX$}am#J*YFy^!eXii8g&ZFa+y409Hk>00hB=n# z{>LA(vnvnlA^X*A`~7FkzcFjp;jDkHyO~JsQQDNpbkxehwzhHg`bSL!T2!Fzr3M;gWWHkEXTV^)r~-f zZD-F>t7vr|ehIs#*nbx%+6LLdB+9@=$Ez1E}+|_*!wCdT_+|j7D%rq5p@^{A~#j$dkGwf zZ*8rleHw(lTg`7MjG_c+HKmb}QmW!d)i4;xxJCjp+5M5%B48e9`^75K{FgdkmEW<` zgKl#~!8(iGTPrNvqiQwuQ~J*=l&_LXryZ~L8ea@A`dM(XgMxI|6}SeT#()wa>>GgB z+q4Ym-RGS1g4t`c>XV6qiT>7@X*b|j3_--7H+Jm!O9hiC5`6WW(V}zY{)_p2*V6sU zlO^Mks|#=R0#7Fz&UK}Y#`~|%px3O?=|ZSgA{Y7AuB-8Z;*RN)JbUO_qyiaB%VFnYEU6RzXS5&foNT%%{ z)6Ygx7nbBCo)!(F>c)FVi(h1r+RqwRDG#9JpVy)Hlm1l#hUP368;zQ`pQNEZu>uF; zh01<*Q%P&j;7`1swe#m`YQFr_NtmpZ_e=lAQPfN`utPI=xi0*2@;-P*DuDu7Wu4%DrPDt9MpD#H*hAcQI6-spB4#mz3`Y*#Tlj1-{uXPiY;Yl;6h3sv%b&aLvHe4Gh`m_J|=@DLi<>CR%N~o{s7h z3WUn4l2cidD(irgz9`d@y&yi;!|I#e3yoSA?NJ53M0%U8m~jb=cas>xm0%B`*(UI0 z+d0z%i~DLuU8)zr5I6b22^TsyT!4oh3DhSg>8}?^aHvv4L-uDV*#I{@>R(r#k`_1) zhkGZ_Ig|#})yQA?`1+()m^V?QY)emHWZu$g-@F6(HVxgE-<`F0XN*Z%S;%?4h#Kfc zyy5F(k^DIybmAoz!fD5u*Pg}8G3jj<-(ycH(l*~z$@Au7{;S0%Oi<87cQx4lRm3Vf z!ad=(4!FI--77E1JW5|7^#T7y&`JT?AE24BhB*1K^6QNLMI&5JDwQ;^&wG&y z(WygecVE|V6^(Oy8DGQ%0;?!B1M7fD=*x7j>Fpc zIIcM?73*sf8XX>uGiR;;-G6zA&J2;mhun7_0I2x5Yka1QzCyKYSf*%OB*L}mTj56e zB_g`MCR7V>MLSM$Y-uQAL*2o9*hYmL}l`aHw z{TT)Zhr_9kjnfC!FLA0t<;`2NbRY>T{r8;Kx@Y+u_TII^<&fTYoM9Zu3UKuO;FZF? zEck6RiK!!b8-yEF8lCQG!u-@r{AY+^!mI`WdzDOkL6+q()y^}N(=g2d@A%X#jX54s z^cFuxR0YJ?yq@5-k~<^1+nJf9=|OkwDHhxH_noh2JszCgzd^-Nd)<0r3&QC8A2n59uYlqi=;=~LhO{9#>MY=MnRO)3VJA3CN~74=sUv0GR$ zadMtx&PJcCnE)9c6lL=>HUE1iG$MxVOhYcwpJ5Z;G*v0W_{yqVoF>yx*}Tu@ItF5X zt3jX%^v`FeYJu?i&YCIftl!L&P5srbwcyj*6va!==e2?2BeY#4hr9jlF8nq%oo^-f zq6pG{smfRG&9hz3af9i_slg)g0bdBibb4}jJ_|ILQYia#!${M=`Bqk!PxP(Utk1+> zkI_xhi zwY=~yZg?g*m%)Ms)RIiGyAec|l&qMtkLzUwR}p_X9wY>z6%A*)&$qt<@c5+eR7>tt zOB2k_Ix)3dCAQo2$!FL|i9JgG?3H(ROqcgJNX+?(=s z$7%4X%`ssW}44Y zEu7vhFAjw9Fsp*^)Cd$Rsq`S!22cMDLKx(2otz!KPLQg_=4{`wfK8P_5+K#7WD1Mu zh%G$7V@H5u-kmVF66Ypnn5?{WEuNU_rB$uL)yLAlcIP4G)@Vg6g&WyDuSM!T56iK6 z_pr=3bJw>SnN*O8L?$o5$^OM52f|tKk>4>CE}$|etEVK~_%8Z9Lm2qR233oH%QTos zQ94sB(XR4|cLs|vd}yX0<34R*5p|5;v^eq>=O1z`s(=s04t+UHN!>pOD$qC4u|4kC zM2_uJ4<(c?25V;cFnB~ef8H80-!)RJl976)vD+;SAAA&urMQ5urMtB6*g&r_jXRVd zWuL2jvr)b0Ja7drl1(z6k z81uMXM>*^c?~koRxa36Rgz)@p%)L~mg>ygrWuPFrc5N*!tvUz)a^WB7(I#cf>YcZr z){cr4r9DD6R)OQ;Ewn*BPQJODZbCE{+ut{%6)Lqt>>`H=(wxiJ%Yw^%cEFBcl{cij zvYNV|1LWpE7VaN0Xsxa5va54aTC)E8M8X=^aaF*9ga=)q7;aM7VH%8wPu1#ogn!xr zll!Sb_IASx%7ZUadIJa0SN)L`x~P-T7-?GIV&l#WO$O7Gyozu1HE1=zcn_kDVzJN~ zf^c=x(;K5rV@Z9ih<0ITLR2z|Nam?(Y2~fp1ZtN&7#xKRO55_DVmdfHIm6L+kX*vzq_Gna(S?>6v;=<|8P znWFp8Q^!c)LWrO>q#{~9!lf*(f#6sa%sGC#t=LQxd2EI0`H`q!xbG((~=Csh3pt zny9#L@>cq`@SN;i4^AT*o8NCD*ekp*2%n7I>8cuhvLOn5>eBidOq_j}1?}8h@&1eo z;rKEm;O=DBd`p96O*?un_0fXiPiE3U1?Q>f4Za$)B7ckHfcqDleeK9`hH~rj>(1-VChV!D*LF5~z{^V&VIa|oEm?=n3WHA*EX9W6*n*a26QvVWH2E2IM#IGf z3*5ThF}_4r;dRbZkAyWQzDZ47Hv%R0?Ng1YZ|jU;TYTF|6lXR6MsNkBqlaBQ$6XGw zd1&7^6jdiFoIatQlXtDOFxj?qo_4)imP&g?5+^Yf<(*Hlu&)Wn>-BM3Xb=M${50Eq>T~*o zbq6_kSqE)B=C)c%F{1@;_w9+3WnQuT65g5z()N$jCKV8Lud%VH*Y*P1q@w>ML=QuKIZ>;gPnt!m4(;;{S7em7myeW@`vSK{)c{Q^zMHFm%c4a literal 0 HcmV?d00001 diff --git a/doc/source/train/api.rst b/doc/source/train/api.rst index 424aedfd0acf..61ea64fefa31 100644 --- a/doc/source/train/api.rst +++ b/doc/source/train/api.rst @@ -2,6 +2,7 @@ Ray Train API ============= + This page covers framework specific integrations with Ray Train and Ray Train Developer APIs. For core Ray AIR APIs, take a look at the :ref:`AIR Trainer package reference `. @@ -14,12 +15,17 @@ Trainer and Predictor Integrations XGBoost ~~~~~~~ +``XGBoostTrainer`` +****************** + .. autoclass:: ray.train.xgboost.XGBoostTrainer :members: :show-inheritance: .. automethod:: __init__ +``XGBoostPredictor`` +******************** .. automodule:: ray.train.xgboost :members: @@ -29,12 +35,18 @@ XGBoost LightGBM ~~~~~~~~ +``LightGBMTrainer`` +******************* + .. autoclass:: ray.train.lightgbm.LightGBMTrainer :members: :show-inheritance: .. automethod:: __init__ +``LightGBMPredictor`` +********************* + .. automodule:: ray.train.lightgbm :members: @@ -44,12 +56,17 @@ LightGBM TensorFlow ~~~~~~~~~~ +``TensorflowTrainer`` +********************* + .. autoclass:: ray.train.tensorflow.TensorflowTrainer :members: :show-inheritance: .. automethod:: __init__ +``TensorflowPredictor`` and ``TensorflowCheckpoint`` +**************************************************** .. automodule:: ray.train.tensorflow :members: @@ -59,6 +76,9 @@ TensorFlow PyTorch ~~~~~~~ +``TorchTrainer`` +**************** + .. autoclass:: ray.train.torch.TorchTrainer :members: :show-inheritance: @@ -66,6 +86,9 @@ PyTorch .. automethod:: __init__ +``TorchPredictor`` +****************** + .. automodule:: ray.train.torch :members: :exclude-members: TorchTrainer @@ -74,12 +97,17 @@ PyTorch Horovod ~~~~~~~ +``HorovodTrainer`` +****************** + .. autoclass:: ray.train.horovod.HorovodTrainer :members: :show-inheritance: .. automethod:: __init__ +``HorovodConfig`` +***************** .. automodule:: ray.train.horovod :members: @@ -89,12 +117,17 @@ Horovod HuggingFace ~~~~~~~~~~~ +``HuggingFaceTrainer`` +********************** + .. autoclass:: ray.train.huggingface.HuggingFaceTrainer :members: :show-inheritance: .. automethod:: __init__ +``HuggingFacePredictor`` and ``HuggingFaceCheckpoint`` +****************************************************** .. automodule:: ray.train.huggingface :members: @@ -104,12 +137,17 @@ HuggingFace Scikit-Learn ~~~~~~~~~~~~ +``SklearnTrainer`` +****************** + .. autoclass:: ray.train.sklearn.SklearnTrainer :members: :show-inheritance: .. automethod:: __init__ +``SklearnPredictor`` and ``SklearnCheckpoint`` +********************************************** .. automodule:: ray.train.sklearn :members: @@ -119,6 +157,9 @@ Scikit-Learn Mosaic ~~~~~~ +``MosaicTrainer`` +***************** + .. autoclass:: ray.train.mosaic.MosaicTrainer :members: :show-inheritance: @@ -132,16 +173,32 @@ Mosaic :show-inheritance: -Reinforcement Learning (RLlib) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Reinforcement Learning with RLlib +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``RLTrainer`` +************* + +.. autoclass:: ray.train.rl.RLTrainer + :members: + :show-inheritance: + + .. automethod:: __init__ + +``RLPredictor`` and ``RLCheckpoint`` +************************************ .. automodule:: ray.train.rl :members: + :exclude-members: RLTrainer :show-inheritance: Base Classes (Developer APIs) ----------------------------- + +.. _train-base-trainer: + .. autoclass:: ray.train.trainer.BaseTrainer :members: :noindex: @@ -165,8 +222,12 @@ Base Classes (Developer APIs) .. automethod:: __init__ :noindex: +.. _train-backend: + .. autoclass:: ray.train.backend.Backend :members: +.. _train-backend-config: + .. autoclass:: ray.train.backend.BackendConfig :members: diff --git a/doc/source/train/architecture.rst b/doc/source/train/architecture.rst index 2c44247fe843..ec78a225f3f1 100644 --- a/doc/source/train/architecture.rst +++ b/doc/source/train/architecture.rst @@ -3,42 +3,61 @@ Ray Train Architecture ====================== -A diagram of the Ray Train architecture is provided below. +The process of training models with Ray Train consists of several components. +First, depending on the training framework you want to work with, you will have +to provide a so-called ``Trainer`` that manages the training process. +For instance, to use a PyTorch model, you use a ``TorchTrainer``. +The actual training load is distributed among workers on a cluster that belong +to a ``WorkerGroup``. +Each framework has its specific communication protocols and exchange formats, +which is why Ray Train provides ``Backend`` implementations (e.g. ``TorchBackend``) +that can be used to run the training process using a `BackendExecutor`. + +Here's a visual overview of the architecture components of Ray Train: .. image:: train-arch.svg :width: 70% :align: center +Below we discuss each component in a bit more detail. Trainer ------- -The Trainer is the main class that is exposed in the Ray Train API that users will interact with. +Trainers are your main entry point to the Ray Train API. +Train provides a :ref:` ``BaseTrainer```, and +many framework-specific Trainers inherit from the derived ``DataParallelTrainer`` +(like TensorFlow or Torch) and ``GBDTTrainer`` (like XGBoost or LightGBM). +Defining an actual Trainer, such as `TorchTrainer` works as follows: -* The user will pass in a *function* which defines the training logic. +* You pass in a *function* to the Trainer which defines the training logic. * The Trainer will create an :ref:`Executor ` to run the distributed training. -* The Trainer will handle callbacks based on the results from the BackendExecutor. +* The Trainer will handle callbacks based on the results from the executor. -.. _train-arch-executor: +.. _train-arch-backend: -Executor --------- +Backend +------- -The executor is an interface which handles execution of distributed training. +Backends are used to initialize and manage framework-specific communication protocols. +Each training library (Torch, Horovod, TensorFlow, etc.) has a separate backend +and takes specific configuration values defined in a :ref:` ``BackendConfig```. +Each backend comes with a ``BackendExecutor`` that is used to run the training process. -* The executor will handle the creation of an actor group and will be initialized in conjunction with a backend. -* Worker resources, number of workers, and placement strategy will be passed to the Worker Group. +.. _train-arch-executor: +Executor +-------- -Backend -------- +The executor is an interface (``BackendExecutor``) that executes distributed training. +It handles the creation of a group of workers (using :ref:`Ray Actors`) +and is initialized with a :ref:`backend`. +The executor passes all required resources, the number of workers, and information about +worker placement to the ``WorkerGroup``. -A backend is used in conjunction with the executor to initialize and manage framework-specific communication protocols. -Each communication library (Torch, Horovod, TensorFlow, etc.) will have a separate backend and will take a specific configuration value. WorkerGroup ----------- The WorkerGroup is a generic utility class for managing a group of Ray Actors. - -* This is similar in concept to Fiber's `Ring `_. +This is similar in concept to Fiber's `Ring `_. diff --git a/doc/source/train/config_guide.rst b/doc/source/train/config_guide.rst index d152e958beb6..0b6c0381ddcb 100644 --- a/doc/source/train/config_guide.rst +++ b/doc/source/train/config_guide.rst @@ -1,13 +1,13 @@ .. _train-config: -Configurations User Guide -========================= +Ray Train Configuration User Guide +================================== The following overviews how to configure scale-out, run options, and fault-tolerance for Train. For more details on how to configure data ingest, also refer to :ref:`air-ingest`. -Scaling configuration (``ScalingConfig``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Scaling Configurations in Train (``ScalingConfig``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The scaling configuration specifies distributed training properties like the number of workers or the resources per worker. @@ -22,8 +22,8 @@ The properties of the scaling configuration are :ref:`tunable `. :start-after: __failure_config_start__ :end-before: __failure_config_end__ -Sync configuration (``SyncConfig``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Sync configurations in Train (``SyncConfig``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The sync configuration specifies how to synchronize checkpoints between the Ray cluster and remote storage. @@ -69,8 +69,8 @@ are :ref:`not tunable `. :end-before: __sync_config_end__ -Checkpoint configuration (``CheckpointConfig``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Checkpoint configurations in Train (``CheckpointConfig``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The checkpoint configuration specifies how often to checkpoint training state and how many checkpoints to keep. diff --git a/doc/source/train/dl_guide.rst b/doc/source/train/dl_guide.rst index 9f0e208a543f..3fa2e6d812a3 100644 --- a/doc/source/train/dl_guide.rst +++ b/doc/source/train/dl_guide.rst @@ -1,7 +1,7 @@ .. _train-dl-guide: -Deep Learning User Guide -======================== +Distributed Deep Learning with Ray Train User Guide +=================================================== This guide explains how to use Train to scale PyTorch, TensorFlow and Horovod. @@ -16,8 +16,8 @@ In this guide, we cover examples for the following use cases: .. _train-backends: -Backends --------- +Using Deep Learning Frameworks as Backends +------------------------------------------ Ray Train provides a thin API around different backend frameworks for distributed deep learning. At the moment, Ray Train allows you to perform @@ -38,15 +38,15 @@ training with: .. _train-porting-code: -Porting code to Ray Train -------------------------- +Porting code from PyTorch, TensorFlow, or Horovod to Ray Train +-------------------------------------------------------------- The following instructions assume you have a training function that can already be run on a single worker for one of the supported :ref:`backend ` frameworks. -Update training function -~~~~~~~~~~~~~~~~~~~~~~~~ +Updating your training function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ First, you'll want to update your training function to support distributed training. @@ -173,8 +173,8 @@ training. To onboard onto Horovod, please visit the `Horovod guide `_. -Create Ray Train Trainer -~~~~~~~~~~~~~~~~~~~~~~~~ +Creating a Ray Train Trainer +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``Trainer``\s are the primary Ray Train classes that are used to manage state and execute training. You can create a simple ``Trainer`` for the backend of choice @@ -273,8 +273,8 @@ To customize the backend setup, you can use the :ref:`framework-specific config For more configurability, please reference the :py:class:`~ray.train.data_parallel_trainer.DataParallelTrainer` API. -Run training function -~~~~~~~~~~~~~~~~~~~~~ +Running your training function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With a distributed training function and a Ray Train ``Trainer``, you are now ready to start training! @@ -407,8 +407,8 @@ of the :py:class:`~ray.air.result.Result` object returned by ``Trainer.fit()``. .. _train-datasets: -Distributed Data Ingest with Ray Datasets ------------------------------------------ +Distributed Data Ingest with Ray Datasets and Ray Train +------------------------------------------------------- :ref:`Ray Datasets ` are the recommended way to work with large datasets in Ray Train. Datasets provides automatic loading, sharding, and pipelined ingest (optional) of Data across multiple Train workers. To get started, pass in one or more datasets under the ``datasets`` keyword argument for Trainer (e.g., ``Trainer(datasets={...})``). @@ -444,8 +444,8 @@ For more details on how to configure data ingest for Train, please refer to :ref .. _train-monitoring: -Logging, Checkpointing and Callbacks ------------------------------------- +Logging, Checkpointing and Callbacks in Ray Train +------------------------------------------------- Ray Train has mechanisms to easily collect intermediate results from the training workers during the training run and also has a :ref:`Callback interface ` to perform actions on these intermediate results (such as logging, aggregations, etc.). diff --git a/doc/source/train/examples.rst b/doc/source/train/examples.rst index 12e482c12272..2bb938d402b7 100644 --- a/doc/source/train/examples.rst +++ b/doc/source/train/examples.rst @@ -7,47 +7,115 @@ Ray Train Examples .py files in ray/python/ray/train/examples. Below are examples for using Ray Train with a variety of models, frameworks, -and use cases. +and use cases. You can filter these examples by the following categories: -General Examples ----------------- -PyTorch -~~~~~~~ +.. raw:: html -* :doc:`/train/examples/pytorch/torch_fashion_mnist_example`: - End-to-end example for PyTorch. +
+
All
-* :doc:`/train/examples/transformers/transformers_example`: - End-to-end example for HuggingFace Transformers (PyTorch). + +
PyTorch
+
TensorFlow
+
HuggingFace
+
Horovod
+
MLflow
-TensorFlow -~~~~~~~~~~ + +
Training
+
Tuning
+
-* :doc:`/train/examples/tf/tensorflow_mnist_example`: - End-to-end example for TensorFlow -Horovod -~~~~~~~ +Distributed Training Examples using Ray Train +--------------------------------------------- -* :doc:`/train/examples/horovod/horovod_example`: - End-to-end example for Horovod (with PyTorch) +.. panels:: + :container: container pb-4 full-width + :column: col-md-4 px-2 py-2 + :img-top-cls: pt-5 w-75 d-block mx-auto + --- + :img-top: /images/pytorch_logo.png -Logger/Callback Examples ------------------------- -* :doc:`/train/examples/mlflow_fashion_mnist_example`: - Example for logging training to MLflow via the ``MLflowLoggerCallback`` + +++ + .. link-button:: torch_fashion_mnist_ex + :type: ref + :text: PyTorch Fashion MNIST Training Example + :classes: btn-link btn-block stretched-link trainTorchFashionMnist + --- + :img-top: /images/pytorch_logo.png -Ray Tune Integration Examples ------------------------------ + +++ + .. link-button:: train_transformers_example + :type: ref + :text: Transformers with PyTorch Training Example + :classes: btn-link btn-block stretched-link trainTransformers -* :doc:`/train/examples/tf/tune_tensorflow_mnist_example`: - End-to-end example for tuning a TensorFlow model. + --- + :img-top: /images/tf_logo.png -* :doc:`/train/examples/pytorch/tune_cifar_torch_pbt_example`: - End-to-end example for tuning a PyTorch model with PBT. + +++ + .. link-button:: tensorflow_mnist_example + :type: ref + :text: TensorFlow MNIST Training Example + :classes: btn-link btn-block stretched-link trainTensorflowMnist + + --- + :img-top: /images/horovod.png + + +++ + .. link-button:: horovod_example + :type: ref + :text: End-to-end Horovod Training Example + :classes: btn-link btn-block stretched-link trainHorovod + + +Ray Train Examples Using Loggers & Callbacks +-------------------------------------------- + +.. panels:: + :container: container pb-4 full-width + :column: col-md-4 px-2 py-2 + :img-top-cls: pt-5 w-75 d-block mx-auto + + --- + :img-top: /images/mlflow.png + + +++ + .. link-button:: train_mlflow_example + :type: ref + :text: Logging Training Runs with MLflow + :classes: btn-link btn-block stretched-link trainMlflow + + +Ray Train & Tune Integration Examples +------------------------------------- + +.. panels:: + :container: container pb-4 full-width + :column: col-md-4 px-2 py-2 + :img-top-cls: pt-5 w-75 d-block mx-auto + + --- + :img-top: /images/tune.png + + +++ + .. link-button:: tune_train_tf_example + :type: ref + :text: End-to-end Example for Tuning a TensorFlow Model + :classes: btn-link btn-block stretched-link trainTuneTensorflow + + --- + :img-top: /images/tune.png + + +++ + .. link-button:: tune_train_torch_example + :type: ref + :text: End-to-end Example for Tuning a PyTorch Model with PBT + :classes: btn-link btn-block stretched-link trainTunePyTorch .. TODO implement these examples! @@ -63,9 +131,19 @@ Ray Tune Integration Examples * Example training on Vision model. -Benchmarks ----------- +Ray Train Benchmarks +-------------------- + +.. panels:: + :container: container pb-4 full-width + :column: col-md-4 px-2 py-2 + :img-top-cls: pt-5 w-75 d-block mx-auto -* :doc:`/train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example`: - Benchmark example for the PyTorch data transfer auto pipeline. + --- + :img-top: /ray-overview/images/ray_svg_logo.svg + +++ + .. link-button:: train_benchmark + :type: ref + :text: Benchmark example for the PyTorch data transfer auto pipeline + :classes: btn-link btn-block stretched-link trainBenchmark diff --git a/doc/source/train/examples/horovod/horovod_example.rst b/doc/source/train/examples/horovod/horovod_example.rst index 42830d38ecc2..0593a275be09 100644 --- a/doc/source/train/examples/horovod/horovod_example.rst +++ b/doc/source/train/examples/horovod/horovod_example.rst @@ -1,6 +1,8 @@ :orphan: -horovod_example -=============== +.. _horovod_example: + +Horovod Distributed Training Example with PyTorch & Ray Train +============================================================= .. literalinclude:: /../../python/ray/train/examples/horovod/horovod_example.py diff --git a/doc/source/train/examples/mlflow_fashion_mnist_example.rst b/doc/source/train/examples/mlflow_fashion_mnist_example.rst index 5f04779badbd..fb3d6106d36c 100644 --- a/doc/source/train/examples/mlflow_fashion_mnist_example.rst +++ b/doc/source/train/examples/mlflow_fashion_mnist_example.rst @@ -1,6 +1,8 @@ :orphan: -mlflow_fashion_mnist_example -============================ +.. _train_mlflow_example: + +Logging Ray Train Training Runs with MLflow (using Callbacks) +============================================================= .. literalinclude:: /../../python/ray/train/examples/mlflow_fashion_mnist_example.py diff --git a/doc/source/train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example.rst b/doc/source/train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example.rst index 5237191880ba..4c923e9a94cb 100644 --- a/doc/source/train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example.rst +++ b/doc/source/train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example.rst @@ -1,7 +1,9 @@ :orphan: -Torch Data Prefetching Benchmark -================================ +.. _train_benchmark: + +Torch Data Prefetching Benchmark for Ray Train +============================================== We provide a benchmark example to show how the auto pipeline for host to device data transfer speeds up training on GPUs. This functionality can be easily enabled by setting ``auto_transfer=True`` in :func:`train.torch.prepare_data_loader`. diff --git a/doc/source/train/examples/pytorch/torch_fashion_mnist_example.rst b/doc/source/train/examples/pytorch/torch_fashion_mnist_example.rst index 5b5e0d47d480..2955441efaf0 100644 --- a/doc/source/train/examples/pytorch/torch_fashion_mnist_example.rst +++ b/doc/source/train/examples/pytorch/torch_fashion_mnist_example.rst @@ -1,6 +1,8 @@ :orphan: -torch_fashion_mnist_example -=========================== +.. _torch_fashion_mnist_ex: + +Running Distributed Training of a PyTorch Model on Fashion MNIST with Ray Train +=============================================================================== .. literalinclude:: /../../python/ray/train/examples/pytorch/torch_fashion_mnist_example.py diff --git a/doc/source/train/examples/pytorch/torch_regression_example.rst b/doc/source/train/examples/pytorch/torch_regression_example.rst index 133f091c1f0a..7ddad6fe21c7 100644 --- a/doc/source/train/examples/pytorch/torch_regression_example.rst +++ b/doc/source/train/examples/pytorch/torch_regression_example.rst @@ -1,3 +1,4 @@ + :orphan: torch_regression_example diff --git a/doc/source/train/examples/pytorch/tune_cifar_torch_pbt_example.rst b/doc/source/train/examples/pytorch/tune_cifar_torch_pbt_example.rst index 6d9c1eaf3abe..a7e9d54e2086 100644 --- a/doc/source/train/examples/pytorch/tune_cifar_torch_pbt_example.rst +++ b/doc/source/train/examples/pytorch/tune_cifar_torch_pbt_example.rst @@ -1,6 +1,8 @@ :orphan: -tune_cifar_torch_pbt_example -============================ +.. _tune_train_torch_example: + +Tuning Hyperparameters of a Distributed PyTorch Model with PBT using Ray Train & Tune +===================================================================================== .. literalinclude:: /../../python/ray/train/examples/pytorch/tune_cifar_torch_pbt_example.py diff --git a/doc/source/train/examples/tf/tensorflow_mnist_example.rst b/doc/source/train/examples/tf/tensorflow_mnist_example.rst index a63bca284a75..0a03a9462d76 100644 --- a/doc/source/train/examples/tf/tensorflow_mnist_example.rst +++ b/doc/source/train/examples/tf/tensorflow_mnist_example.rst @@ -1,6 +1,8 @@ :orphan: -tensorflow_mnist_example -======================== +.. _tensorflow_mnist_example: + +Running Distributed Training of a TensorFlow Model on MNIST with Ray Train +========================================================================== .. literalinclude:: /../../python/ray/train/examples/tf/tensorflow_mnist_example.py diff --git a/doc/source/train/examples/tf/tune_tensorflow_mnist_example.rst b/doc/source/train/examples/tf/tune_tensorflow_mnist_example.rst index efc7db7cdc22..724710861565 100644 --- a/doc/source/train/examples/tf/tune_tensorflow_mnist_example.rst +++ b/doc/source/train/examples/tf/tune_tensorflow_mnist_example.rst @@ -1,6 +1,8 @@ :orphan: -tune_tensorflow_mnist_example -============================= +.. _tune_train_tf_example: + +Tuning Hyperparameters of a Distributed TensorFlow Model using Ray Train & Tune +=============================================================================== .. literalinclude:: /../../python/ray/train/examples/tf/tune_tensorflow_mnist_example.py diff --git a/doc/source/train/examples/transformers/transformers_example.rst b/doc/source/train/examples/transformers/transformers_example.rst index 6325b1c4f088..7f7eeb4547fc 100644 --- a/doc/source/train/examples/transformers/transformers_example.rst +++ b/doc/source/train/examples/transformers/transformers_example.rst @@ -1,6 +1,8 @@ :orphan: -transformers_example -==================== +.. _train_transformers_example : + +Ray Train Example for HuggingFace Transformers with PyTorch +=========================================================== .. literalinclude:: /../../python/ray/train/examples/transformers/transformers_example.py diff --git a/doc/source/train/faq.rst b/doc/source/train/faq.rst index 94cfe9f6c799..76534e86f16d 100644 --- a/doc/source/train/faq.rst +++ b/doc/source/train/faq.rst @@ -14,8 +14,8 @@ distributed training communication is done with Torch's ``DistributedDataParalle Take a look at the :ref:`Pytorch ` and :ref:`Tensorflow ` benchmarks to check performance parity. -How do I set resources? ------------------------ +How do I set training resources in Ray Train? +--------------------------------------------- By default, each worker will reserve 1 CPU resource, and an additional 1 GPU resource if ``use_gpu=True``. @@ -28,8 +28,9 @@ you can initialize the ``Trainer`` with ``resources_per_worker`` specified in `` can still be run with Ray Train today without these functions. -My multi-node PyTorch GPU training is hanging or giving me obscure NCCL errors. What do I do? ----------------------------------------------------------------------------------------------- +My multi-node PyTorch GPU training is hanging or giving me obscure NCCL errors. What do I do? +--------------------------------------------------------------------------------------------- + If you are on a multi-node GPU training setup and training is hanging, or you get errors like `RuntimeError: NCCL error in: /pytorch/torch/lib/c10d/ProcessGroupNCCL.cpp:911, unhandled system error` it could be that there is some networking misconfiguration in your cluster. diff --git a/doc/source/train/gbdt.rst b/doc/source/train/gbdt.rst index 8855050624b3..a4f1aec6cb68 100644 --- a/doc/source/train/gbdt.rst +++ b/doc/source/train/gbdt.rst @@ -1,12 +1,12 @@ .. _train-gbdt-guide: -XGBoost / LightGBM User Guide -============================= +XGBoost & LightGBM User Guide for Ray Train +=========================================== Ray Train has built-in support for XGBoost and LightGBM. -Basic Usage ------------ +Basic Training with Tree-Based Models in Train +---------------------------------------------- Just as in the original `xgboost.train() `__ and `lightgbm.train() `__ functions, the diff --git a/doc/source/train/getting-started.rst b/doc/source/train/getting-started.rst index 5a5b483fa506..cb59e5f64c9d 100644 --- a/doc/source/train/getting-started.rst +++ b/doc/source/train/getting-started.rst @@ -1,7 +1,7 @@ .. _train-getting-started: -Getting Started -=============== +Getting Started with Distributed Model Training in Ray Train +============================================================ Ray Train offers multiple ``Trainers`` which implement scalable model training for different machine learning frameworks. Here are examples for some of the commonly used trainers: diff --git a/doc/source/train/key-concepts.rst b/doc/source/train/key-concepts.rst index bec0bed8460f..c477099d2104 100644 --- a/doc/source/train/key-concepts.rst +++ b/doc/source/train/key-concepts.rst @@ -1,7 +1,7 @@ .. _train-key-concepts: -Key Concepts -============ +Key Concepts of Ray Train +========================= There are four main concepts in the Ray Train library. @@ -22,6 +22,10 @@ The output of a Trainer run is a :ref:`Result ` that metrics from the training run and the latest saved :ref:`Checkpoint `. Trainers can also be configured with :ref:`Datasets ` and :ref:`Preprocessors ` for scalable data ingest and preprocessing. + +Deep Learning, Tree-Based, and other Trainers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + There are three categories of built-in Trainers: .. tabbed:: Deep Learning Trainers @@ -65,8 +69,8 @@ There are three categories of built-in Trainers: .. _train-key-concepts-config: -Configuration -------------- +Train Configuration +------------------- Trainers are configured with configuration objects. There are two main configuration classes, the :class:`ScalingConfig ` and the :class:`RunConfig `. @@ -77,8 +81,8 @@ Check out the :ref:`Configurations User Guide ` for an in-depth gu .. _train-key-concepts-results: -Checkpoints ------------ +Train Checkpoints +----------------- Calling ``Trainer.fit()`` returns a :class:`Result ` object, which includes information about the run such as the reported metrics and the saved checkpoints. @@ -91,8 +95,8 @@ Checkpoints have the following purposes: .. _train-key-concepts-predictors: -Predictors ----------- +Train Predictors +---------------- Predictors are the counterpart to Trainers. A Trainer trains a model on a dataset, and a predictor uses the resulting model and performs inference on it. diff --git a/doc/source/train/train.rst b/doc/source/train/train.rst index 5d9523b965c1..c79ab6ea278a 100644 --- a/doc/source/train/train.rst +++ b/doc/source/train/train.rst @@ -42,8 +42,8 @@ There are three broad categories of Trainers that Train offers: * Leverage the :ref:`Ray cluster launcher ` to launch autoscaling or spot instance clusters on any cloud. -Quick Start ------------ +Quick Start to Distributed Training with Ray Train +-------------------------------------------------- .. tabbed:: XGBoost @@ -79,10 +79,11 @@ Quick Start .. _train-framework-catalog: -Framework Catalog ------------------ +Training Framework Catalog +-------------------------- -Here is a catalog of the framework-specific Trainer, Checkpoint, and Predictor classes that ship out of the box with Train: +Here is a catalog of the framework-specific Trainer, Checkpoint, and Predictor +classes that ship out of the box with Train: .. list-table:: diff --git a/doc/source/train/user-guides.rst b/doc/source/train/user-guides.rst index cd636041ffe0..175c3de901c7 100644 --- a/doc/source/train/user-guides.rst +++ b/doc/source/train/user-guides.rst @@ -1,5 +1,5 @@ -User Guides -=========== +Ray Train User Guides +===================== .. panels:: :container: container pb-4 full-width From 8a56ee5da5421316f9092d474382930160f5405b Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Thu, 12 Jan 2023 16:38:18 +0100 Subject: [PATCH 2/8] typo Signed-off-by: Max Pumperla --- doc/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 08bec54c7794..95750148649f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -364,7 +364,6 @@ "trainTuneTensorflow": "TensorFlow,Training,Tuning", "trainTunePyTorch": "PyTorch,Training,Tuning", "trainBenchmark": "PyTorch,Training" - # TODO add and integrate tags for other libraries. # Tune has a proper example library # Serve, RLlib and AIR could use one. From cd1c06fa9613496aa3661cbd7a8a1cbf896988d2 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Fri, 13 Jan 2023 09:57:23 +0100 Subject: [PATCH 3/8] Update doc/source/_toc.yml Co-authored-by: Amog Kamsetty Signed-off-by: Max Pumperla --- doc/source/_toc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 72f8a424294a..7622dbf4ef90 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -85,7 +85,7 @@ parts: - file: train/dl_guide title: "Deep Learning Guide" - file: train/gbdt - title: "Boosted Tree Guide" + title: "XGBoost/LightGBM guide" - file: train/architecture title: "Ray Train Architecture" - file: train/examples From 9c743a02bf0e286820a89effea0d79e04cd061b0 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Fri, 13 Jan 2023 11:19:28 +0100 Subject: [PATCH 4/8] rm console log Signed-off-by: Max Pumperla --- doc/source/_static/js/tags.js | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/_static/js/tags.js b/doc/source/_static/js/tags.js index 943953ed8a4e..52297919fcf0 100644 --- a/doc/source/_static/js/tags.js +++ b/doc/source/_static/js/tags.js @@ -19,7 +19,6 @@ window.addEventListener('load', () => { // which is the "div" element that we can later toggle. // Sphinx Panels is too inflexible to allow us to attach data tags // directly to the container. - console.log(element); for (let i = 0; i < 4; i++) { if (element.parentNode) { element = element.parentElement; From ff2603877d8c1e274a62c40fb842e1276158765e Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Fri, 13 Jan 2023 11:21:58 +0100 Subject: [PATCH 5/8] benchmark and hf logo Signed-off-by: Max Pumperla --- doc/source/_toc.yml | 2 +- doc/source/train/examples.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 7622dbf4ef90..dd94b1375a60 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -106,7 +106,7 @@ parts: - file: train/examples/pytorch/tune_cifar_torch_pbt_example title: "Tune & PyTorch Example" - file: train/examples/pytorch/torch_data_prefetch_benchmark/benchmark_example - title: "Ray Train Benchmark" + title: "Torch Data Prefetching Benchmark" - file: train/faq - file: train/api diff --git a/doc/source/train/examples.rst b/doc/source/train/examples.rst index 2bb938d402b7..40200eb5821e 100644 --- a/doc/source/train/examples.rst +++ b/doc/source/train/examples.rst @@ -46,7 +46,7 @@ Distributed Training Examples using Ray Train :classes: btn-link btn-block stretched-link trainTorchFashionMnist --- - :img-top: /images/pytorch_logo.png + :img-top: /images/hugging.png +++ .. link-button:: train_transformers_example From c779d929a338a9988af41ad5e74a86f593de211b Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Fri, 13 Jan 2023 11:46:30 +0100 Subject: [PATCH 6/8] fix css, links, ignore generated apis Signed-off-by: Max Pumperla --- doc/.gitignore | 5 ++++- doc/source/_static/css/custom.css | 7 ++++++- doc/source/train/architecture.rst | 11 +++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/.gitignore b/doc/.gitignore index 41480f0bed64..11ecb2797767 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -4,4 +4,7 @@ source/_static/thumbs .ipynb_checkpoints/ # Ignore generated gallery txt files -eco-gallery.txt \ No newline at end of file +eco-gallery.txt + +# Ignore generated Ray Datasets API docs +source/data/api/doc/ \ No newline at end of file diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css index 5f62d82e16b7..cec765ac69a7 100644 --- a/doc/source/_static/css/custom.css +++ b/doc/source/_static/css/custom.css @@ -464,7 +464,7 @@ footer { color: var(--pst-color-text-secondary); } -/* override default colors used in the Sphnix theme */ +/* override default colors used in the Sphinx theme */ :root { --tabs-color-label-active: #0475DE; --tabs-color-label-hover: #0475DE; @@ -486,3 +486,8 @@ footer { :root { --blue: #0475DE; } + +/* Make the hover color of tag/gallery buttons differ from "active" */ +.btn-primary:active:focus .btn.tag:hover { + background-color: #B2CEF59E; +} \ No newline at end of file diff --git a/doc/source/train/architecture.rst b/doc/source/train/architecture.rst index ec78a225f3f1..733f86c6cfbe 100644 --- a/doc/source/train/architecture.rst +++ b/doc/source/train/architecture.rst @@ -1,5 +1,8 @@ .. _train-arch: +.. TODO: the diagram and some of the components (in the given context) are outdated. + Make sure to fix this. + Ray Train Architecture ====================== @@ -11,7 +14,7 @@ The actual training load is distributed among workers on a cluster that belong to a ``WorkerGroup``. Each framework has its specific communication protocols and exchange formats, which is why Ray Train provides ``Backend`` implementations (e.g. ``TorchBackend``) -that can be used to run the training process using a `BackendExecutor`. +that can be used to run the training process using a ``BackendExecutor``. Here's a visual overview of the architecture components of Ray Train: @@ -25,10 +28,10 @@ Trainer ------- Trainers are your main entry point to the Ray Train API. -Train provides a :ref:` ``BaseTrainer```, and +Train provides a :ref:`BaseTrainer`, and many framework-specific Trainers inherit from the derived ``DataParallelTrainer`` (like TensorFlow or Torch) and ``GBDTTrainer`` (like XGBoost or LightGBM). -Defining an actual Trainer, such as `TorchTrainer` works as follows: +Defining an actual Trainer, such as ``TorchTrainer`` works as follows: * You pass in a *function* to the Trainer which defines the training logic. * The Trainer will create an :ref:`Executor ` to run the distributed training. @@ -41,7 +44,7 @@ Backend Backends are used to initialize and manage framework-specific communication protocols. Each training library (Torch, Horovod, TensorFlow, etc.) has a separate backend -and takes specific configuration values defined in a :ref:` ``BackendConfig```. +and takes specific configuration values defined in a :ref:`BackendConfig`. Each backend comes with a ``BackendExecutor`` that is used to run the training process. .. _train-arch-executor: From a8e6105023bf920e60229566c133aef5b738ef03 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Fri, 13 Jan 2023 12:07:24 +0100 Subject: [PATCH 7/8] hover color Signed-off-by: Max Pumperla --- doc/source/_static/css/custom.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css index cec765ac69a7..d3dc3afd9c35 100644 --- a/doc/source/_static/css/custom.css +++ b/doc/source/_static/css/custom.css @@ -488,6 +488,6 @@ footer { } /* Make the hover color of tag/gallery buttons differ from "active" */ -.btn-primary:active:focus .btn.tag:hover { - background-color: #B2CEF59E; +.tag:hover { + background-color: rgba(20, 99, 208, 0.62) !important; } \ No newline at end of file From 289e39cd56486d157b0f21bfcbf73b6efe50be99 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Thu, 26 Jan 2023 10:21:19 +0100 Subject: [PATCH 8/8] Update doc/source/_static/css/custom.css Co-authored-by: Kai Fricke Signed-off-by: Max Pumperla --- doc/source/_static/css/custom.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css index d3dc3afd9c35..8f0bfe15d567 100644 --- a/doc/source/_static/css/custom.css +++ b/doc/source/_static/css/custom.css @@ -488,6 +488,6 @@ footer { } /* Make the hover color of tag/gallery buttons differ from "active" */ -.tag:hover { +.tag.btn-outline-primary:hover { background-color: rgba(20, 99, 208, 0.62) !important; } \ No newline at end of file