From 5e0b7a332e4fcbc9d882f3b9c4a04b0c0eafec5d Mon Sep 17 00:00:00 2001 From: Aravind Nagarajan <143961880+AravindXD@users.noreply.github.com> Date: Sun, 10 Nov 2024 03:33:35 +0530 Subject: [PATCH 1/5] feat: implement saving and checking of pretrained models and logs before training;fix: add error handling for improved stability;feat: add user prompt for training options and enable Metal Performance Shaders;build: add requirements.txt for easier installation;feat: enhance test.py with addons and speed optimizations --- .gitignore | 5 + PPO.py | 103 ++-- .../RocketLanding/PPO_RocketLanding_0_0.pth | Bin 0 -> 45074 bytes README.md | 20 +- plot_graph.py | 160 +++--- requirements.txt | 6 + test.py | 214 +++++--- train.py | 319 ++++++----- utils.py | 506 +++++++++++++++++- 9 files changed, 957 insertions(+), 376 deletions(-) create mode 100644 .gitignore create mode 100644 PPO_preTrained/RocketLanding/PPO_RocketLanding_0_0.pth create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a932b82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +__pycache__ +PPO_logs +PPO_figs +rocket \ No newline at end of file diff --git a/PPO.py b/PPO.py index 49dae36..2f85585 100644 --- a/PPO.py +++ b/PPO.py @@ -5,14 +5,18 @@ ################################## set device ################################## print("============================================================================================") -# set device to cpu or cuda -device = torch.device('cpu') -if(torch.cuda.is_available()): - device = torch.device('cuda:0') + +if torch.cuda.is_available(): + device = torch.device("cuda:0") torch.cuda.empty_cache() - print("Device set to : " + str(torch.cuda.get_device_name(device))) + print("Device set to:", torch.cuda.get_device_name(device)) +elif torch.backends.mps.is_available(): + device = torch.device("mps") + print("Device set to: MPS (Apple Silicon)") else: - print("Device set to : cpu") + device = torch.device("cpu") + print("Device set to: CPU") + print("============================================================================================") @@ -47,31 +51,31 @@ def __init__(self, state_dim, action_dim, has_continuous_action_space, action_st # actor if has_continuous_action_space : self.actor = nn.Sequential( - nn.Linear(state_dim, 64), - nn.Tanh(), - nn.Linear(64, 64), - nn.Tanh(), - nn.Linear(64, action_dim), - nn.Tanh() - ) + nn.Linear(state_dim, 64), + nn.Tanh(), + nn.Linear(64, 64), + nn.Tanh(), + nn.Linear(64, action_dim), + nn.Tanh() + ) else: self.actor = nn.Sequential( - nn.Linear(state_dim, 64), - nn.Tanh(), - nn.Linear(64, 64), - nn.Tanh(), - nn.Linear(64, action_dim), - nn.Softmax(dim=-1) - ) + nn.Linear(state_dim, 64), + nn.Tanh(), + nn.Linear(64, 64), + nn.Tanh(), + nn.Linear(64, action_dim), + nn.Softmax(dim=-1) + ) # critic self.critic = nn.Sequential( - nn.Linear(state_dim, 64), - nn.Tanh(), - nn.Linear(64, 64), - nn.Tanh(), - nn.Linear(64, 1) - ) - + nn.Linear(state_dim, 64), + nn.Tanh(), + nn.Linear(64, 64), + nn.Tanh(), + nn.Linear(64, 1) + ) + def set_action_std(self, new_action_std): if self.has_continuous_action_space: self.action_var = torch.full((self.action_dim,), new_action_std * new_action_std).to(device) @@ -137,9 +141,9 @@ def __init__(self, state_dim, action_dim, lr_actor, lr_critic, gamma, K_epochs, self.policy = ActorCritic(state_dim, action_dim, has_continuous_action_space, action_std_init).to(device) self.optimizer = torch.optim.Adam([ - {'params': self.policy.actor.parameters(), 'lr': lr_actor}, - {'params': self.policy.critic.parameters(), 'lr': lr_critic} - ]) + {'params': self.policy.actor.parameters(), 'lr': lr_actor}, + {'params': self.policy.critic.parameters(), 'lr': lr_critic} + ]) self.policy_old = ActorCritic(state_dim, action_dim, has_continuous_action_space, action_std_init).to(device) self.policy_old.load_state_dict(self.policy.state_dict()) @@ -173,29 +177,24 @@ def decay_action_std(self, action_std_decay_rate, min_action_std): print("--------------------------------------------------------------------------------------------") def select_action(self, state): - if self.has_continuous_action_space: with torch.no_grad(): state = torch.FloatTensor(state).to(device) action, action_logprob, state_val = self.policy_old.act(state) - - self.buffer.states.append(state) - self.buffer.actions.append(action) - self.buffer.logprobs.append(action_logprob) - self.buffer.state_values.append(state_val) - - return action.detach().cpu().numpy().flatten() + self.buffer.states.append(state) + self.buffer.actions.append(action) + self.buffer.logprobs.append(action_logprob) + self.buffer.state_values.append(state_val) + return action.detach().cpu().numpy().flatten() else: with torch.no_grad(): state = torch.FloatTensor(state).to(device) action, action_logprob, state_val = self.policy_old.act(state) - - self.buffer.states.append(state) - self.buffer.actions.append(action) - self.buffer.logprobs.append(action_logprob) - self.buffer.state_values.append(state_val) - - return action.item() + self.buffer.states.append(state) + self.buffer.actions.append(action) + self.buffer.logprobs.append(action_logprob) + self.buffer.state_values.append(state_val) + return action.item() def update(self): # Monte Carlo estimate of returns @@ -252,11 +251,13 @@ def update(self): def save(self, checkpoint_path): torch.save(self.policy_old.state_dict(), checkpoint_path) - + torch.save(self.policy.state_dict(), checkpoint_path) + def load(self, checkpoint_path): - self.policy_old.load_state_dict(torch.load(checkpoint_path, map_location=lambda storage, loc: storage)) - self.policy.load_state_dict(torch.load(checkpoint_path, map_location=lambda storage, loc: storage)) - - - + self.policy_old.load_state_dict( + torch.load(checkpoint_path, map_location=lambda storage, loc: storage) + ) + self.policy.load_state_dict( + torch.load(checkpoint_path, map_location=lambda storage, loc: storage) + ) diff --git a/PPO_preTrained/RocketLanding/PPO_RocketLanding_0_0.pth b/PPO_preTrained/RocketLanding/PPO_RocketLanding_0_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..e03e83048279671d2b5ec0353519d857f263d177 GIT binary patch literal 45074 zcmbrl2UJwSvMvk=k|akZBOodnQH1H*h>8hBF<~MK0>%Lhh^XWwNKlD_h@yfbW`y0f z5d#c)Z1+1nzc@jLPEPle0?{pUF+u? z8n|J7h?HyaD!*XARnr1}L#6maW8L_|#_qBLXm8)p4Z&s>W?TFMR|kahMgDyB4KVZE z6dJfTM9MSR&u3HM+Et#Re(OUv1bc3_lHw0DHg*>m{4+%GOYhZwe9^yuk#btQ!8>%p zU!fFV%w1UU8w>t$_hEwfbsIzY;x0lWp?rz?bA;wN%#n!WOU{v)BOJ$>>0 z!i_KEKH^_E`2>20@MQx8Hw+LB_zP&smkSV|;}FW1{|8PiUtx}LEMM_2hEgbB*^RIA zFVAK5AJ3)wZzxuLwEzJf`B1+4UpfwP{E-4y0y?AqLTQBZHQo4H|3b&g>>qTr{|(5R zuk)9V?my}1{h_1(7h`lN-@uJ;_%C!O{0AMQe?ytTA0wdSAkdES9~j9w0y<;=(i!&` zYJ4c)#EozIFLWmSrDOJQKsJ1H0UgOuzQsRqV)>ST=ve*5uny%8ypxK z==*=oKJnjRCh{l!QICL+?O*i>tTp+swe0?4*@yCZZhYb{^RHO{&=33@(j-3fS3eH_ z)X$VZgdG2ZObz8vbK_6{S3>`3FQj{M|4QheCiD1rBul=h07*EM@AVI)SibikMm~R0d_(!G-1vT* zBKZF9l7jnsuJa4^{xi$@tN$@;0*u`SQXzg|1V88>lXA0PaEM^O^ViG~4sjO~eD#m{ z&0ibAUl;HX*`JyF&pFRuAHm=7U!|7(jS+nQf0bJCgCqDM|5a+u4~^h&`mfRn{LK;k zE&o+&!`~Xg5Bn<}F5vKY1oF2<@WcNK|2Vq;7?%9)5&Ve%Cbs17h~V%1Z(=L{t_XhQ ze-m5tqaygx|4lrB9}~fk{fBr{h>zL4IpPv0do(MJ1iJcj&HEp{u8xq*|KN3f!b1Hv z{Bf^Nj*>#l{^PYaFf??n-yi%~@@nLtNB(pD&-j``7XJhOAA6fyI60d9U-199$^Y`+ zJW+XTi((w^pEKfSOO;( z=ys^Tk`IacLr`Pzn{3qH4JL;zxo^?UL}QZ~r@!n8m<+RH5vj_;4Q6 z@u&%cP2B0uFT~B2o#X0QNk;m$|DNKf|9d2n#sIIavV}9M8&Ga~k z(T}F-*wQ$g>say}+Lk5bdQEl44ZRuPRtx;P#fuL2x(JVKv*2v}d)l^yGVhY7Gn;;_ zU_Oqz2$@R`K)Le;rr3G`7cXwg5LSz+%R2zkE0q~{y*_eTy`T5+>nGUp{vLj{o6l@N z70ea-uELe?Be6F24mVe6A70$yz*Ja<(|OZwk&@r?oW<#LId z{i*iM(2^DS;Mq}R+eNtrN)6oPIMJG!@62h%)l-b9<#SZ{smmQ1rOVuPJ_o_`UqPEn zIy22%7e#LVBD#A5IiX?kAax~(?dR@5r`K%GXYVuahPoY@NQXP**1yO3GmRMSl7)0A zOoDOUe-m9-HFKW(oA9yF59Y?hVw5wU4v#Lq#H!5UT(JBtI!?}+H+uF%9JMHji{Cy5 zJ9b7ecsUM!jU1po8%>!vOU5udS+SU!7KD2*hHx4#!4`R@@sAc~D4vzznFrN6)9~ zHF9#I^v;D?U~Lp|7K_;6+|Hd5hnfq=YfA5HoE*236WKGSX23EKU3~8`!w*U_XR0s3hK?9!jc6Pm z(az?EGJkQu#UGH!8`l_}`p+=BU=J)7j18YXqRa&CT!xn`%NZnODq^73N{e7`Tr-RZT(BYh1#7nu`~-)PF2*jqAh9ABXGzyUIE?lESbuqKmI z_MDT8I?S72?22*rvq2c$VQj@R{Mt2*i<_^)y;iMYhUr@}emw)QI=BhbjMPEN@Hn$| zQ8Ck8xeL8EbG5{R z@eg!5!l`AV*DM*zpa zG7lF_tYX?#q`0f`3^#S^Z~So}llrWEi0sLHxVm=&taj4iPWHMpJGR!qg9b4ir;`iJ z#J!wFtb9#*)o;2eeTb={65PGCH24}l8y{WSK$BM&b2h<-H7^=Z;l}CZj6t0&*L3VT z&b*<*80&eXqh}s-g_Cwz$JW5K0I8az+r=2VZY+l6m@*;C5AkPtEw@CL2}|peAZit85S?_Y}s(K@XcrtuRISLl`CPalr;A(p$=BaW->#Q zUN927)OeZ}jocc31hYG7F{kun8qijjh;MyEeYZ8k%7T$tcx59eYp9H${ep4Vra1EK zWDT7w`-w4{DC%I|sm;{Zs&dJ?Ld>UDWzPCVDA%kR$|XHG!if)G#f?$0hrWxQcyNa* zH+v69S|>Cj3C;ran&dE6Jl>%oxdhWx<7%cR4}z2EW3uPh1Bjd*%_jIt)Qr?$Tl11p z#7lkEJN z;gOudH=Prm;qAu_4!f*!@ugKp&C8ba7P&w=fs$HJK z?ddmy-J=Op`(zp`+t&bU%?ixJqH)ZyxwBz^<4P{R&XKFnNT>5+uEF$aCvaip7;Zq| zSM`cjIL|UW?s>K^C$WDdv-Z(F#ydg_oJxOlBfpElk?10P(%bve-aU@tz9 zS&QPs*ANFQZ~QV`lD=2{M0HsS>`HS3pKp{^xGo8DU2)j^cpS-I_lVSW#uH<1EKEId z0lhsF!F=EnDa%cO_^2_csbWv^9&%*0+gx7Z*sD}xUnFm*^a$`$driwebM`04-ym!F z0dQpaNZ6`X36t(dLd@HvplP6h>YL8uAN_l*9hNBcXCExM{%8Gb3EBLw`WJ+LmjCYt z80zVUp_qMW+188)mej)CI1`K-cgg-~$5t*XJO_KnR1j;8WU!4+ge|6tuq}BK7as5y z6Gzz4&cH2r^j#p_%6mk&9IFGVT`4%PYaZBdI)>S@H^Bc_7;d_w#n{JZksHYjzU`fe z@{d2kQX5h3(Zwg^>2XQat=)}lU#-RNXZLWZrT`RY0d_X_(w*ZX(fxWMH$m;>c0vjaZal;y5nD)>KaYXdvyi;DL$B&-*tLBHR!XJgo-hi>=iDRD z&!j;9zA)_gIvG^+TS0xOmsU6X!J%1!+zZon|ImNwhaB_&|N0k9fd3ErS4y1?f5yO- zvU5`fi5x-B=6@RlnnG6p+ZeENay0vY90C?OyYQNpHg!*thMh|{!Okrd{r2gSv~$aF zM}ZPfTXq;KTFt2H_@i{R{d=NUo(0dtCGcaaBla1Lp!06$qma5Xd#P&%T`~0|J>J0B zn{_;g7a|eR=JADB)?iJ0ZAV~H5m3ih&Lq1|k$imHgiDk+fK9489WbM;)%u&*li&xQ zhrHo?cQ#x;f0brAZelksts~#|ZlODZqsYRW*WvyteTeLNOM7MnvX^bNV7$M3E0?+xmezpr}A*H*eb>l$4*dLMbF5{!$wdAN3p1I+M!Omezqna>ZM zaQh5>oM=mkZHo=2=INu|=K_0Ai-lzQa5sE3vlqvQ*x|X^d^j9XM*BNNxySMySaNj+ z)lm~=4y8;-HS=pVUBG{ftrQWeMD@E5OiQBO#v6hwiC$#AbCCKInfy zj3swsVZS|r%qU37Yvu*RNPHC5OAJSh#MSosvxn=&c?4Q2HWW6&^n0VU7JlAxBe z>{#mxc8c~b%#zn+g+%Al2KikyvelGmXgEODlP+rh%#rFv|HN_H#&{|AD_%4XW`&#N zxqiuw@YrU5kct#S}~>#-Ju!2KThmaQ9Sua`*5@`q6C!_wk}G z*ckDMNc=dAIXOT#Zn5MUO}0b*;08Q@HIyz1K=c~Y;&`u4qWR!dd~&e>tE;OqU`z;Y z)j13&;%>wE!v}EvjNzP7$bE8N#+q{$tkI=!3#fZ%8{Uih3Z*w!)6;1gG&XNP+b86J zudZd$>%aIIa_c@SXYlNw+P{W#Hdf z$5DkuBIpK5@(o4WcNv7$jO-)|vcb&Jg6Ub(4?{~dIFHa~>ULoz+tVFJI&V~wKrsne zJS!AG7c>$3f^Paew1G}MWyM6Ec|#ieM`EbA9wzx&qUF{Mnkw>u$Q{YFpXjy~=SyFt zh3zWvbz~x4G*|{Ya}*)lq5$ssyHkVRGudey3aC(0IZAyLhJ`O*lF4ZgapbAtkbi#+ zUQ2t#x;oFq+3F&2?nN**7v2EteWS?wU-BsbelKace}c8yxd@At+d=nLBCm8cWz7Oc z&|9a04wux&84GjqP@NBaX*oz=t&F!XJG>8-%kQy~gGjgb?6Kc#ydUJ&R$`%R6wS%A z!pW+YRMA%&E(&R3%gzT_xpX`4$j6CzW9~-sAcSZC?z9#z+15hOS&l@Rx_*4iia_&i z8E#Z8V!QYbDt}#qiM5aePNoSN`7}(JI+|1sTZJQ2JF$kIW&3kiQL6{qZLozoiVcIVfmPVpA~kbar{JYzZr&Zz=d7(;VB&X9)arTAsBGTd1bNBSoj!f$&O zoOXx6#W6RC{-PbY?I6$(e0Scb2yG^OsuS!>I0wZ%3-t1lV!GaFf%5rWHmT(y>CxIw z)cShK3e&Ari+f5^j6%_@%K-YGa;&794DF~ghA-|hV7#N3s_EQ7h0S~6>}5WwJ)Q}U zU*oaxS{i;zx5p;SC}Pp~gjS`CqPi`nIkr6hqH#HL*_5Zif`l{ZvD?$dFw>{S=UTCLPY-xyop z%hDxBjWNt)5wxzb!Ld^&0u#TMt}blEVc%t`?a^2~VjTcitX&}L+Ek`uYYtHtQlS%v zWm50pJ=kzem4>^2u8!Cv=7WbdhmQGWwHy!~k_ksyK?Xx@Ib z+_{*TJ`Es=3T;rlqsjim8W#L7^$^K#`VcoM2Idzi5JqPbe$^C(@xDqV{=*pdpi&j} zC4q3Pa0*V|B7}n$+F)to1+x>HX>Nuo@oa6QgV(N;6Xr^IvoMQVre(n0yYo)_0VwhH5&2w3Km<@XJjs37{s9J}ajPDpTIxxQ)*kYyz#Pq7tLdc3d9Y>Kd3b+^Prp*`P;gs?QbV)0wvxuMlGEch^Q#Mz)~x4cE9cg701dAo@OstJ)8 zu}xtAsfFBIwga0^J;ZNk2jSqgRI<4|9ReT5lf-WvnXz>ugbh=GACIlcIIC;)LU}t( zGYg`3~xsh91BSmwa|6zC_Z0u8SbbV5|wLvh{C{OnrmMLQ;c@7cb4R1#ztxQYQ$jf z$tH|8Vact#;ZU8Ehq7Z1;I3L7)L-zGxBSgT9-bZrC$kUGxB*d|Bl(`bdp#VLR~KW# zg9J#QDTen>%Hc;5H=fGwqgeWB60c!qK53$c7}T;E7~#zj{j80BXdq0b-b4UTCmFks zzo4Q^)?%5ZFxeXE14-H;Br)b441C;&F`GV9hxP(kr;`HqiGeWXUKVcYN~W)Oo}G^-2~Ztus~N&@F9Xof@cuz`sm0Is$gAF?LUnr&-ZI$=7?kiR#2a`R%OxyRnX!?8?@3*NvqnddY&BD^) z?!hkjaZV2sPHS;y>aN`JvCXLaLlvACoQLg``Pk%A2<_ev>Gc~EIO$C-yxL`~z0IT( zP@rVMDFr8B{?Ita{enDR*Y5z_c%WLV@F|SFnvbvBQ{am&hq|Bqfh`$;3lRtK*I+p` z)hTkr&W`2QFRp{>t!tT4pVvSMX@x`eg}7IFfLFW!25ua*#V+4iy8J^o#)*&PbiFrX zs^Tb6o&TAf*n1w#qaty^`#027Mh%OU)Zv9>7de$*Od~38qEYaDj54ucY*pLgNwW$> z9n&Xk7?$>ibQAx+%S7&p8*0jIM#)94aImJ7SD`)v7g$Si7cxtrr!*B*qLShBxT|c9 zOgxdB62?1v@i_?=@}|k2&!MN`1rgb-f}gy3yvh4H3f_v$<}YDbnr8r;#EtO+|E+ys zn=rn+lulEx4w1DPog{yu3i4csKUU@NT*)algHjl*p z(@wEIhMzFoXCL)BcL^PpCqe)G8oRjl4WO|i1xrqO)2k^Xxnnyz>6Djh ztkw_U~OK^p|2eiH^#Uxt=oVlfsotD`RYns#XOwt*i zp|B!svlzxX**zuaC#`13bZ6qwM>`r*5x`a0c4MqBLtZ@*gXWd7nCq#=rV&5l6uTb7 z_nrXu!!@>dYc_mG$=0y;*9j6 zS?{>Lzr-E5wdxW1khBX{E5~8ju541~Va5x36^sQ*25>G?6CByj9;5X->8* z9SB=VFPX}*QwIBJhRhi9s^6A+sN2JfiRPdZYDzlK&8}KmB+6T{s+%``xjsppWB~4l z$u!US7a7R2=hUYO)&XIC$j_S$Vqcx`W5pAg-7d({z0Jd@#}|pIO&u>yeg#I~IZL*! za-c)K53pS(5Z(mj(AyP@k+I!RPZw@P^Jl|}&&#L8yVM)5uP`S)Q&K?s=P-P4D-T-Q ztLbE?TRiE31Ts9j0@ABa(4uV>M3L{ot?Rf$Jh=xjNXtNXk3Hit$%+Yc@`9LDDLhoz z$J?Erk2fXWkhU%rEc)&XYkv#It!oCEx3L>PB)6cX{{Jj;&R#U!j3!kW2@WgxhzE{ zM`af|WbuH?wk6<`Uo{XOk&V2cLfn_du8gYTOtvfa2Y%hP8(s{^Vf!yV#`x3+)(>K- z(7xq(?$dm%@K#_}R}MnPmP=?VRZh%YR)LAdTyz~*hvd>VK-I}yuS7B`Rqq59`itt$ zu0qb-6@Inn+uvQf092T_L?pnLh-(j!b`f)`7&aZ|?&@HletSf_le(yO;XOLTN*Q`r z>eIttGvL(fES}nZ9r|*@Nj7=*ecq;^+eH122kSDR40{8Mkw2r6eq+89fA9}Gwc^PhV~TAB(>#wtas;4>Qf*~rGJlu<#Ww(cIq6Y z$Hv0kA&yiO%3!a|WN5g1o=EggC!^!9kefT^U}5MblF-tGl|An8c*a?3F8`bUbUc9d z2Sg$MS|x4i(S+T0c|@(PK;VDNO!aPItl`pEh^#BI7uf8M)DE9N)TWy$6Ay?P7T z()XDjJljrgG-y$`TdydW7EU#nS7KFOD;;Z6!Y-;lif>wi$bq}DVEafCH&^a}xwXK2 zpW8~~9v`HsYcuH5$=6XTbdbgrEy2TWyWzm59&kG{lC9sm07gAa#&3pR_*FIr{KEh{ z3|PpW+{25T;!Gz!>P8uVaZV&9foJt40Q@o~V1%bCiCMEAY$H|iwa6I^ezOT$OeL}0 zaxv^Yu$bFtVve3tUhqhF5~Q4*3wHZL3o7C?wo`Z8}&pmi{vD#WulyT*a{*Tbt&>5B?0$ZH+-g?+jq1s9z7#v48OnxxpK;*XErG`Z#$@1}<;d`WE+ ztSvH_eUQLIe?`3VZGbJCBgqsRrxC098%X=vXdM4)BB>j2BUgF>)noG4e?$cI^X-?1hQi{Rz-n{1M7HW{2c9K(x?v2V>S zTAMNhT|RWd&^Z^d-k=AnhL5n#MjF&V6u_nvo+PW@4xFRn;Rnl-?iH7q~O@OCf4Z#%bpaUfSdFch~cMwcxD8p z1GmqSdro#VDYpcsvSpC9ZZfmZOr6sfn`mFr#^=VL7{w_Lw!s^VTBs5D;TevUSDPco z7Rao?->1IAK{~_e=p+T7_+>^3`03Q?ds1t=1 z-Ivjx9mU;#v6%!am_qb9eW)FEp022pC1mF>1}gxd`A+v6xM*FLrks?=q6Q~?BVi6cX%W<0 zJ)UlTHwKiGda?AtMcS1*foVJQike6%V#*qQkkO3g9U2~uC!Z)V^=D7Rqd9H3 zvMGXRl}818hH;>9pd5BCROh`oIvdX(I7d}~CPVeCb0lHiXJkW^x#M5_aEa#uvh#B| zn%Ms$7tPYE5{mV3;r(?OaOEh^-L8&+uNSXn(L;OZyj7gVDK~02TZ!kT&;UabHh9`M ziq5^h9FDi0p+1MFV&lXz*mBkk@9(_Bj+2VVQ3mN;-;qS_XYmx~rso*c7%RzG{f=Y$ zC5veGH*L;uf)MjnZ4(9^A~@OJ7~JoVA-B6`(doPlobptJx2&ZQi=%H6$1et8X*mt+ z?MvBI0oR9UPJ|1kIF}uH@a?T0d=T^!1$b1_AH}k`E^IvK7LtuBgZru4lU?YTRY~1+ z6Clq`1S=-!bGKT}V3}MRe0aGHe%5`*O+O5n_!ng$v`q$Wyg%{UBO>9$a%)V=&)|tS zH(`roKF`f#Y;|ABbe>kwM>2YG49=C^!{ua8WTKai;(~{d0m&^pV1fT4W^>4UR!TGv z{0Cdf%Cu}wt6d95DrB(rf|xJ3P?Y>i`-l^V=5P;}UV;y{=FIs~2Qc_ZG{_Fw!ZXDU zXyN<}l)f_VFwZhy4Wtx$XZ^%PkOVpQNke~b9VQWpL?c| z*$>5`|63_l*?p2%o0o!gwIcQTT11b|yGTWpx6r_WCd$Q@@KVbQ>E6-N5MTU+9#a|) z)6&9urB?Eo=<$|n^c`hK_qnnQ%(dvw)?Cdtb;5Tyl>1p~@)t0lZ{6+VS(}igFF(@f%2ws-Sn3e4a zl|^#g2VrUE#MC@cNIgn+McsjjC(7hhgcrT*uSJU^f8zR}EZ(cRA7Mf6NM_GOK9`;E z29K|+;Zc2UF5tp$NHqC{qucy((wXPH>@E4Q^5A}aW|@I2nD?mN!{MBMUtWuXB6nn` zH_b$8>d+;`92>D1jt=iT`@80!c$$U#VVeJYE{%^p* zu^nzd7sO3w-N;E7;w+m4d^zC@HCg|gKHtBN)MT9}C9x&MuPm4_i*kC%`zwSCmr zC5$K2d8OKBTNKgV=}LEh7(>~hOxo1DgO@(V-7ceD1l0IeaISk1nynj!5)(DZcI%6H z#^(|(*r;xQ11dNZ*B2p0?7LPO}94SYx>;4H@dY!-*`kh{R7* zig+d$T-YXkF|sm80b>tt#Jzn+7^!`VJb%{C`o% z#fE6&_7M=IW&}%B6R{n;h=sg1%H_5Z+b^e*M-sWGWkUHMhQMvY>guuI!DrrsS`kRSQU=334!hN_7R<|c<8Oq zq$dU=p{DgHS+gRaT(jE`m*i8)?PIaL(Nc-L5Jh|Jp5J0`b7vfEp5_7LZyZ7=y9ZEn z@-CX4SLBp-k02Lz{@?|TX=Cr~K1@x&se#76Bcx+R2xx3LLr#x*M18|2flv8jw7Z;4 ze`u;>%E|{cba^Z>K4?S2^sF&#ek2q;&4kBxzU;VL2G~7(H}U*@7n?@=RBv6Hg(fqV zaQEwLMDzE3s-LS5b<2=`Ui^g2GwL8Z9s|TkH3FU=$-(TlB0Aqpw+o8dv*_-8g1u|K3!)y);moXWBRrf$e9qs7``6;xcfvbh z-v}GvUIn1r;R9|U<_+hq94x^Wer58(bs8IZdWW5569W8BH)6DRRPd>5fUo=Ey5VMC zc=KcSemW&r%?ju|nn<0T;-M!=m@!gS;sS$*(XjU!M2X#nyC+C8=7(cvqBDTzXU|%*7Zr){fLTWovbIpZF zp>*~-UIO-UI5jvS%9SlDrb{MVpuaptp}FQX##~FI#n)$1n*?))m48NKj{KxWePuB9 z`di+{1!r)_wFcZjU$si|U=R^!HPmWHpP} zO&m!rkI7)gl@17fqsV3N)u1`v`Q(<@2U@7C0m}OkO|I;2B7Cz^}VQu)MGtPF$*iS=o0cn zp-isSnzgvO%KbS>Ve+Nk-U=2NF{`np)Decbn5Ek zkh`#;MpHv)NU9L!8U>V_b&4L3>!cl@CXy|&FX=(S*vd?bgVV!gh!K>Yv za$K5z7*8e_h&7jN=t0v{DD|f)m36|*>i>H-uA?wRYEZ1@eE=SI+AhKSH+Kd?oLvTnf)FZM4M5(p)7Ab; z$MBYcElJt63-V_xp+o&aa1_6VPb<11$6JG&dMyJr{f1EfmN^#6O(M1JA-px(1lpG5 z!mQmHXb@+Em9alzm3k?>F?|kKn%AKEEH7{rnunYDBXRvRU0`L#z;Qiwj5>P&&ur6! zxr0~H8J_@WqeN!b*Rf;zOW1Kwc>cu(cu3UlRx{BX@ROK!RIM^fL^ z!IM37lIL!{AJ~F4nkQ2OT64s(AhrnQw64J;t7jzO$bMiZ^wHW^QaEX_gEVT!kvTc{ zi1edFJSmB2@YcFUx5{zUw&euvPuxs31htssX^t53&Jlu?chj!#!i3BkT-qG2vM2cpmwIM|gQlC5X>h8Rn3i8@)8O2=wRQqW;f*u*qgML^)oX zG#4hr`{F8m6=etwqlK~Rz8JH-?k+yB$skg%&GGrtE@I+ogGzJlaiiIFw(z16^3|2t z+cFt~+;SwXN-zP->%mZ?B+JwmiE`GvPq2Hgbb-u=lTgvW6Ec#lvF+*zx_+l5EKRkc zM{X(+vD%SLPNf@>o>@)P-K0^FcBB{mcah(RdP&^_VcgqXhU#%*P-XCf9)1yl8%KWz zM;BElK-7TQd#nVf@q{?xx8*pnZz|Ul#n313gvio`rRd=?0X{j+L?@SGd~@y|9r>(? zge|YkyK1J`Q^{dC(Rk_&NfA61z#};pcnnSLz)9Dns^WGbq>gV8?hd)8zeIE+x zT}KBcG3X7GBu!<6_Kc4B> zKJPfLXtjlH?a01R9oCb`@(lUSbYcyY$%8$_vS%&En*h-!obk zp^Y;OCV@`YYB+DH4X6Fz^PVSJV(k8t`1Yy`*TR$MD))&o@4^#cdiYtG;Byc%S)}Td zT5#odBl`9-B0cZPYOKcZz4%kXp=Pkz{KbF9SO;^a8wdJ^K%Y8a2 zKNDN4KUNPue}L)*B@nB47))N&LPO3e9G<)v7ys*^>p6=2_?(Oro)2eu z19D`F#t6=+`YFvdJ_-SgCqa9+B^r2kz@TtH873pnZRH<8r{xYz-iv7Z>YzNFkhjFK zdX{u?#{x1k=L>z%T*(`8IttqtR^gHreIUA^57T&!^zil;_HJnnd^!`1+0kk6<8n0U zTwTlg_Hew+iFri!!DhUbVorK{_rV?2UodmK5GL2%#^V;wkP|V2b?qP+p6dxlXT70+ z#t_Z0lEn}STl%3Y7(a*1BGY%%zHBh=dV}fbWFFxFY6e{AA1z8?x{n0 z6;tuBO<MN6>HeJ4_XX^sexHrfYJ?A919A&N-+t z0YOeo1W)TX5;@yVP_sgksVI)e{Mo|Pj(SwLu6N|6_49b|OZBO_r5mnL|3t|%1@7C- zC8C-J+lBMSKM26S9CJKyCzeyr zX~f&&uh|X8qXcIY6}j2@r*YE^S*Gh&BshwcV)3fS_~EiSec}?rz0B^w_4hB}A%!fw z8P12_<*&#!RZC2tJV>|o%%K+6Yta1Q4Xi)D7%#6>MYDu(ND&v*#76AIG2Mr7EgA)law*gKr_vbx~j=_izA;8q1Bws3SfJNCJe6vEGX&0OyxOrTgX|!u1 z7oRDj*;2vjs}a+o=HVzD(bbM=@vETWss}zg;EA@|TtV1SjU8~(MYbtR&`VPZ!viLB zgFk7M@h1~T`K&qaTs19f*!U3-cn%QhwQcxdL;_?VP~k@Z6b8p* zqHv+E3eryT(Y`|mS2T8!x0a4{yh<0W^O(vM4gZM}lgD$;-61gUb|f5sCdY*bZ9~4F z7QixL?v?U8%Wc78EV-BV3h?p}#6N+&{RStd+>r;YXIYPeG*4NiT@$K|60 zdnWk?JSlnr-?UC9y_XBg#T_ZI%5e^SjtR!qM&f{LRJb3@t+>I~p zpf-9&@(R8;IxRS3*+$12PQeSWUK4deu}rG@9L-&K2t_r!@yZmIK3P%$O;@CG7-!F2 zw%v#eNC!EoSP9Y1U+Mjy7m$vhM&9j~=J3HGy1=pmP0gGkb(#iKy+nuX{3(QMcAQ45 zFN$>277-36UP8^6g;XKb4PO`PfX0a}FtaZOH)Tg+N0%dgDt&}!EFpml<0ZgPY>wa_ zC4ydu=Qt;LE<9A;4!!;+@N(IHY`ByH{x9ZNyVb;y2hRSGkRF9)>%_rHHyFfHM$w`< z`dE@?Y=7sIA};({$lIHc1|z~AQIGv6@QkXUkFg;D#kCnc!kl0gcCV)QhXg%4>mxYR zAIcol`2mFX-^Uvp<;YdJtFT%%h8wyf*yo8{hxV>X#8B80$E}lq^6$$ba=8vj@;YEZ zT8h`NzXwLFt)Q0fM`=|}3thEVkhAQ*4H;!wkf)f(i{5q(#w-p+8wpL6^6Q7#U5e;5 zFMu9397gBQ9)xkNURXNO4GxKZ#HoR2pkC)4fd~22C0(C!S5$`ZUE16lIdAMdxQhs4 z*{ZFv`>=e`Eow6PAq04&;16dW7JXZW?yb%+M^&D-E0{9$P$Oh6z04D8EQXL%v+>NJ z1t_Lnq;mxOD9!W&s#AUwlzN>pCQpM_niRn)L7yVKYZ`aXPY;5}O~ruq5?uX<2-5bd z4w)_L+~sK%?9)amra7vao(R5*n*}|d!IgFN3p}AgrsoAYuHV#WbtSeZcrd{wM)0n7 z7QAvCV%;NDu&~$=bxK17b7McfA4owr5kTQ~PMjI(2TH=3BVnx(jTNPGG#1 z%5bZA9o2d?A9@^a;kf*@SS&k>(SKb6^FuG;=ufWDcC3spSyvA;K0d>z`S0MX!35Ip zB96J5)7iJv1@*7vsVF=&5A!4~VDYH87|hmEDLG|0bmcJwUJ7A;9=7FlcPHT3s3@2y zsEwNybGS;;4$E~*K}639txXrxS+A@iY{w#+J0%OoFLQv-wryCOOHp}M0T|b=fzpoM z;42+Xv>!=e@(MAqn0+3HST?8>N`$}HP?V&x1d=hfQ1sycK&`E6t zNW3qiEdg?%+BXST?~sG10mkSuMH@W@=TekLeuBs9mx+mXCS4`S>nu+=L?UCSymZ|?A(%w6q%kcgGx5-FGp=gK(O{0w0bzW9OiWF(l4ykAlQe;$ivSk%26+(*F zbzU!;R5X;97AozbH1)mTpFe-c@%wy_#Ydo96x?M!jOtmUvQt%;fSoQBKqb4b{(bmH{S1NWR*M5< z-PaZQT)i4JDO4g+xhg=SKe4_!>%^Hvj-=$N@{>;;@cv~HTWe5EZ0l=rcwZ}gawP&o zGuoNB>v(N%+!^%!*Tz0I4kN3y%&#=x{H%v0Y7(Bi?^=h7moiYgS_^+Cdg9R7MPO1rmhDXm z5G6PZ-l+6NFyq@zoRP%E>)&PI?Bqu*XWkof&Tu?>wWx49Fp%YJLs9s&deS~;2*#9m zVe-rrmSeFEcT7&gKZ<9aQYb3u2n)kFJ%i1G%?#lN93%c$iuDtVUk5Nl`?ehO&vbRwA~m(H9{3>Uq9;fBk{tY`B)^4W!kK+?B19bb;N zV;)zup|4g5`qcZvO&MWUqI47AX+`2f{pncL?S)%HM`5HW310mLnE2ojTpFlOtCAPO zoQHdB7rX*6KK%>a9-0a>?>LO8lfu{0#=r`tsM@KeEd9JDtdDCW4#Ip`8_#L)OQ$4y9$fo&8A{hqn2pa z*9LuZ>R5Z~D&+l>M0)uHp!Zw>)|d3)lnu>n-Es%oHT@);dS2iIG)>@|?FYerFS4@w z!?5|#V9fcN39qEb!FQog8?q=Lsr_yny`IVJw&6T<7`YK_$4K#Ms#(Iftct44jD<4Y zOW5l)nA?7j!%+)V$foO=kTGf=@mD><{wDP!i|wD`BvVWAf5vNdcN~~|=TOWzVF!T) z2T1pZz1aU%1Kb<@fE_cs$3U22ED4Gs?VsJTQ#}LkWnY1x){Dqe{T1Y$Q(R3+fD|St z9u*a*?t!AJ6k?~~2^~+*)LNM@B}dBN*_svIuvKckkC)UJ6PXWQ#N^eq8iRy`Y!RJd zD_)z3~or7rW`)urh4d8V?7JvQiLpv6KU~l@*f(>ux zq4Qxi@H(4ArWcP!yS0tvuY{1n&Y}2umkg<$E=M;QzhaO79mhitZjk!*MdZcLD)MUk zb8<1}3<)paC*E8eL3~x7k$o;2>`YpQsJCAQb3Sm1*+o{fKd)6#jb*ZhRv*~k%_4GZ zr6tsw~xTJ&EsXV6r)Kc-8Oh#V<`^n{OS8O%ez3(65U!c%VehZQGTQ7yl}Ql@Yy_^9!Ksr=yC5EE3hp1 z7dd@q0M?#o7?U7qoH{E}W9bY~BZas{`7Ftuah_fGE+K3FtguxXX^n-KhO_9ev+-Ke z08*el3_PaR!&}c9+-EwF^>2MqtDYDIkA6jCbyq&q`8NC)OHSY`-VLi4{VRt=X5U@4ZRfEw)3k)B^I|w3O_6n*_xz z@z^Nv1-@mcf|KtK;(0g~$4hI9>^GW#*F_x~bUF(9s(6rD6Q)6(@o2d7Cr@PNs)gF= z+T390Ay9l;3s({v;HuGcD5XP)W#b6(=RPmtaojNeUBF+I&3gn<+wa2wa^v_F!!l^=xdYdu zOTgaI7SonTvkyLF@!iSuB9(1ovGIx1Y;NEZ)|1wcsR}*U=XcI{wKWD(<*wWZD7Lb{%OG znXSNP4O#?RH?I*l14@(|4}yw+I-E>Y!4GQ7pOeQ#+|N~#8qQ5SVz{tg40=8ePlO#Vk`?~TaOWL zwI9Y!{05tLEWm!&^$=?`pRO*N2j$lW;?HUG@tXf%Xu@L<_}LGv;4IXSzX>wQ`NV7+ z!Fkd(I8h>kXtz9K&=m<(q(Fuf6}tIj30eQ+r|pk(GH`u*9DcJIjq8Ks;Cah0Q1h;4 zPd@uFpL^e#8e|bJSHw;}(}zCN5;!0pjvia<#jYi_cGuc@e;AHJ*@eqNl5&J-k#r#_cDksudohRJjkwp%RBMa8Q zV!I<9U`tS{?HTX&Y=e=QrJ4am-U~w`_pNNow|~sbZ7T8e*TCuvvq33q7+l9|Sa>D2 zCiU`Tw$}YG^L-eJ4_g%kKa8hX_tOM2G3fm>m)RoBt?*?&+&wGbI^JI@i{!$sQLatIBXcuM@B`Y_aGs=HVdMY?#XJ ziHTk%RuzPSLFQw!#!;S+E$PRbbf@4V>sT;3QUlXmi$&HZnV1~9P{=fkAZ-0p4DghY zR5g8YNE?lQi&e3@{tVln63-s43TMl9bF$#l7}0MbH+bj5S@?AL5UJiCiATr85&!Qo z;&0E!z(~gTPsA5!q=epZB6EJ`ozkuyg_+d7#!TXPrQEUd$hK*#ksE*VZ@@vn7lg;*#~EM(=CtXJ-N`|sR{BH z7m2rAuObiSa>4h66uc-1fm@U7AV}*T+tz&vCb_R7(}y$(vngY!7as>48P9e&37+SR z6?nPm8q6oV!AiLQ_^b$=`8xsM#TST1&#hop+XY6GsSM;?_Gb51*`hX`%(gHE%o!NY zuG-}g|5xioe2x>mwAl~hIjInmG#k%u(tynT7&KATBSkwV3x4KPtZ)2#BI!F6n-1M& zYwLu(+glM{Uz|rOa~I*xxiW&*vO$zMq!i;8eT0e)+p%6L5trY#$I9S7Jk_}fxK_p171;9qfKKzC-2R7X5hk?5X2(@4}YHO5})C3jQ zdi^CfUOb5DpU;EcmOZ%9G)=T2;{-a4pHM5gsR0S6m$5m;v1I%q8*EA6jkmZB%eZy~ zjqUZ=50@sEUhx=`c2+U(DK$i-U&v(U@4<;9bJ56Y9jN=9fjr?j1D`1Ix?B0IbYC4B z(<>!r&gx+C6JJ>WIC2|9Fh(}!$p3{tn-S^&z0xn*KF-3h)S`vki zk8;Qi83f8Z8gNgU;-?zXVki3nAbD~P^88N$G5*6mZVrVI-xBt& z4$#}D2h7ieV1suaA#(LZYKs+Uhh8UG_}*sXu_5SJ@rBr|8i%ppr$Uhb5zv3|h$vU| z6G!<=v4wWIaA}97P)i15s?A7Tw%&^^+B{u6@Mrd+3D3)j1b6TLA z+(yF(rNpFvFDX(p#|b_=;O)u>%)?>@tbe@>hHF>Aj>leT?N*5X?ve2A+Ho+L(8*@D zw$*05O2ohgebKr26EnK(M27#H0oUgDBljnMWr~?*=+ixeHKe7HNkIef`rY|(Ao~n_ zoYD`N({a&oZF8z^)*$vCb&bVD7h-!!9Atit110(OFwi~?C#u%s`s31=InP}fnAO0I zNx#UfC)-8GR9eYfVcueG>`82X567da`^cq`8zf+GKijL~JkjW_Q^=HFd(m)OP(V^L3SJqC|iN+zfWVO-q>e51P; z`YU?Cst1#C$$gH$g}mM&ak;RM5S45jKxkpx+PVkPBH|WY3tqWZIi|8~1?x+9)8d>oF|o(csz^4K%`EOGJ9 zU>RN~YIpv9N#vCLAlGBJXw&8sf-lJyAA0y>{W}So>}(^`1<$yXg&{f|eg`*11-7+g zvPoWOCH74{V7qc@Fh0ykhAL-e{5U8Azpr~rOkE5?<=`azwaF95>~Vpbmm8T@qws&b zS4WhaeDT=S9-=;TJPCf?gf1FCAm^zx8T(5X6Q^dtRmEKpT7zno}Z zNr4G1Z*kqyQ7nGTPPSHZg-AKd`_QMwSzp4>eUsikjY8q3d zPhk3HV|HMcICwD;S_jdMljt=A(+OV$< z#f-OUQa{gActS#_+&4^ec>I zX6-A{sW}=GWlV8y|KE6T(lc?xhr^&>-Jb{9>M-q_?`(_uhSh#pyA~g`eTD_8?J#BP zGm`bl0^SBXp?6Cz%3jIF&mmLc`;SRX{iY)G8Pd!ODi#aRrV{FvSCe@2qvXmX6#tEo zCv__>Vr<1Q3=Db1e%|)PBBOnn`au=KYz~9_r^l>#U?kj^So}}@W!T?xiqwC&$-?zt z*T0tkUH$d8Tm`e&q>(pzdKfgSjGg_d0Z(Q6!slWYW{D0ut`%L`u>0TV4n^5kVPtkA;ow$jhhle=geG+O?%@c9Z$uOHTIz0#hxfu zu;+$2WX4JO*W6d4bX-Of<)31wrB;vY&u)^u|F@4=PuGTZ=4Zsk8gXzT=a8L;`gA)- z%MloRtXi@#BaiewAjQ7N4dgWy4qVd3_UQ=&prcZXXeuE7ERYA! z3(h zjH{ccmWYSD&2ZqUER>gc@fHnF$&q)j`I=X%xO__*O-?)vgJ%2kNA+_APt!44UO$4% zTxde2EQG1)m3)jvG~ak@AurE5Nu;AT(jk?ql8MU%9-!P(NVkup3w&L8_n=6On3}=Y z4ID)8`ZUn&q&Ql;e=A=jdxNqgNjUPv8NBjx7Z**3Nv*1Gz|C6bfLG@6lvU+G`rFARd(r_ z>OLpS?{^KUE(w1HB&Q7 z!Y@m7!SHkhlTwJ2sOHR*aQ}a{#jjiK#@v}eU7T~_n^iu2u`QLZTx-e=RCIWVNjf~- zpdl%#+(F)jf5qgsp?notPlsqUQn6V9FFk%*WK0{ln#nWNG7=aX6Fd0h%3pBb-P+FB zzl**}zeb*3^5drl3VabRe2e`fX=md$x@)P0o&5J=yQob%Y@xGO-Gh-ES=8(%__HB~ z(u3_d?C>2PkljM9>Sb|BbS-KQ{YKm!a>=Fh^y<(unQaVfG&3N#4kVZ^VP2v z;jmE}25Xej#q1@N$t;w7Y#JeXe9r+_2@IMWp{~@dNP+gpWU#y)MQ6|5VP|&8$Zk>k zOZKNANz$snoKL-~&R;C6;H~DFc&h0jrp>rdM;{u?y+>D5Q;PxQi)lGp`$o_KouhfE zMYg0TFPHamtY#kv>qwGzU8MQXRbW3H15tH9G*be;VUtLbFff_GQCa&7^fdFn*FnBC z8qjY~kJ9KrrnK6RND@}q!|>q4)OE^S@t@)2plH}|YMRhY*GIb32KxtmnOhWO&ko{^ zE(>7OiUoY9t0y&F*hM`XeR+4AGdJIG9ggnL;E&2`1omRGxbJ*rI`-ob`eDVt+B>bP zc4o#g_OKPgI;b#}Bp0vVYE{|yY>mhby z)@$Jgjc|#NhXM_LqAc;N--*cr^J2V`r$p(`3_I2KL|EUPFZm9;xQsUu-Vdkg?{}vB ztYwNsaZELCR(eT{t`ch0atV%SS<-9S5maGLG)?X{uSUrgt(I!;Jht*C_TaYnW3 zV|I&z-?J&IPe3zt8SGaM7yWpYDydwyO!Dfn2i7_k*bNXk+jCXpA;|nNmroJ6>RF!j zW57tegKriyPro}PCzPke|c6=v20sc6wIvx9?;mCFB-5 zRjt9z=P%LOgGP`E&9kwxyO_>6bcbdPN|Ed}nkCtmlP%g>=ViBT+If^Iv4EUa(Gb&j z2WFHkle|8#OY%Z<61~{k%zpkPiJV!~!`ua;Q4mrpQv^DL&Oglg-tOJf7D(i5 zC&1exBm1@CgIHB@8a2LqkZ;M&&(-K~6q?*z}#eK>Z-5|p3+WxIYMy2%@$mh3Z@aWf5OZVZP;zY3P^+D&xF zuj2m3S?u|#$Iv@bL7aJ7gkI%wP@Oax-W2x|z46fyy7La{eq}@NM=T^oqas;!a0HCp zUWun_AClDx&b;i)aU4{kgubG)xL|oE4u09dR!>R76WW6R(ou?UO$@+?IaR2z=mI!j zJO{o;b0Oh}EHC|0X?rBm7v8Sj3MnpHv@AK2EPE3Qs^2;xV1~e%ej|ee-%p0o8}37y z*L6}eUWK~|JnO&GLWc2c2?=}O$d~9viXN@%Lkd_Fe8U6VLagcf zm3g9lik$UPG~*f{V=!PzESj{}V8Wi;HX60^Y}cX7=rnW{eIsNU&iw0z1MWY?chlUt z!>UH{oWZs1@FydI71jhZcJCI=pbB7Lz6{+43#q81)ntuX0Xl7~6j|)jr=F|y>E?N+ z=<2D3D>r>(t-ZTN@zbZ+3|-|#I~@k2m&Q!=2pxgSbFRUtqrOo7WiOw!Ql8K1pv->t zD|S|;98VhF=P~}*iQCo$e4@IEJ3uV1tGhwwDGA(+)C^#YW%h!08*LgUL?mSTAg)D25;#IOVGbi6EI ze{C09XElJ^W?LK^*Ml`abKriEyLc$cf@kA$5kD57b!VyVzB!4|J~IW)-D{ZvUBEY+ zZ{p_4r@<=!EA+`WrM^!_5UH*VfiI8_P45EGKWQs9zR&|v3kJcfe0dJ$N65BYt9X4~ z7Vp09fc7ODMF#%A!D_2K?cy_PU1n6%h`1)2L9AiLS0j3(F^20rRhPU9cZQkU*V0`b zVZ1TR2O29q_{nA?emH#*?V2$ZtX{gYeQH_Y7`F|Bh!;QG(oJv3T%je3bHG7>b6wcB zfbPCJmVcOho*X_p44TO#xZ5v@W<`XfRM@D7*kV%3 zU5+<%*1Q~u4S%6oMHhtrz_U>rR3ovRKAyY_J=#4*%g&9% zl9Zj$Qo9!-?zDh;pW}Q{RUO~#H3DD0(x$i4`t$Wa`_qnab95QEn;*?D5YIpIh$pQu zgp-R-Oml9Z7pD;o#h4v5m z1{L9}_&-A*?5Hk=__j09^Dq_;)Vfm90c-g4AqZ51by>0sPGC=Yw5jpUDn4A+ArntX)D1TJ3Z!}E`h zK)bYG;&FXjnbuhqK61VW>?%Kj9|G+0Ma(TUH&>$ZA3uPemjvFdJ4FuX4aP~)=^$V4 zMoflGqNCg`=*MqMV8Qg^bk-qNc+`=JyKo~uB~ciz&=(mmpI+nAJa96%IJ4S3aSj&!0Th7;2MAM{QBl)aNf~Q1g zI<4=i5|>ePUWqkVWK zI$@6Dc(IGlUG6jM1T|7Q!;M1BsjpHN{d!1_XZVe#FUDnIuBSeJiK-OKhmWDT@8)pr zHRb$skRNUcs{zBi#(ayH34IZ9SghThfg1ItHGxAH@-q4T;NC0bSR3^r$>tO~u1F9) zdn|^Q-XPdf*cT5D{U(-Qs)Oyd3AkV2y^MI{jq#!D#YMIcL4R*2D%>z(iBeLu;O$?c zd)}M3H)PN!Zcg;njpNMg<$dwo_Y;|UBd66z9bkue0&UmXL0#-lV#kG4etOPgnjG2$ z!%w}1r}bHMTz(Fjw9T2$A2OaMswzR>brGaI&XAUURHc)G|A2|~TEOxP;0Yo4YQ!MO zk-bJ*`{`1pLn$J4yIP#>tiwM(wqT!cK0w1$R^0cWBQNQEh`SUYLq&G0D0JC#az98(lk*@p3(49Wc1(++Pv_EZ!@oAMcW{{}+*H?Zh)5FH>BK&n$1 zmN){1pA)$=Y$K($>NG@WEI!@Vmu}yzOgjd)!jG-i^vU9@Bv2&<->00%hW$Z2X3k3X z(d+>kboT>n-<}N%WD)o0?}np|zhO$WGUROf4d=NPPt6ldagyOj)~!!ymxv*>UXbr^R%y>}42iWyH@PSH%0_?@5WGA3YHA0G% zJZd^Uoh2bxJb#JhT5ZXYDUNIx9;F{c7vbA}`?={u4%4rCkVY?gGU=8FJ2TNls6htO z5tjQwe(X0mvb_c7pFIr!7JPuxb#7QUYa0vc{R~!H>X5%Mr(-r<$A|!BdZf&QZ&lfV zer}1VW_1mYc2DEQ&#ZZ3vpj8H{ttHyv*WF0CRDBD3+&C5W+k>qQNH;-{94$T`)E93 zaFOsK-3Ht$U=X*Li6Uu>&)}24FY*4f&(NFbOy^jP=RP?mv@P=}dpS84!j4K){kqBU z`*HwrJJgq}n`!W!DpR@duW(wg*`F>vuE3?gZUnDgd)WNBPk;_}cLUWYx&?3!@82Uz-UyuU8LKx@NFp%L+ud*C}&Zg%D5_JoMA{ zG^4F$Jj$6i!`k`w*i+(79?Ph*heCE?(Q|H${ zRM~too?PWg_XUonx)1%K<3cxt*KI}c8jSnbo&!%8O}1ZnH(Q41ldntdU|4n{EVwy? z@7kA2muc>&!Gj)Q$(8*u@5vClPG&fDzdw_%SDFakk4nMrYAp}&d&$-9E9lN^BZ>Ds zO0zW1!Yq}4&>P-HBm1AGG8anF)O7^dd* zKE}M3q0ZK^e#lT@>sBYg=&xhh{HY&B(aQx5wSu#n1XEs=zg^V+*&ZLxi9ySFQ+Bsl z3AM8?kt?H8#IhG7@Y3;A*jH-J)+p@5dmEi`kLwyp{V|KLPW+864`gx0Wr0^QQJO#b z7{c@GJz1^lWXOLxm(PFG1gCF}qSBvFqR;K|P^R$?Z|UeTExj91d8UDnStZMFM@Pd6 zB{?*^n~cTRJ@BfAkWE{z#sk(U&;j$x@X#e&zC!*jqH#F?VwEgv9UCChjwCefQZn2a zl}7t7KP}`)%Mr7ZArD2d1mSC-pYaxuVT8v`XMu&U^ML@d3Unb7qAn0%*G122{Eh53bEea79lZ zu@b!YAAJjWdv65O3<#sC7tQG9{n;$m@|U=#-Iten??+{|%OErVC7QkN7US0z<~H*y z-pQ{6IMD}xw0ZI^vfKFASX;jE-e;_RFJ|YAEokbCKTvU~4VE`a(S4&w(EHzeh-JGP zp3FMIe@<{i4x^jT7P0~6Q)tq{tL&{!DSvR= zg1^&Ig$px}W9Fl5;;<$On$mtlOPU##!<0x$dY4!WJg@l1(~$l%#4cKGK?zDF?wO?GS3 zvhWD@HBFsgQeH|Fr0dydM+@Rm{T+Q?*z;1itr+>U7+)o4!N+)HJ*UiQ!k0f#EaaM2 zhYb;J`4c00`{o1bX>P~Ag|q1! zMZt-?mBc_+iZ35KmYbfPNo^_?0GXH1O)f9sx@JO7CS)xCmm*DX3A5PU`EvC8z;XPE zg(00WW*5)?$KdN{H~QK82^ssd3>2j%@rv96RJ4u8xwm%UivxxDBFCJoBvbyZ`W}31 zF`_O_<*?xX4rV^Wm?vy<$Bwmwc$j7yJZ_mmW$uU929B*qhf(p^JLe$WIq(|0w+meQ zxsf8pN^LF;OcbxsLlUZzd110Llj|!cA4}H=EFB|o-Q-FuwIrZeo&XvO(sos{$7#i~ zQy9H*CsC=L&X51<18Xz;3cmSO)URq9_Q&-|K;74T258%dm z>HJsud@g=@3r-9D%Yu0Zw)TF9c+kN<^hm`UIG$StiwpHoxvwh!aBVEDih2txbHhO{ zRF)ojQbKy32$|JwV(eJHf(kTKa(TdN_`9SJAJ7pB(Sz!sw#C(xePU~ z_eFOb2T@}WpbF8OF;95`-}a@5kRV++c1wkKG`nI?myrM57J#i6d&v2Zj>z?2L+<7v zJTE(dJ_{_PAA$;?>{c=QYE9T-*IU0rW87ARCr>%8r*m{5xav2PP-HjhTz)ZcL8*&8Pm`k)T{d7y|1&l#Z|w$U{UE;ZR0Q?br~-rE`(eLF1)cn~ z4DZAjVduLhcKFa37JKL*Y+W<}ea+{?saZ*2t$0f4K@-@%+b2M^$v}8-v_PSBBv&hr z;RE6iLjE}kOctf^_o;u;=U*Az`72MFMolJXXRO2*hoq^A>lWyruFr4G`@_!tjuPvX z3uYOCZL0YA8)$7FLAJcAC$$5Y((T5dNmauLk!4;NnRl&`cxQgWHiK{;66C>COzco1 z+mHY1w&B6OsyukLAMbme(ydiH$Q+A)xOH&?GyHuBYlmwhn<&Mt95bQGpn@9CnaE^y z6``c9*fv+bndD6$i(}^YqeErGM6Ih9Z~;$_Hxd_%qpH5zZZ=y8`XR%q;nTO+=Hkuw zURTGZp_^cBgDE>P?mNhbZN?}64xv;3BJ#6YnkzOoyHpPwZ;*zwy~p7z zy-~GWf9hgalOuE*m4HowF*ew#^K~s#Ma!+{!JOpnsJeTx$kMF_a@t36+53KcLBnlQ z)0)X@)J51NWbAtj6EV2@B$R6S;i5s+^f{cMcYD8L+UZx!r)miI=@lz++S&iPk;VrkPuHVd*C)l6Ebb92`FgQ&p7d+{;B|__v8zmc>zL+(gWH zg6KE92qsro66I$KU=XIqzn5>NEug|je0+uvw2J#m6Ys8z<{7Ff$ zYgZKxKbJuNG)$lYs`XeMKL)&XKR|(cA)TFikq@zD5E?ugHwTTTo5E-Etz!cD^8jhM z<~Eowt23hVOS^05Ub+C+hR=kA$#VRXQzBg?cu$`>pvzzpNQ{}b}kS)(+io-B{$L|iJ+f78z5taHq~2|OINM45xvb+ zqWobUsoyk!Zn!X&Rz91DLw>9zN7Z!s#t(jAACt?Azd7pc6)X>oSL>4q{b`qzJpUiR=khretZpI48y=MycFHyui)|XPvA5Q z;tek!H6jWlN`I4wL-ygaV+r`>GJ;gP22bAA4>mOm zJ6okg>4?D^7`H(l2N|Z*$G^<+wdQHj!L#FW==mA=%}>l1Olimb?i+Z2mI>?cau7#M zSuVQi909lU_wtuBqlo^}N7S+_3DU*3pmMqm7G87YgJMK{?uG|wt{F-BD~fXk&Hu`d zC|+cLfKUA~ovUw{$jj?vU{%5{dbei-sr)&Rt`c2mE-i0iSGqKH*uIFgu8V=u!M}0j zZ$}(I?=(66ejiwFHDqcVWx0lSB1YX#LhDNfq-ou382c(6y_6TxyBEJ<(@YoqvSX#d zoUwww=d;1teg!S;`U@M4!{~x4IT}C366QEAgyxVy-mmjJ+%})cz490E@(EAjxMKo# zs8*6rJnp}UfH#;70 zWqDv^zrDC-l_96OZCpkznI2ZErWa=~;LXmuGR`Aqce-$V1Se6H-<-9ag*@5^E7ve` z+x_@b*9UNi2-14-JiNMXFpip<&O7(J^EGuaNo9!bQKxF-p5V&Xp0k1+=)uF8_855gI8SX!=V#Zt;GX=$+{v_q$jU@g=jRP<-Ue5m zJ8C%{)G(0e8@&PZ0D1bu<2AJA>=VaS-xC`pj-!^pK0>Z~7s!{a;F<3|`3L`pY}dgt z$Q?2X9iK?2Pi(`*a^`qztPT%UvSr(5N3xRaeti4WNcy(ld_E`lHw-&O=)jy_+-*CC z{?*z{5)7*WL&NFqLqqxILuTBk@i?5cyDIE(*$9KS4T8Rpu99$YW2ZN?3HtFgYTGB3 zc6u4%uzka6yZZyIGJg$r1^zU5nLQMlnWCG1Ca;vW=iQg1Xl%nSSY4AuJGm+J&3+F5 z#Ll7)r`3*o^BM;Nrd9UB(<(u`1Fn*W<%?Gp`Xj@b! zN1LU7blUl;Fl**15`I_UCkZV5Z}zsl>egs#Uc8Oal^5XpWn(z-U4`Et+5jUvfdA8& z$p_Y1qpRRIxw^FkM)x=8^imk@nJWvd`~X-tj-gvutRWL-j^|O%#;|9d3Qtk1<_&r2 zq_i*_nmxWkqEsyy%S&^&yWMQ8wJt9zXa!%(Z7?EV9{P>E4coW0K^ zPsaFij~{Y8^=drc>sIEGr`Mv}fB|$zVhid2I)vu-cNE_@(0~>FwfKQDS6-@8NcO+D zE?%yiiD>>xeB0}rtxV}0y39RN{C8d}nok`mel%3Xe7ARFd)-&jJufSGs5MO7t1RNz zy&70`q@S=yQ}{odH-OdP{lw5znpbD7#X}R;^3cvDRC&G*nfl3yT2C)Qi#iF;xL`^( z&lI48cssLLy%wKz$?}(}v(Zb?AxFy$!6(y`dE5I5JX}YOetiFsU7mWI?3}6yesTS{ z{1jgpa&|ZOWy7hu;4O&Rm&PoWOKgVfzhIY-yn=GUpZpNF5GM~ePzamG&%B?>7l%}d zw@x(SBfCqi&$dZXvkU9khdZ~?RFusc$18)>=J)vBGk|KUI>Ex{9vJ`U9j8 zt>Pnp*FfLPE`2L7Cn5pi&s1b${)r)+ro`9A#6h~czyZ6fi(W$3Lvksi!3;qEEHLd`S{ zeE$SNb&&<`c$EYmJ9l!ZnZZJiNU*W&H164*4Nb-m&?jO!U784#|EvJ8FonU!d;U*p zX987I`}Y5m21Rp96RC_LMT2wpbw|dECNhPJq@oNF;dvS~kWx<3pi(Fmg+%Jy*G*)M zl36HaNQgoR`ESqr{{9Q^?|FV}z3*=Aea<>-pYOW%S@(VK-F090=gWIy7eP$JGigllCewfF%4es zjy0?o`pjDeb#wNJosA~Y?~Jg0<4?kATfSB@{te!nHJ`b9y~25B{PW?sSagv3( zlV|rPB+L!_n#>|9xpG9RtOrq+35O<7V~@AxVb*v#^1U^fCDqz9(<_DC%$0M=f-% zUyYz~d=!gGDIhYRthmT&<}h&ILGYK0W?yz?z#hK#{)Jl>hz|55Ge!-@hS7=C!Bmwc z7P(PP^*lHebDWA?KVji3-fJ__6q;90CT<@cSjIlhRcgAdn&`%)2!3|$RshPJ3UgOJZpXOiwSov^U&1QDL-g(CefdcQ24ecbQJ#!fO~ zYl4sAR@Wdh=jA=InnWb&e@BKnFUur~%ie>>9V6WKW&q)y4`uIC!|9+Z@A9f8an!tG z2RK{HlaLMXP`V?4C3G*P!Zqrky>%AWP0fV-i;K`9s0#1Rj=}-SWh{Mh2G&F zJUv{7J#|X}^^^c%sOAU|IeId?#1OXBY9#hMJbR0;| z(3eff*-0EyR)WXlix6^kId`>Mo%Q+Tyg4PyTu3wCWBOjMf?$Bi> zIo|kaUq*$ibP#L%R)gX#1z_XiLWVz%z!SlTA!Lpd&Uu5JW{M#h{AUt4-9LxZi40Se zET#^1rCgB;0p~a4+0l|Qu&CLR)ejlQGDTf5PJ1Gpx_2ES{0-SH5>AX4Y~noa3$VlC zEhjkINCUUHkUjR};ij}VF^Rp$X{B}p$#bT=HV-2cKO7`iv%6u%lW=Z!bSik|HDgbc zNaXn1Wx3H~Sk5huShh%Td#uFV``mhVEo&+98eq@lKQ%$jwK`C@$;Su%J-}v`6O!kJ zFpT%}efTw!9Q$I&6o%KcN?{SLPxwRJ5$cMIw1<#K2iD*T|1G#7Bo0$2uVuPNrf^DQ zzM^>38F(Ay&Z3tkz-LzqOI0G_b(0@`e?t`$Zl+>?8*O}`HwK>dI*5I-jk{PnnM$eo z?Bc1 zo7oDZTxMG`jV$RO1|L38Ac33sHWjxC{1(=e{gT!Gj@wRe8v-xgUaQtpFNcE|ub~AB-EN$+gFJ*uei56O@m$H% z(xf}hi|le$B+ZRqz+rkHRGF(tE`3T7-aHzD79tVW+O)tv(fRU0g4>Y%O$wv`P-iO# z$6@OR0rZtU4>7#kY0l!`Lx#Dkv>2!Q(G` zlk^rZ$QGnhGyZOp?kB|-X!<~`=Rj^YNfv$-dSc;tB^L8R5^r+-*z%NzxOvGkCO7^o zUiO?pyRVy)SDW5~#I^-�?2DB=*!N3RiLvw52i_}GNa+~!bi6=>7etM=smtRxy3G?n_@A5pSy~KV=f_J`!ogjMh+sX zJ7@DovUQxyA9G>Db$>X&`6S&juNd2kWQEhiOQ9~voD6g94YiMB@vuVx234!Fjn&;9xc9EbdufpuBPVu$=BiNOXdhEh^-g~HA z7Hzpzq+yW>Tk|#)%eo}U;DGTgeiA<~SX#tFzQ ze9hgac(f~hgpC_gA>xV!nW8w0t95b4h#_XMbV?{4qrmg<&P|45qeM)YKcC5Y%43V) z3J_cx%Hme#h+Bn*Tx(N1cQ&sOl^@I{%bo;qipwfs<3n3=HjZbYrD!vosT)8Lx{TEh zIV5_vw(sZH~P=hI8s?gm#TE!sg& zYG|;KwgI>^ryYF`dSS_(O{93?X5{unk{NQ2Y@+r7CcEqmTJyQ?6pwbO4q3=9SGd8V zQN~=?H$Fe{>>3uV6v5mZ3{(GjiV5$(;Q2o?gu&<4$!wQ=8at~ETc>WPTD`isw2rl0 zxXx};U?~!pO9tTaRs~`kP{N&5E{El{Hk|FrBXnzeEuH*LfS&_)kjE<|;0Ql&_N;yQ zIO15q`mQmW>jP*pC7Nqb+E318rE<3tYU#5r=g_017E+eTa%wNq z(61Ho(?u(80AzsDh;*it?L^lJdytp)RYFn4PEOr&5(!LM!1=eC@qIv-EnB>iNbw#v zw?B7O)V%MD@yq+doC2O_`#6hE={!SUJhH1Wx_kt(dTFxBP5!id=|?cpH3saLhf?c% zalH*It&?~A;nN)wWUuQ|PFFmNEQm?RK~M6id+lr}t9=j6x}KoZCBldAzMw6)j?%-2(A_^0)NH1LdDm%B zpDhQYsyCwIHB&N`N>igw8{*M@UU4l^YWMi=BF6;m)gw@*{zRiPDe0jt{rMT zX+{70SD~)AKN=0K7y z%cdRBUIveQ04f=2Uol2!|A9mblfBtHg;qfci*v*3l@Z7b(}sq+cl7RUhBh7pRI(_ z4Kc(`@wBjJy%bz)^JX(w+=cOdw28vR80zpS6c!|B(pK9c(5lVX3eFlL{&IRhQBu4J z<gFhbFRU!OaYc;DOw zpVI~Cf9L`&%;J5|-hG6~Ojlg(y9UEgOR^@#(Kvd2E)|>Fq9|EKSZkui)~US{YMQ>m z4Yw^=5}#{6SX~Y^3m0IKCt-66HAtnR8ys9`$aK|OskurO9t%{(>gC78r5g^?X9kC; zQK||oEvbSSom4Q*4TVicD)=Nc3~j%*Kx>;k_*i@|cg(iKbtlYW_qq#s+FPGpj9G!B z(&v-P_5-+U)iSouSOlp&8+-L*z81AUoqeI_VbwhalG7T?qP+rz(l^$_#;q#MQ#>6z zwhyEm-1LaK$rny~izX|BtvEwsCb=agOP2RFCE)?niPq9Q+H>7EZgSQ>FcCk*_RbdY z$FC3`_*Q5gCPMxFKFqpcEUw%fi=mreVfo>U)J|a=kr^)_@mKf4r*R`l@2q#?#7U1J z?~p#JHlIrWRK5v@w~s?YTpiSY??V)WXYn4v1@0f5#j1MhG55h~VDvhTw#EdK z_f(A;rD~A;&NS-yZW`Li`0;hXj?Bqiz(smVlWnd7a=fw)HLpG7w#XY|uSX_Cb(B2u zu{I!MGkdU`?orUA_5zNMeN9igg(396OZ)g5lYW!7lJUX)nOSvTQfLx`-3mh?Aj<-# z9jT-Kg&XO%RRn9o-7wA;qaD7mW|gI@ujzr2_n-1wbmP)ME$xQ;7C-n(luRKiNjvEc{ zL7qI8;r)VdyutBb?LeY_H@UR52#eIGlMPe#nWnlaXZcJ*oXO|r#?tLvq^Fo0X~@s} ztqw)uh&1TrYr8$?n!)o2!SM8RAX&=2M_n5c>XbK7=Y1w{ce^3HjsHlk%l1KrZV8r6 zn#X;95eicMHlamSA=g`(XF*?B$PT%$r>|R`@%!kd(CN1wG82}dLr5vCDr*O)@dfln z{|pd(?a!K%Ov$6o-FU~tkeObM6{5Wh=ltHE)x{lSJ9g~Ho6B}`?++WZN&Y3`8?vf4 zeYPj#l+D&~2Cl;}Z5DP9Q#a@+HnnRbwOSJkY3M4_HAGRMA8v{3M_dQfcRKWqLm^z;`wDJem?mD-E{P+}6ER@D zChI<3{})@>cKcX}-#!Xw)YgGhb{3V*ujTs8+X4zNVpwa?2b#fkL80RUjDG4u+YW3- zRqOr2;DtkppjMXavh##=yJtAvK$AUBH2@!tc(K#gMjYY9^QV{SGskVGsQsfYSlGH3 z6Z6Cn>RpER8}}1;e%uXNDB``Awh|KD0GY?6V84$PiF5UZ$R}lxJ0t<~_DP8EM~xsJ zKF3kdpoDWa8ISWNoyqu}J=pb^`r^Ygd2dy(bn(+|KHT+9htN#E2W!9V4KV^a$`!b? z)eo)sd&4Qt-TVMvzIGP6zTJjxA&%%mhC!g>9$fkI44&torTd`^o%VV$=7<=4O`Qo7 zRu3kBwueEFZ;9w0@J2AUXgYYTibc`OG`zj31{*umS?lpKDAo4ldiA|PwO;gL>OC&e zXj3ilUK`9xCpU8kcl=--8gpMejaU@_fA~{rWlRW`f~}KQF@uSpaObclarK@}5a6c8 zqMz}6tCv@x&9+robgPQzE=_03eFrd2!4&v7gRgO_n2KLB=8{Kq_p;RW`d}lTL-M8l zVX;#K_f&Tm)8*$8M)5vcE!Qgr_0IaJGoQa+xmVyyW65%xm2_w7JwDH+08M&^B*Cc^ zp4u6syTl1T*ICZ>U#13f;f?fZqaLZWG$p4RzEbnA^V#$l0eEM%ym-G1qY~e(S=y;S z0L^8>)NMEDecdT!{N^MZwrtrC}J^^Z^*z;vt``3h8?@5QJJzE1aO98SwPf*0?s z!`4zi&f?P!vdO&&a;9lRSmH>clF|TX&b*J&p?tW=MI*=aaYOD3u~6wtsteVs|(rQboj^Q*C@d_UzY{+S-{8#1R!K<4ckfG)Lnp=VDDHYFJq zr{>(oZMXBFLFqB<=JTs}dx`P1*+$k9z&}5j7$Fx+INgj~GA1(;rX??DCdaL)z2pwC zNDL)KrHX`A>9E1hl~{OiCkD8Eh6|~LPOQHy+}J&c6tA$OPX-?m8{9gJd+)#Cy80N? z)`C>>arq~_f4iC!Tv)@l8oh>NuL{9O&mMgS1vnc|^!s!UPMgb97tu!!6|>S(qGtn^l5X+|Zu zXm~So>sqWx3IpHcjnvn=2Id70Vi%G{#5V5(+V4^Vx* zE)gd6W?^TV#4KfI7I_!2x-Cq&4-YD^&I)Cm`fxEL~+e#9MOEOfIV&*Lh`eN zI5{aBcFTH#=IGHddcryWnkul*>aAd>>&ETv+$}yQsf_zBzJQl42f#EY6O!D|fK~J{ zv8|^p+~zr82isqBWnKl?m79iv1q#Gn;KR-Suoc?W81xf5!?omP%+^H?ZX}H1YqAab z^@Mw&`AsK2soM?Th7<`V8|$$J8^xggZYGxB8cp`T-3c}A>7*`T7lf=GLT=w&3_ZJ) z$k|sRoUh+GD2`9Y_G65@tF@oC-_K(W^=UXXZUu(qT;TH!-ei^bTkfuW1YH$%5hRsr zuqLht=muGmjT;p(US}v2-yM(I=`}DWUx%LSodq$U!|)!@lxS=20kd_-vMbgF7+s)4 zJ}C#o35hW@#AG}4o_iH?RJMVvSt++OeK!U_`7Xx0`)I(q>r_K;trHWg0o(qw164cPs{&thdwIkK2v z|KRi45$;+~Bb-(e%&{6T_|VMrqg_?7UU3d<*K81GZuZB-1`!z_B0(Bd2hiuYGswO7 zUZg0-5F`TS$UWoUBx0%wiPxJ#rr;%bCNq@1y51nJ^VK4y#s%=^fiziM@fo#B;)Dxw zrC7B4LvhzjhN{kgvIo3BnX`->K9}{USGYl>du;+f-0+pV;Tu6!m-H4l>h!?*yvKm# zb`IxXu|PS!@x(sk2zNWO5yyVYz?cW8VL{+tpjywl0p*rhHC&G=8!E5{kB^`oWWkCp zlIiDUV$msW@FTw0@_mSPEN1j>xb@^ad$_S!nyK`IC*CzUR)MSlW)~abKkmBV9?X@trrTIQDk_cCq>-(1MS(4;mbR|c0K8Za34 zjqyU=7Xw*+_ZbKYlmcV5$(+O|NeD<+hYy=oxy?Lx%A3zw$>r^Z&8<^$%;aqDWLG(K zU3g1nathJw#9%?3ohcg`%Cn4n4Mp9;QKV${cJwPYAuX2Ih`gL4=!E18uPJQf`}j!V zhE*4EiLilvQh32K6TefLDk~Og`cjZ(yhWV4^n_6HY7ZOZm)Wc)Fr9S7@Ri_Y1KBGh zEwXD*5uO=%LU`p#2!`9vhB@0;vzJ@d$lBs!QdP2m3r*||e@IS+che>@8|hFw%V8J& z+NlNI%Q%ua^Mjx|jL(|~%*7=lM}h?+o)6uG8q4Qm(G3f@d3_x3TR51`*?x>mxqS}f z-M@iL%5EAWR>EZx#oW7$UDP!27LJ(|3u${_!)-HUCWFVZ#DY;3vr32K>igYzXz)um zpuaMTBr54!^*lEBlqFGH6GXgAUSX-|7`Ci8V9Vq?(2KK!s}G7=tq3ZTAm`VL=%V}gp^vBr_D?V*)6TeI=7c8hW+x(w zZiZbCjkp?~4;+th_{LD5;6IZ5Wg>ml!P%eoI3He)w|#wOlx;y7oYqmSZ=qgXBF zNa<@O^5^y_GQf2nInlb1yJ^^uGw8wR*q6EEs!gj%k!7#H_z%%Bm;3#n`LEXhJN|&x zj9V~BUzN^YdRyGV*Fbrh@$=B#2f4-X6xiCKx#EtXBQ$4!Go$J!tTLGfwyA@Ob|T zT)KlY%+urRW~PK=t6wk%9lgTMeHIH{{F?rm=N`i3Z}+*cX)7@z_8hg&$)r~fspG7U z4%BE2!i*7`sIqkd&Uhw=YCU$NyP23?^{K&Cu63B%*hpnq7OD%LZ~?m#g_Aez6|?Ww zz$#*3ockTvBdf+Kx9QUSZ5hJmZr%%q_Z}Uir1=;B;l$y4H~%yLK~G}r->|*bd9PmM z8yGNo!sy@NUi1F*W-kAO0CGLkXI2lMEcRpnd%BnjPakt~a$MvRxY*x2$iXvUiEn_< zBGW~tqed?A4DuYc#(TA|=W^c=&mcaUMZQZWPZ;nY12VO;Ff+BbFdaR{)Qo>W+T6;* z!qjrC*;oq;v$2-umX=my$C{g2j{Onk)H@>45I({myOX`-k6>gypz^OuPA>l#l=<&U zKcfMERjPIUN2M{pEBy@c`&G$l!9ObTP5;-X{xh!USEcv=tYq=K($C}6#BrJyf0;{R(Y|M$Cpy_WsxQ2h3Z*Z;Bc-;@3EdZhTrQR1)O>trwe QV~CPa=Kp@2|6k|+Kii}q_5c6? literal 0 HcmV?d00001 diff --git a/README.md b/README.md index a8b2a39..5d5f213 100644 --- a/README.md +++ b/README.md @@ -90,22 +90,26 @@ These states provide the necessary information for the agent to understand the r source venv/bin/activate # On Windows use venv\Scripts\activate ``` -3. **Install Dependencies** +3. [**Install Dependencies**](requirements.txt) ```bash - pip install torch numpy matplotlib + pip install -r requirements.txt ``` -4. **Ensure CUDA Availability (Optional)** +4. **Ensure GPU Availability (Optional)** - If you have a CUDA-compatible GPU and want to utilize it: + If you have a CUDA-compatible GPU or Apple Silicon Chip and want to utilize it: - Install the appropriate CUDA toolkit version compatible with your PyTorch installation. - - Verify CUDA availability in PyTorch: - + - Verify GPU availability in PyTorch: ```python - import torch - torch.cuda.is_available() + import torch + if torch.cuda.is_available(): + device = torch.device("cuda:0") + print("Device set to:", torch.cuda.get_device_name(device)) + elif torch.backends.mps.is_available(): + device = torch.device("mps") + print("Device set to: MPS (Apple Silicon)") ``` --- diff --git a/plot_graph.py b/plot_graph.py index 8a7ff6f..97f02f8 100644 --- a/plot_graph.py +++ b/plot_graph.py @@ -2,7 +2,6 @@ import pandas as pd import matplotlib.pyplot as plt - def save_graph(): print("============================================================================================") # env_name = 'CartPole-v1' @@ -28,115 +27,110 @@ def save_graph(): colors = ['red', 'blue', 'green', 'orange', 'purple', 'olive', 'brown', 'magenta', 'cyan', 'crimson','gray', 'black'] - # make directory for saving figures - figures_dir = "PPO_figs" - if not os.path.exists(figures_dir): - os.makedirs(figures_dir) - - # make environment directory for saving figures - figures_dir = figures_dir + '/' + env_name + '/' - if not os.path.exists(figures_dir): - os.makedirs(figures_dir) - - fig_save_path = figures_dir + '/PPO_' + env_name + '_fig_' + str(fig_num) + '.png' - - # get number of log files in directory - log_dir = "PPO_logs" + '/' + env_name + '/' + # Setup directories + figures_dir = os.path.join("PPO_figs", env_name) + os.makedirs(figures_dir, exist_ok=True) + fig_save_path = os.path.join(figures_dir, f'PPO_{env_name}_fig_{fig_num}.png') + log_dir = os.path.join("PPO_logs", env_name) + # Get log files current_num_files = next(os.walk(log_dir))[2] num_runs = len(current_num_files) - all_runs = [] + # Load and process data for run_num in range(num_runs): - - log_f_name = log_dir + '/PPO_' + env_name + "_log_" + str(run_num) + ".csv" - print("loading data from : " + log_f_name) - data = pd.read_csv(log_f_name) - data = pd.DataFrame(data) - - print("data shape : ", data.shape) - - all_runs.append(data) - print("--------------------------------------------------------------------------------------------") - - ax = plt.gca() + log_f_name = os.path.join(log_dir, f'PPO_{env_name}_log_{run_num}.csv') + print("Loading data from:", log_f_name) + + try: + # Read CSV with specific column names + data = pd.read_csv(log_f_name, names=['episode', 'timestep', 'reward']) + print("Data shape:", data.shape) + all_runs.append(data) + print("-" * 90) + + except Exception as e: + print(f"Error loading {log_f_name}: {str(e)}") + continue + + if not all_runs: + print("No valid data files found!") + return + + # Create plot + fig, ax = plt.subplots(figsize=(fig_width, fig_height)) if plot_avg: - # average all runs + # Average all runs df_concat = pd.concat(all_runs) df_concat_groupby = df_concat.groupby(df_concat.index) data_avg = df_concat_groupby.mean() - # smooth out rewards to get a smooth and a less smooth (var) plot lines - data_avg['reward_smooth'] = data_avg['reward'].rolling(window=window_len_smooth, win_type='triang', min_periods=min_window_len_smooth).mean() - data_avg['reward_var'] = data_avg['reward'].rolling(window=window_len_var, win_type='triang', min_periods=min_window_len_var).mean() - - data_avg.plot(kind='line', x='timestep' , y='reward_smooth',ax=ax,color=colors[0], linewidth=linewidth_smooth, alpha=alpha_smooth) - data_avg.plot(kind='line', x='timestep' , y='reward_var',ax=ax,color=colors[0], linewidth=linewidth_var, alpha=alpha_var) - - # keep only reward_smooth in the legend and rename it + # Smooth out rewards + data_avg['reward_smooth'] = data_avg['reward'].rolling( + window=window_len_smooth, + win_type='triang', + min_periods=min_window_len_smooth + ).mean() + + data_avg['reward_var'] = data_avg['reward'].rolling( + window=window_len_var, + win_type='triang', + min_periods=min_window_len_var + ).mean() + + # Plot + data_avg.plot(kind='line', x='timestep', y='reward_smooth', + ax=ax, color=colors[0], + linewidth=linewidth_smooth, alpha=alpha_smooth) + data_avg.plot(kind='line', x='timestep', y='reward_var', + ax=ax, color=colors[0], + linewidth=linewidth_var, alpha=alpha_var) + + # Update legend handles, labels = ax.get_legend_handles_labels() - ax.legend([handles[0]], ["reward_avg_" + str(len(all_runs)) + "_runs"], loc=2) + ax.legend([handles[0]], [f"reward_avg_{len(all_runs)}_runs"], loc=2) else: for i, run in enumerate(all_runs): - # smooth out rewards to get a smooth and a less smooth (var) plot lines - run['reward_smooth_' + str(i)] = run['reward'].rolling(window=window_len_smooth, win_type='triang', min_periods=min_window_len_smooth).mean() - run['reward_var_' + str(i)] = run['reward'].rolling(window=window_len_var, win_type='triang', min_periods=min_window_len_var).mean() - - # plot the lines - run.plot(kind='line', x='timestep' , y='reward_smooth_' + str(i),ax=ax,color=colors[i % len(colors)], linewidth=linewidth_smooth, alpha=alpha_smooth) - run.plot(kind='line', x='timestep' , y='reward_var_' + str(i),ax=ax,color=colors[i % len(colors)], linewidth=linewidth_var, alpha=alpha_var) - - # keep alternate elements (reward_smooth_i) in the legend + run[f'reward_smooth_{i}'] = run['reward'].rolling( + window=window_len_smooth, + win_type='triang', + min_periods=min_window_len_smooth + ).mean() + + run[f'reward_var_{i}'] = run['reward'].rolling( + window=window_len_var, + win_type='triang', + min_periods=min_window_len_var + ).mean() + + run.plot(kind='line', x='timestep', y=f'reward_smooth_{i}', + ax=ax, color=colors[i % len(colors)], + linewidth=linewidth_smooth, alpha=alpha_smooth) + run.plot(kind='line', x='timestep', y=f'reward_var_{i}', + ax=ax, color=colors[i % len(colors)], + linewidth=linewidth_var, alpha=alpha_var) + + # Update legend handles, labels = ax.get_legend_handles_labels() - new_handles = [] - new_labels = [] - for i in range(len(handles)): - if(i%2 == 0): - new_handles.append(handles[i]) - new_labels.append(labels[i]) + new_handles = [handles[i] for i in range(0, len(handles), 2)] + new_labels = [labels[i] for i in range(0, len(labels), 2)] ax.legend(new_handles, new_labels, loc=2) - # ax.set_yticks(np.arange(0, 1800, 200)) - # ax.set_xticks(np.arange(0, int(4e6), int(5e5))) - + # Finalize plot ax.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2) - ax.set_xlabel("Timesteps", fontsize=12) ax.set_ylabel("Rewards", fontsize=12) - plt.title(env_name, fontsize=14) - fig = plt.gcf() - fig.set_size_inches(fig_width, fig_height) - + # Save and show print("============================================================================================") plt.savefig(fig_save_path) - print("figure saved at : ", fig_save_path) + print("Figure saved at:", fig_save_path) print("============================================================================================") - plt.show() - if __name__ == '__main__': - - save_graph() - - - - - - - - - - - - - - - - - \ No newline at end of file + save_graph() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8ae20aa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +matplotlib==3.9.2 +numpy==2.1.3 +opencv-python==4.10.0 +opencv-python-headless==4.10.0 +torch==2.5.1 +pandas==2.2.3 \ No newline at end of file diff --git a/test.py b/test.py index 4015ef5..3c7b3ba 100644 --- a/test.py +++ b/test.py @@ -1,93 +1,185 @@ import os import time -from datetime import datetime - -import torch import numpy as np +import cv2 +import torch +from PPO import PPO +from rocket import Rocket + +def create_confetti_particles(num_particles=30): # Reduced particles + """Create confetti particles""" + particles = [] + for _ in range(num_particles): + particles.append({ + 'x': np.random.randint(0, 800), + 'y': np.random.randint(-50, 0), + 'color': ( + np.random.randint(0, 255), + np.random.randint(0, 255), + np.random.randint(0, 255) + ), + 'size': np.random.randint(5, 15), + 'speed': np.random.randint(5, 15), + 'angle': np.random.uniform(-np.pi/4, np.pi/4) + }) + return particles + +def celebrate_landing(window_name="Perfect Landing!", duration=1.0): # Reduced duration + """Show celebration animation""" + width, height = 800, 600 + particles = create_confetti_particles() + + start_time = time.time() + while time.time() - start_time < duration: + frame = np.ones((height, width, 3), dtype=np.uint8) * 255 + + # Update and draw particles in one pass + for p in particles: + p['y'] += p['speed'] + p['x'] += np.sin(p['angle']) * 2 + p['speed'] += 0.5 + p['angle'] += np.random.uniform(-0.1, 0.1) + + cv2.circle(frame, + (int(p['x']), int(p['y'])), + p['size'], + p['color'], + -1) + + # Add celebration text + text = "Perfect Landing!" + cv2.putText(frame, text, + (width//4, height//2), + cv2.FONT_HERSHEY_DUPLEX, + 2.0, (0, 0, 0), 2) + + cv2.imshow(window_name, frame) + if cv2.waitKey(16) & 0xFF == 27: # ~60 FPS + break + + cv2.destroyWindow(window_name) -from PPO import PPO # Assuming PPO is your policy class -from rocket import Rocket # Import your Rocket environment class +def get_test_config(): + """Get test-specific configuration""" + config = {} + + print("\n====== Test Configuration ======") + + # Task selection + task = input("\nSelect task (hover/landing) [default: landing]: ").lower() + config['task'] = 'landing' if task in ['', 'landing'] else 'hover' + + # Rocket type selection + rocket = input("Select rocket type (falcon/starship) [default: starship]: ").lower() + config['rocket_type'] = 'starship' if rocket in ['', 'starship'] else 'falcon' + + # Rendering preference + config['render'] = input("Enable rendering? (y/n) [default: y]: ").lower() != 'n' + config['frame_delay'] = int(input("Frame delay in milliseconds [default: 16]: ") or 16) + + return config -#################################### Testing ################################### def test(): print("============================================================================================") + # Get test configuration + config = get_test_config() + ################## Hyperparameters ################## env_name = "RocketLanding" - task = 'landing' # 'hover' or 'landing' + max_ep_len = 1000 + total_test_episodes = 10 + # PPO hyperparameters has_continuous_action_space = False - max_ep_len = 1000 # Max timesteps in one episode - - render = True # Render environment on screen - frame_delay = 1 # Delay between frames (in seconds) - - total_test_episodes = 10 # Total number of testing episodes - - K_epochs = 80 # Update policy for K epochs - eps_clip = 0.2 # Clip parameter for PPO - gamma = 0.99 # Discount factor - - lr_actor = 0.0003 # Learning rate for actor - lr_critic = 0.001 # Learning rate for critic - ##################################################### - - # Initialize the Rocket environment - env = Rocket(max_steps=max_ep_len, task=task, rocket_type='starship') # Adjust for 'hover' task if needed - - # Set state and action dimensions based on Rocket's configuration + K_epochs = 80 + eps_clip = 0.2 + gamma = 0.99 + lr_actor = 0.0003 + lr_critic = 0.001 + + # Initialize environment + env = Rocket(max_steps=max_ep_len, + task=config['task'], + rocket_type=config['rocket_type']) + + # Set dimensions state_dim = env.state_dims action_dim = env.action_dims - - # Initialize a PPO agent - ppo_agent = PPO(state_dim, action_dim, lr_actor, lr_critic, gamma, K_epochs, eps_clip, has_continuous_action_space) - - # Pretrained weights directory - random_seed = 0 # Set this to load a specific checkpoint trained on a random seed - run_num_pretrained = 13 # Set this to load a specific checkpoint number - - directory = "PPO_preTrained" + '/' + env_name + '/' - checkpoint_path = directory + "PPO_{}_{}_{}.pth".format(env_name, random_seed, run_num_pretrained) - print("loading network from : " + checkpoint_path) - + + # Initialize PPO agent + ppo_agent = PPO(state_dim, action_dim, lr_actor, lr_critic, gamma, + K_epochs, eps_clip, has_continuous_action_space) + # Load pretrained model - ppo_agent.load(checkpoint_path) - - print("--------------------------------------------------------------------------------------------") - - test_running_reward = 0 + checkpoint_path = os.path.join("PPO_preTrained", env_name, "PPO_RocketLanding_0_0.pth") + if not os.path.exists(checkpoint_path): + print(f"\nError: No checkpoint found at {checkpoint_path}") + print("Please ensure you have trained the model first.") + return + + print(f"\nLoading model from: {checkpoint_path}") + try: + ppo_agent.load(checkpoint_path) + print("Model loaded successfully!") + except Exception as e: + print(f"Error loading model: {e}") + return + print("\nStarting testing...") + test_running_reward = 0 + successful_landings = 0 for ep in range(1, total_test_episodes + 1): ep_reward = 0 state = env.reset() - + for t in range(1, max_ep_len + 1): - action = ppo_agent.select_action(state) - state, reward, done, _ = env.step(action) + # Select action + with torch.no_grad(): # Faster inference + action = ppo_agent.select_action(state) + + # Take step + next_state, reward, done, _ = env.step(action) ep_reward += reward - - if render: - env.render(window_name="Rocket Test", wait_time=frame_delay) # Adjust for Rocket render method - + + # Render if enabled + if config['render'] and t % 2 == 0: # Skip frames for speed + env.render(window_name="Rocket Test", + wait_time=config['frame_delay']) + if done: + # Check landing conditions + x_pos, y_pos = next_state[0], next_state[1] + vx, vy = next_state[2], next_state[3] + theta = next_state[4] + + # Stricter landing conditions + if (reward > 500 and # High reward + abs(x_pos) < 20.0 and # Close to center + abs(vx) < 5.0 and abs(vy) < 5.0 and # Low velocity + abs(theta) < 0.1): # Nearly vertical + + successful_landings += 1 + print(f"\nPerfect landing! Reward: {reward:.2f}") + celebrate_landing() break - - # Clear PPO agent buffer after each episode + + state = next_state + ppo_agent.buffer.clear() - test_running_reward += ep_reward - print('Episode: {} \t\t Reward: {}'.format(ep, round(ep_reward, 2))) - + print(f'Episode: {ep} \t\t Reward: {round(ep_reward, 2)}') + env.close() - + cv2.destroyAllWindows() + print("============================================================================================") - avg_test_reward = test_running_reward / total_test_episodes - print("average test reward : " + str(round(avg_test_reward, 2))) - + success_rate = (successful_landings / total_test_episodes) * 100 + print(f"Average test reward: {round(avg_test_reward, 2)}") + print(f"Successful landings: {successful_landings}/{total_test_episodes} ({success_rate:.1f}%)") print("============================================================================================") - if __name__ == '__main__': - test() + test() \ No newline at end of file diff --git a/train.py b/train.py index 9ce0018..54bdec6 100644 --- a/train.py +++ b/train.py @@ -1,8 +1,6 @@ import os -import time from datetime import datetime - -import torch +import utils import numpy as np from PPO import PPO # Assuming PPO is your policy class @@ -10,196 +8,183 @@ import matplotlib.pyplot as plt +def get_latest_checkpoint(directory, env_name): + """Find the latest checkpoint in the directory.""" + if not os.path.exists(directory): + return None, 0, 0 + + files = [f for f in os.listdir(directory) if f.startswith(f"PPO_{env_name}")] + if not files: + return None, 0, 0 + + # Extract run numbers and find the latest + runs = [] + for f in files: + try: + # Format: PPO_RocketLanding_0_13.pth + parts = f.split('_') + seed, run = int(parts[-2]), int(parts[-1].split('.')[0]) + runs.append((seed, run, f)) + except: + continue + + if not runs: + return None, 0, 0 + + # Get the latest run + latest = max(runs, key=lambda x: x[1]) + return os.path.join(directory, latest[2]), latest[0], latest[1] + +def load_training_state(log_dir, env_name, run_num): + """Load the previous training state from logs.""" + log_file = os.path.join(log_dir, f'PPO_{env_name}_log_{run_num}.csv') + if not os.path.exists(log_file): + return 0, 0, [], 0 + + try: + data = np.genfromtxt(log_file, delimiter=',', skip_header=1) + if len(data) == 0: + return 0, 0, [], 0 + + last_episode = int(data[-1, 0]) + last_timestep = int(data[-1, 1]) + rewards = data[:, 2].tolist() + return last_episode, last_timestep, rewards, run_num + except: + return 0, 0, [], 0 + + ################################### Training ################################### def train(): print("============================================================================================") + # Get training configuration first + config = utils.get_training_config() + ####### initialize environment hyperparameters ###### env_name = "RocketLanding" - task = 'landing' # 'hover' or 'landing' - - render = True - - has_continuous_action_space = False # Discrete action space for Rocket - - max_ep_len = 1000 # Max timesteps in one episode - max_training_timesteps = int(6e6) # Break training loop if timeteps > max_training_timesteps - - print_freq = max_ep_len * 10 # Print avg reward in the interval (in num timesteps) - log_freq = max_ep_len * 2 # Log avg reward in the interval (in num timesteps) - save_model_freq = int(1e5) # Save model frequency (in num timesteps) - ##################################################### - - ################ PPO hyperparameters ################ - update_timestep = max_ep_len * 4 # Update policy every n timesteps - K_epochs = 80 # Update policy for K epochs in one PPO update - eps_clip = 0.2 # Clip parameter for PPO - gamma = 0.99 # Discount factor - lr_actor = 0.0003 # Learning rate for actor network - lr_critic = 0.001 # Learning rate for critic network - random_seed = 0 # Set random seed if required (0 = no random seed) - ##################################################### - - print("training environment name : " + env_name) + max_ep_len = 1000 + max_training_timesteps = int(6e6) + print_freq = max_ep_len * 10 + log_freq = max_ep_len * 2 + save_model_freq = int(1e5) # Initialize the Rocket environment - env = Rocket(max_steps=max_ep_len, task=task, rocket_type='starship') # Adjust as needed for the hover task + env = Rocket(max_steps=max_ep_len, task=config['task'], + rocket_type=config['rocket_type']) # Set state and action dimensions state_dim = env.state_dims action_dim = env.action_dims - ###################### logging ###################### - log_dir = "PPO_logs" - if not os.path.exists(log_dir): - os.makedirs(log_dir) - - log_dir = log_dir + '/' + env_name + '/' - if not os.path.exists(log_dir): - os.makedirs(log_dir) - - run_num = len(next(os.walk(log_dir))[2]) - log_f_name = log_dir + '/PPO_' + env_name + "_log_" + str(run_num) + ".csv" - print("logging at : " + log_f_name) - ##################################################### - - ################### checkpointing ################### - directory = "PPO_preTrained" - if not os.path.exists(directory): - os.makedirs(directory) - - directory = directory + '/' + env_name + '/' - if not os.path.exists(directory): - os.makedirs(directory) - - checkpoint_path = directory + "PPO_{}_{}_{}.pth".format(env_name, random_seed, run_num) - print("save checkpoint path : " + checkpoint_path) - ##################################################### - - # Initialize a PPO agent - ppo_agent = PPO(state_dim, action_dim, lr_actor, lr_critic, gamma, K_epochs, eps_clip, has_continuous_action_space) + ################ PPO hyperparameters ################ + has_continuous_action_space = False + update_timestep = max_ep_len * 4 + K_epochs = 80 + eps_clip = 0.2 + gamma = 0.99 + lr_actor = 0.0003 + lr_critic = 0.001 + + # Setup directories + directory, log_dir = utils.setup_directories() + + # Initialize PPO agent + ppo_agent = PPO(state_dim, action_dim, lr_actor, lr_critic, gamma, + K_epochs, eps_clip, has_continuous_action_space) + + # Setup training state (includes checkpoint loading if available) + i_episode, time_step, episode_rewards, run_num, log_f, checkpoint_path = utils.setup_training_state( + directory, log_dir, env_name, ppo_agent + ) # Track total training time start_time = datetime.now().replace(microsecond=0) print("Started training at (GMT) : ", start_time) - log_f = open(log_f_name, "w+") - log_f.write('episode,timestep,reward\n') - # Initialize logging variables print_running_reward = 0 print_running_episodes = 0 log_running_reward = 0 log_running_episodes = 0 + window_size = 10 - time_step = 0 - i_episode = 0 - - episode_rewards = [] - window_size = 10 # Window size for moving average and standard deviation - - # Initialize the plot for real-time updating - plt.ion() # Turn on interactive mode - fig, ax = plt.subplots() - ax.set_xlabel('Episode') - ax.set_ylabel('Reward') - ax.set_title('Training Progress') - plt.show(block=False) - window_size = 10 # Window size for moving average and standard deviation + # Setup plotting if enabled + if config['plot_realtime'] or config['save_plots']: + fig, ax = utils.setup_plotting(config) + else: + fig, ax = None, None # Training loop - while time_step <= max_training_timesteps: - state = env.reset() - current_ep_reward = 0 - - for t in range(1, max_ep_len + 1): - # Select action with policy - action = ppo_agent.select_action(state) - state, reward, done, _ = env.step(action) - - # Save reward and terminal state - ppo_agent.buffer.rewards.append(reward) - ppo_agent.buffer.is_terminals.append(done) - - time_step += 1 - current_ep_reward += reward - - if render and i_episode % 50 == 0: - env.render() - - # Update PPO agent - if time_step % update_timestep == 0: - ppo_agent.update() - - # Log to file - if time_step % log_freq == 0: - log_avg_reward = log_running_reward / log_running_episodes - log_f.write('{},{},{}\n'.format(i_episode, time_step, round(log_avg_reward, 4))) - log_running_reward, log_running_episodes = 0, 0 - - # Print average reward - if time_step % print_freq == 0: - print_avg_reward = print_running_reward / print_running_episodes - print("Episode : {} \t\t Timestep : {} \t\t Average Reward : {}".format(i_episode, time_step, round(print_avg_reward, 2))) - print_running_reward, print_running_episodes = 0, 0 - - # Save model weights - if time_step % save_model_freq == 0: - ppo_agent.save(checkpoint_path) - print("Model saved at timestep: ", time_step) - - if done: - break - - print_running_reward += current_ep_reward - print_running_episodes += 1 - log_running_reward += current_ep_reward - log_running_episodes += 1 - i_episode += 1 - - episode_rewards.append(current_ep_reward) - - # Update the plot - if len(episode_rewards) >= window_size: - # Calculate moving average and standard deviation - moving_avg = np.convolve( - episode_rewards, np.ones(window_size)/window_size, mode='valid' - ) - moving_std = np.array([ - np.std(episode_rewards[i-window_size+1:i+1]) - for i in range(window_size-1, len(episode_rewards)) - ]) - episodes = np.arange(window_size-1, len(episode_rewards)) - - # Clear the axis and redraw - ax.clear() - ax.plot(episodes, moving_avg, label='Moving Average Reward') - - # Shade the area between (mean - std) and (mean + std) - lower_bound = moving_avg - moving_std - upper_bound = moving_avg + moving_std - ax.fill_between(episodes, lower_bound, upper_bound, color='blue', alpha=0.2, label='Standard Deviation') - - # Set labels and title - ax.set_xlabel('Episode') - ax.set_ylabel('Reward') - ax.set_title('Training Progress with Variability Shading') - ax.legend() - plt.draw() - plt.pause(0.01) - else: - # For initial episodes where we don't have enough data for moving average - ax.clear() - ax.plot(range(len(episode_rewards)), episode_rewards, label='Episode Reward') - ax.set_xlabel('Episode') - ax.set_ylabel('Reward') - ax.set_title('Training Progress') - ax.legend() - plt.draw() - plt.pause(0.01) - - log_f.close() - print("Finished training at : ", datetime.now().replace(microsecond=0)) + try: + while time_step <= max_training_timesteps: + state = env.reset() + current_ep_reward = 0 + + for t in range(1, max_ep_len + 1): + # Select action with policy + action = ppo_agent.select_action(state) + state, reward, done, _ = env.step(action) + + # Save reward and terminal state + ppo_agent.buffer.rewards.append(reward) + ppo_agent.buffer.is_terminals.append(done) + + time_step += 1 + current_ep_reward += reward + + if config['render'] and i_episode % 50 == 0: + env.render() + + # Update PPO agent + if time_step % update_timestep == 0: + ppo_agent.update() + + # Log to file + if time_step % log_freq == 0: + log_avg_reward = log_running_reward / log_running_episodes + log_f.write('{},{},{}\n'.format(i_episode, time_step, round(log_avg_reward, 4))) + log_running_reward, log_running_episodes = 0, 0 + + # Print average reward + if time_step % print_freq == 0: + print_avg_reward = print_running_reward / print_running_episodes + print("Episode : {} \t\t Timestep : {} \t\t Average Reward : {}".format( + i_episode, time_step, round(print_avg_reward, 2))) + print_running_reward, print_running_episodes = 0, 0 + + # Save model weights + if time_step % save_model_freq == 0: + ppo_agent.save(checkpoint_path) + print("Model saved at timestep: ", time_step) + + if done: + break + + # Update rewards and episodes + print_running_reward += current_ep_reward + print_running_episodes += 1 + log_running_reward += current_ep_reward + log_running_episodes += 1 + i_episode += 1 + episode_rewards.append(current_ep_reward) + + # Update plot if enabled + if fig is not None and len(episode_rewards) >= window_size: + utils.update_plots(fig, ax, episode_rewards, window_size, config) + + except KeyboardInterrupt: + print("\nTraining interrupted by user") + except Exception as e: + print(f"\nError during training: {e}") + finally: + if time_step > 0: + ppo_agent.save(checkpoint_path) + print("Final model saved at: ", checkpoint_path) + if fig is not None: + plt.close('all') + log_f.close() + print("Finished training at : ", datetime.now().replace(microsecond=0)) if __name__ == '__main__': train() diff --git a/utils.py b/utils.py index 058d0f2..9a56d27 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,138 @@ import numpy as np import cv2 - +import pandas as pd +import json +import matplotlib as plt +import torch +import os +import json +from datetime import datetime + + +################ Training Management #################### +def get_training_config(): + """Get training configuration from user""" + config = {} + + print("\n====== Training Configuration ======") + + # Task and rocket selection + while True: + task = input("\nSelect task (hover/landing) [default: landing]: ").lower() + if task in ['', 'hover', 'landing']: + config['task'] = 'landing' if task == '' else task + break + print("Invalid choice. Please select 'hover' or 'landing'") + + while True: + rocket = input("\nSelect rocket type (falcon/starship) [default: starship]: ").lower() + if rocket in ['', 'falcon', 'starship']: + config['rocket_type'] = 'starship' if rocket == '' else rocket + break + print("Invalid choice. Please select 'falcon' or 'starship'") + + # Visualization preferences + config['render'] = input("\nEnable environment rendering? (y/n) [default: n]: ").lower() == 'y' + config['plot_realtime'] = input("Enable real-time plotting? (y/n) [default: y]: ").lower() != 'n' + config['save_plots'] = input("Save training plots? (y/n) [default: y]: ").lower() != 'n' + + # Training parameters + config['max_episodes'] = int(input("\nEnter maximum episodes [default: 1000]: ") or 1000) + config['save_freq'] = int(input("Save checkpoint frequency (episodes) [default: 100]: ") or 100) + + return config + +################ Checkpoint Management #################### +def find_checkpoints(directory, env_name): + """Find all available checkpoints""" + checkpoints = [] + if os.path.exists(directory): + for file in os.listdir(directory): + if file.startswith(f"PPO_{env_name}") and file.endswith(".pth"): + checkpoints.append(file) + return sorted(checkpoints) + +def load_checkpoint(directory, env_name): + """Handle checkpoint loading with user interaction""" + checkpoints = find_checkpoints(directory, env_name) + + if not checkpoints: + print("\nNo existing checkpoints found. Starting fresh training.") + return None, None + + print("\n====== Available Checkpoints ======") + for i, ckpt in enumerate(checkpoints): + print(f"{i+1}. {ckpt}") + + while True: + choice = input("\nSelect checkpoint number to load (or press Enter to start fresh): ") + if choice == "": + return None, None + try: + idx = int(choice) - 1 + if 0 <= idx < len(checkpoints): + return os.path.join(directory, checkpoints[idx]), checkpoints[idx] + except ValueError: + pass + print("Invalid choice. Please try again.") + +################ Logging Management #################### +def setup_logging(log_dir, env_name, run_num): + """Setup logging with continuation support""" + log_path = os.path.join(log_dir, f'PPO_{env_name}_log_{run_num}.csv') + + if os.path.exists(log_path): + print(f"\nFound existing log file: {log_path}") + choice = input("Continue logging to this file? (y/n) [default: n]: ").lower() + if choice == 'y': + return open(log_path, 'a'), True + + return open(log_path, 'w+'), False + +################ Plot Management #################### +def setup_plotting(config): + """Setup plotting based on configuration""" + if not config['plot_realtime'] and not config['save_plots']: + return None, None + + fig, ax = plt.subplots(figsize=(10, 6)) + ax.set_xlabel('Episode') + ax.set_ylabel('Reward') + ax.set_title('Training Progress') + + if config['plot_realtime']: + plt.ion() + plt.show(block=False) + + return fig, ax + +def update_plots(fig, ax, episode_rewards, window_size, config, save_dir=None): + """Update and optionally save plots""" + if fig is None or ax is None: + return + + if len(episode_rewards) >= window_size: + moving_avg = np.convolve(episode_rewards, np.ones(window_size)/window_size, mode='valid') + moving_std = np.array([np.std(episode_rewards[i-window_size+1:i+1]) + for i in range(window_size-1, len(episode_rewards))]) + episodes = np.arange(window_size-1, len(episode_rewards)) + + ax.clear() + ax.plot(episodes, moving_avg, label='Moving Average') + ax.fill_between(episodes, moving_avg-moving_std, moving_avg+moving_std, + alpha=0.2, label='Standard Deviation') + ax.set_xlabel('Episode') + ax.set_ylabel('Reward') + ax.set_title('Training Progress') + ax.legend() + + if config['plot_realtime']: + plt.draw() + plt.pause(0.01) + + if config['save_plots'] and save_dir: + plt.savefig(os.path.join(save_dir, f'training_progress_{datetime.now().strftime("%Y%m%d_%H%M%S")}.png')) + ################ Some helper functions... #################### def moving_avg(x, N=500): @@ -79,7 +211,7 @@ def rotation_matrix(rx=0., ry=0., rz=0.): Rz[1, 1] = np.cos(rz) # RZ * RY * RX - RotationMatrix = np.mat(Rz) * np.mat(Ry) * np.mat(Rx) + RotationMatrix = np.asmatrix(Rz) * np.asmatrix(Ry) * np.asmatrix(Rx) return np.array(RotationMatrix) @@ -109,10 +241,372 @@ def create_pose_matrix(tx=0., ty=0., tz=0., TranslationMatrix = translation_matrix(tx, ty, tz) # TranslationMatrix * RotationMatrix * ScaleMatrix - PoseMatrix = np.mat(TranslationMatrix) \ - * np.mat(RotationMatrix) \ - * np.mat(ScaleMatrix) \ - * np.mat(base_correction) + PoseMatrix = np.asmatrix(TranslationMatrix) \ + * np.asmatrix(RotationMatrix) \ + * np.asmatrix(ScaleMatrix) \ + * np.asmatrix(base_correction) return np.array(PoseMatrix) +# Add these imports at the top of utils.py +import os +import json +import matplotlib.pyplot as plt +from datetime import datetime + +################ Training Management #################### + +def get_training_config(): + """Get all training configurations from user""" + config = {} + + print("\n====== Training Configuration ======") + + # Task selection + while True: + task = input("\nSelect task (hover/landing) [default: landing]: ").lower() + if task in ['', 'hover', 'landing']: + config['task'] = 'landing' if task == '' else task + break + print("Invalid choice. Please select 'hover' or 'landing'") + + # Rocket type selection + while True: + rocket = input("Select rocket type (falcon/starship) [default: starship]: ").lower() + if rocket in ['', 'falcon', 'starship']: + config['rocket_type'] = 'starship' if rocket == '' else rocket + break + print("Invalid choice. Please select 'falcon' or 'starship'") + + # Visualization preferences + config['render'] = input("Enable environment rendering? (y/n) [default: n]: ").lower() == 'y' + config['plot_realtime'] = input("Enable real-time plotting? (y/n) [default: y]: ").lower() != 'n' + config['save_plots'] = input("Save training plots? (y/n) [default: y]: ").lower() != 'n' + + # Training parameters + try: + config['max_episodes'] = int(input("Enter maximum episodes [default: 1000]: ") or 1000) + config['save_freq'] = int(input("Save checkpoint frequency (episodes) [default: 100]: ") or 100) + except ValueError: + print("Invalid input for episodes. Using defaults.") + config['max_episodes'] = 1000 + config['save_freq'] = 100 + + return config + +def setup_directories(base_dir="PPO_preTrained", env_name="RocketLanding"): + """Create necessary directories""" + # Create base directory + if not os.path.exists(base_dir): + os.makedirs(base_dir) + + # Create environment directory + env_dir = os.path.join(base_dir, env_name) + if not os.path.exists(env_dir): + os.makedirs(env_dir) + + # Create logs directory + log_dir = os.path.join("PPO_logs", env_name) + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + return env_dir, log_dir + +def get_latest_checkpoint(directory, env_name): + """Find the latest checkpoint in the directory.""" + if not os.path.exists(directory): + return None, 0, 0 + + files = [f for f in os.listdir(directory) if f.startswith(f"PPO_{env_name}")] + if not files: + return None, 0, 0 + + runs = [] + for f in files: + try: + parts = f.split('_') + seed, run = int(parts[-2]), int(parts[-1].split('.')[0]) + runs.append((seed, run, f)) + except: + continue + + if not runs: + return None, 0, 0 + + latest = max(runs, key=lambda x: x[1]) + return os.path.join(directory, latest[2]), latest[0], latest[1] + +def setup_training_state(directory, log_dir, env_name, ppo_agent): + """Setup training state and handle checkpoint loading.""" + latest_checkpoint, checkpoint_seed, checkpoint_run = get_latest_checkpoint(directory, env_name) + + if latest_checkpoint is not None: + print(f"Found existing checkpoint: {latest_checkpoint}") + response = input("Continue from previous checkpoint? (y/n) [default: n]: ").lower() + + if response == 'y': + # Load model weights + ppo_agent.load(latest_checkpoint) + + # Load training logs + log_path = os.path.join(log_dir, f'PPO_{env_name}_log_{checkpoint_run}.csv') + if os.path.exists(log_path): + data = np.genfromtxt(log_path, delimiter=',', skip_header=1) + i_episode = int(data[-1, 0]) + time_step = int(data[-1, 1]) + rewards = data[:, 2].tolist() + else: + i_episode, time_step, rewards = 0, 0, [] + + # Setup logging + log_f = open(log_path, 'a') + + print(f"Resuming training from episode {i_episode}, timestep {time_step}") + return i_episode, time_step, rewards, checkpoint_run, log_f, latest_checkpoint + + # Start fresh training + run_num = len(next(os.walk(log_dir))[2]) + log_path = os.path.join(log_dir, f'PPO_{env_name}_log_{run_num}.csv') + log_f = open(log_path, 'w+') + log_f.write('episode,timestep,reward\n') + checkpoint_path = os.path.join(directory, f"PPO_{env_name}_0_{run_num}.pth") + + return 0, 0, [], run_num, log_f, checkpoint_path + +def setup_plotting(config): + """Setup plotting based on configuration""" + if not config['plot_realtime'] and not config['save_plots']: + return None, None + + plt.close('all') # Close any existing plots + fig, ax = plt.subplots(figsize=(10, 6)) + ax.set_xlabel('Episode') + ax.set_ylabel('Reward') + ax.set_title('Training Progress') + + if config['plot_realtime']: + plt.ion() + plt.show(block=False) + + return fig, ax + +def update_plots(fig, ax, episode_rewards, window_size, config, save_dir=None): + """Update and optionally save plots""" + if fig is None or ax is None: + return + + if len(episode_rewards) >= window_size: + moving_avg = np.convolve(episode_rewards, np.ones(window_size)/window_size, mode='valid') + moving_std = np.array([np.std(episode_rewards[i-window_size+1:i+1]) + for i in range(window_size-1, len(episode_rewards))]) + episodes = np.arange(window_size-1, len(episode_rewards)) + + ax.clear() + ax.plot(episodes, moving_avg, label='Moving Average') + ax.fill_between(episodes, moving_avg-moving_std, moving_avg+moving_std, + alpha=0.2, label='Standard Deviation') + ax.set_xlabel('Episode') + ax.set_ylabel('Reward') + ax.set_title('Training Progress') + ax.legend() + + if config['plot_realtime']: + plt.draw() + plt.pause(0.01) + + if config['save_plots'] and save_dir: + plt.savefig(os.path.join(save_dir, 'training_progress.png')) + +################ Configuration Management #################### +def save_training_config(config, directory): + """Save training configuration for reproducibility""" + config_path = os.path.join(directory, 'training_config.json') + with open(config_path, 'w') as f: + json.dump(config, f, indent=4) + +def load_training_config(directory): + """Load previous training configuration""" + config_path = os.path.join(directory, 'training_config.json') + if os.path.exists(config_path): + with open(config_path, 'r') as f: + return json.load(f) + return None + +def get_best_model_path(log_dir, env_name): + """Find the best performing model based on logs""" + best_reward = float('-inf') + best_model = None + + log_files = [f for f in os.listdir(log_dir) if f.endswith('.csv')] + for log_file in log_files: + data = pd.read_csv(os.path.join(log_dir, log_file)) + avg_reward = data['reward'].mean() + if avg_reward > best_reward: + best_reward = avg_reward + best_model = log_file.replace('log', 'model').replace('.csv', '.pth') + + return best_model if best_model else None + +################ Performance Monitoring #################### +def track_training_stats(): + """Track various training statistics""" + return { + 'best_reward': float('-inf'), + 'best_episode': 0, + 'running_avg': [], + 'episode_lengths': [], + 'success_rate': [], + 'crash_rate': [] + } + +def update_training_stats(stats, reward, episode_length, success, crash): + """Update training statistics""" + stats['running_avg'].append(reward) + stats['episode_lengths'].append(episode_length) + stats['success_rate'].append(1 if success else 0) + stats['crash_rate'].append(1 if crash else 0) + + if reward > stats['best_reward']: + stats['best_reward'] = reward + stats['best_episode'] = len(stats['running_avg']) + + return stats + +################ Plot Management #################### +def setup_training_plots(plot_config): + """Setup multiple plots for training visualization""" + if not plot_config['enabled']: + return None + + figs = {} + figs['reward'] = plt.figure(figsize=(10, 5)) + figs['success_rate'] = plt.figure(figsize=(10, 5)) + figs['episode_length'] = plt.figure(figsize=(10, 5)) + + return figs + +def update_training_plots(figs, stats, save_dir=None): + """Update all training plots""" + if not figs: + return + + # Update reward plot + plt.figure(figs['reward'].number) + plt.clf() + plt.plot(stats['running_avg']) + plt.title('Training Rewards') + + # Update success rate plot + plt.figure(figs['success_rate'].number) + plt.clf() + window = 100 + success_rate = np.convolve(stats['success_rate'], + np.ones(window)/window, + mode='valid') + plt.plot(success_rate) + plt.title('Success Rate') + + if save_dir: + for name, fig in figs.items(): + fig.savefig(os.path.join(save_dir, f'{name}.png')) + +################ Error Handling and Logging #################### +def setup_logger(log_dir, env_name): + """Setup logging configuration""" + import logging + + logger = logging.getLogger('rocket_training') + logger.setLevel(logging.INFO) + + # File handler + fh = logging.FileHandler(os.path.join(log_dir, f'{env_name}_training.log')) + fh.setLevel(logging.INFO) + + # Console handler + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + + # Formatter + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + logger.addHandler(fh) + logger.addHandler(ch) + + return logger + +################ Training Resume Management #################### +def save_training_state(directory, episode, timestep, stats, model): + """Save complete training state""" + state = { + 'episode': episode, + 'timestep': timestep, + 'stats': stats, + 'model_state': model.state_dict() + } + torch.save(state, os.path.join(directory, 'training_state.pth')) + +def load_training_state(directory): + """Load complete training state""" + state_path = os.path.join(directory, 'training_state.pth') + if os.path.exists(state_path): + return torch.load(state_path) + return None + +def load_existing_training(directory, log_dir, env_name, ppo_agent, checkpoint_path, checkpoint_seed, checkpoint_run): + """Load existing training state""" + try: + # Load model weights + ppo_agent.load(checkpoint_path) + + # Load training logs + log_path = os.path.join(log_dir, f'PPO_{env_name}_log_{checkpoint_run}.csv') + if os.path.exists(log_path): + data = np.genfromtxt(log_path, delimiter=',', skip_header=1) + episode = int(data[-1, 0]) + timestep = int(data[-1, 1]) + rewards = data[:, 2].tolist() + else: + episode, timestep, rewards = 0, 0, [] + + # Setup logging + log_f = open(log_path, 'a') + + return episode, timestep, rewards, checkpoint_run, log_f, checkpoint_path + except Exception as e: + print(f"Error loading existing training: {e}") + return setup_new_training(directory, log_dir, env_name) + +def setup_new_training(directory, log_dir, env_name): + """Setup new training session""" + # Get new run number + run_num = len(next(os.walk(log_dir))[2]) + + # Create new log file + log_path = os.path.join(log_dir, f'PPO_{env_name}_log_{run_num}.csv') + log_f = open(log_path, 'w+') + log_f.write('episode,timestep,reward\n') + + # Create new checkpoint path + checkpoint_path = os.path.join(directory, f"PPO_{env_name}_0_{run_num}.pth") + + return 0, 0, [], run_num, log_f, checkpoint_path + +def setup_directories(base_dir="PPO_preTrained", env_name="RocketLanding"): + """Create necessary directories""" + # Create base directory + if not os.path.exists(base_dir): + os.makedirs(base_dir) + + # Create environment directory + env_dir = os.path.join(base_dir, env_name) + if not os.path.exists(env_dir): + os.makedirs(env_dir) + + # Create logs directory + log_dir = os.path.join("PPO_logs", env_name) + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + return env_dir, log_dir \ No newline at end of file From 8cfba5694b1c2e6ee9107edc131bedab018945c0 Mon Sep 17 00:00:00 2001 From: Aravind Nagarajan <143961880+AravindXD@users.noreply.github.com> Date: Sun, 10 Nov 2024 04:23:14 +0530 Subject: [PATCH 2/5] Update vid --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d5f213..d7ab127 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ The goal is to train a reinforcement learning agent to control a rocket to either hover or land safely using the PPO algorithm. The environment simulates physics for the rocket, and the agent learns to make decisions based on the state observations to achieve the task. -https://github.com/user-attachments/assets/2bc71416-0043-4e8d-8f00-cd0d85a834ec +https://github.com/user-attachments/assets/d1977412-2de8-49c3-b0d1-f602dc28bb61 ![RewardsChart](images/rewards-timesteps.png) From 6e99e243e5eb481dc9cb9a5db39790505749b526 Mon Sep 17 00:00:00 2001 From: Aravind Nagarajan <143961880+AravindXD@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:02:51 +0530 Subject: [PATCH 3/5] fix: Rocket close error --- test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test.py b/test.py index 3c7b3ba..74d3a7a 100644 --- a/test.py +++ b/test.py @@ -171,7 +171,6 @@ def test(): test_running_reward += ep_reward print(f'Episode: {ep} \t\t Reward: {round(ep_reward, 2)}') - env.close() cv2.destroyAllWindows() print("============================================================================================") From 67ba6b54eb8e35bc04ffae1b3917404c5c342cdd Mon Sep 17 00:00:00 2001 From: Aravind Nagarajan <143961880+AravindXD@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:31:43 +0530 Subject: [PATCH 4/5] fix: test.py get latest checkpoint and better threshold for PERFECT landing --- test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test.py b/test.py index 74d3a7a..7685d39 100644 --- a/test.py +++ b/test.py @@ -5,6 +5,7 @@ import torch from PPO import PPO from rocket import Rocket +import re def create_confetti_particles(num_particles=30): # Reduced particles """Create confetti particles""" @@ -111,13 +112,15 @@ def test(): ppo_agent = PPO(state_dim, action_dim, lr_actor, lr_critic, gamma, K_epochs, eps_clip, has_continuous_action_space) - # Load pretrained model - checkpoint_path = os.path.join("PPO_preTrained", env_name, "PPO_RocketLanding_0_0.pth") - if not os.path.exists(checkpoint_path): - print(f"\nError: No checkpoint found at {checkpoint_path}") + checkpoint_dir = os.path.join("PPO_preTrained", env_name) + checkpoints = [f for f in os.listdir(checkpoint_dir) if re.match(r"PPO_RocketLanding_\d+_\d+\.pth", f)] + if not checkpoints: + print(f"\nError: No checkpoints found in {checkpoint_dir}") print("Please ensure you have trained the model first.") return - + checkpoints.sort(key=lambda x: [int(num) for num in re.findall(r'\d+', x)]) + latest_checkpoint = checkpoints[-1] + checkpoint_path = os.path.join(checkpoint_dir, latest_checkpoint) print(f"\nLoading model from: {checkpoint_path}") try: ppo_agent.load(checkpoint_path) @@ -156,7 +159,7 @@ def test(): # Stricter landing conditions if (reward > 500 and # High reward - abs(x_pos) < 20.0 and # Close to center + abs(x_pos) < 10.0 and # Close to center abs(vx) < 5.0 and abs(vy) < 5.0 and # Low velocity abs(theta) < 0.1): # Nearly vertical From 40183c722638e1f4be8d0f607e976357185c6d33 Mon Sep 17 00:00:00 2001 From: Aravind Nagarajan <143961880+AravindXD@users.noreply.github.com> Date: Mon, 11 Nov 2024 00:24:10 +0530 Subject: [PATCH 5/5] refactor: remove repetitions --- utils.py | 98 -------------------------------------------------------- 1 file changed, 98 deletions(-) diff --git a/utils.py b/utils.py index 9a56d27..ec873bb 100644 --- a/utils.py +++ b/utils.py @@ -6,41 +6,6 @@ import torch import os import json -from datetime import datetime - - -################ Training Management #################### -def get_training_config(): - """Get training configuration from user""" - config = {} - - print("\n====== Training Configuration ======") - - # Task and rocket selection - while True: - task = input("\nSelect task (hover/landing) [default: landing]: ").lower() - if task in ['', 'hover', 'landing']: - config['task'] = 'landing' if task == '' else task - break - print("Invalid choice. Please select 'hover' or 'landing'") - - while True: - rocket = input("\nSelect rocket type (falcon/starship) [default: starship]: ").lower() - if rocket in ['', 'falcon', 'starship']: - config['rocket_type'] = 'starship' if rocket == '' else rocket - break - print("Invalid choice. Please select 'falcon' or 'starship'") - - # Visualization preferences - config['render'] = input("\nEnable environment rendering? (y/n) [default: n]: ").lower() == 'y' - config['plot_realtime'] = input("Enable real-time plotting? (y/n) [default: y]: ").lower() != 'n' - config['save_plots'] = input("Save training plots? (y/n) [default: y]: ").lower() != 'n' - - # Training parameters - config['max_episodes'] = int(input("\nEnter maximum episodes [default: 1000]: ") or 1000) - config['save_freq'] = int(input("Save checkpoint frequency (episodes) [default: 100]: ") or 100) - - return config ################ Checkpoint Management #################### def find_checkpoints(directory, env_name): @@ -88,65 +53,8 @@ def setup_logging(log_dir, env_name, run_num): return open(log_path, 'a'), True return open(log_path, 'w+'), False - -################ Plot Management #################### -def setup_plotting(config): - """Setup plotting based on configuration""" - if not config['plot_realtime'] and not config['save_plots']: - return None, None - - fig, ax = plt.subplots(figsize=(10, 6)) - ax.set_xlabel('Episode') - ax.set_ylabel('Reward') - ax.set_title('Training Progress') - - if config['plot_realtime']: - plt.ion() - plt.show(block=False) - - return fig, ax - -def update_plots(fig, ax, episode_rewards, window_size, config, save_dir=None): - """Update and optionally save plots""" - if fig is None or ax is None: - return - - if len(episode_rewards) >= window_size: - moving_avg = np.convolve(episode_rewards, np.ones(window_size)/window_size, mode='valid') - moving_std = np.array([np.std(episode_rewards[i-window_size+1:i+1]) - for i in range(window_size-1, len(episode_rewards))]) - episodes = np.arange(window_size-1, len(episode_rewards)) - - ax.clear() - ax.plot(episodes, moving_avg, label='Moving Average') - ax.fill_between(episodes, moving_avg-moving_std, moving_avg+moving_std, - alpha=0.2, label='Standard Deviation') - ax.set_xlabel('Episode') - ax.set_ylabel('Reward') - ax.set_title('Training Progress') - ax.legend() - - if config['plot_realtime']: - plt.draw() - plt.pause(0.01) - - if config['save_plots'] and save_dir: - plt.savefig(os.path.join(save_dir, f'training_progress_{datetime.now().strftime("%Y%m%d_%H%M%S")}.png')) - ################ Some helper functions... #################### -def moving_avg(x, N=500): - - if len(x) <= N: - return [] - - x_pad_left = x[0:N] - x_pad_right = x[-N:] - x_pad = x_pad_left[::-1] + x + x_pad_right[::-1] - y = np.convolve(x_pad, np.ones(N) / N, mode='same') - return y[N:-N] - - def load_bg_img(path_to_img, w, h): bg_img = cv2.imread(path_to_img, cv2.IMREAD_COLOR) bg_img = cv2.cvtColor(bg_img, cv2.COLOR_BGR2RGB) @@ -248,12 +156,6 @@ def create_pose_matrix(tx=0., ty=0., tz=0., return np.array(PoseMatrix) -# Add these imports at the top of utils.py -import os -import json -import matplotlib.pyplot as plt -from datetime import datetime - ################ Training Management #################### def get_training_config():