From d7fdb4e04f392f5d0ac37673b5bae3739e3b9105 Mon Sep 17 00:00:00 2001 From: huangshiyu Date: Tue, 5 Sep 2023 17:23:36 +0800 Subject: [PATCH 1/2] add gym_pybullet_drones env --- Gallery.md | 66 +++++++------- README.md | 1 + README_zh.md | 1 + docs/images/drone.png | Bin 0 -> 18935 bytes examples/gym_pybullet_drones/README.md | 8 ++ examples/gym_pybullet_drones/ppo.yaml | 10 +++ examples/gym_pybullet_drones/test_env.py | 63 ++++++++++++++ examples/gym_pybullet_drones/train_ppo.py | 90 ++++++++++++++++++++ openrl/envs/common/registration.py | 7 +- openrl/envs/gym_pybullet_drones/__init__.py | 65 ++++++++++++++ 10 files changed, 278 insertions(+), 33 deletions(-) create mode 100644 docs/images/drone.png create mode 100644 examples/gym_pybullet_drones/README.md create mode 100644 examples/gym_pybullet_drones/ppo.yaml create mode 100644 examples/gym_pybullet_drones/test_env.py create mode 100644 examples/gym_pybullet_drones/train_ppo.py create mode 100644 openrl/envs/gym_pybullet_drones/__init__.py diff --git a/Gallery.md b/Gallery.md index a29b2ad5..8d18008d 100644 --- a/Gallery.md +++ b/Gallery.md @@ -1,6 +1,7 @@ ## Gallery -In order to facilitate users' familiarity with the framework, we provide more examples and demos of using OpenRL in Gallery. +In order to facilitate users' familiarity with the framework, we provide more examples and demos of using OpenRL in +Gallery. Users are also welcome to contribute their own training examples and demos to the Gallery. @@ -19,7 +20,6 @@ Users are also welcome to contribute their own training examples and demos to th ![offpolicy](https://img.shields.io/badge/-offpolicy-blue) (Off-policy RL) - ![discrete](https://img.shields.io/badge/-discrete-brightgreen) (Discrete Action Space) ![continuous](https://img.shields.io/badge/-continous-green) (Continuous Action Space) @@ -32,21 +32,21 @@ Users are also welcome to contribute their own training examples and demos to th
-| Algorithm | Tags | Refs | -|:----------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------:| -| [PPO](https://arxiv.org/abs/1707.06347) | ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/cartpole/) | -| [PPO-continuous](https://arxiv.org/abs/1707.06347) | ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/mujoco/) | -| [Dual-clip PPO](https://arxiv.org/abs/1912.09729) | ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/cartpole/) | -| [MAPPO](https://arxiv.org/abs/2103.01955) | ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | -| [JRPO](https://arxiv.org/abs/2302.07515) | ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | -| [GAIL](https://arxiv.org/abs/1606.03476) | ![offline](https://img.shields.io/badge/-offlineRL-darkblue) | [code](./examples/gail/) | -| [Behavior Cloning](http://www.cse.unsw.edu.au/~claude/papers/MI15.pdf) | ![offline](https://img.shields.io/badge/-offlineRL-darkblue) | [code](./examples/behavior_cloning/) | -| Self-Play | ![selfplay](https://img.shields.io/badge/-selfplay-blue) ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/selfplay/) | -| [DQN](https://arxiv.org/abs/1312.5602) | ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![value](https://img.shields.io/badge/-value-orange) ![offpolicy](https://img.shields.io/badge/-offpolicy-blue) | [code](./examples/toy_env) [code](./examples/gridworld/) | -| [MAT](https://arxiv.org/abs/2205.14953) | ![MARL](https://img.shields.io/badge/-MARL-yellow) ![Transformer](https://img.shields.io/badge/-Transformer-blue) | [code](./examples/mpe/) | -| [VDN](https://arxiv.org/abs/1706.05296) | ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | -| [SAC](https://arxiv.org/abs/1812.05905) | ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/toy_env) [code](./examples/sac/) | -| [DDPG](https://arxiv.org/abs/1509.02971) | ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/toy_env) [code](./examples/ddpg/) | +| Algorithm | Tags | Refs | +|:----------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:----------------------------------------------------------:| +| [PPO](https://arxiv.org/abs/1707.06347) | ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/cartpole/) | +| [PPO-continuous](https://arxiv.org/abs/1707.06347) | ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/mujoco/) | +| [Dual-clip PPO](https://arxiv.org/abs/1912.09729) | ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/cartpole/) | +| [MAPPO](https://arxiv.org/abs/2103.01955) | ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | +| [JRPO](https://arxiv.org/abs/2302.07515) | ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | +| [GAIL](https://arxiv.org/abs/1606.03476) | ![offline](https://img.shields.io/badge/-offlineRL-darkblue) | [code](./examples/gail/) | +| [Behavior Cloning](http://www.cse.unsw.edu.au/~claude/papers/MI15.pdf) | ![offline](https://img.shields.io/badge/-offlineRL-darkblue) | [code](./examples/behavior_cloning/) | +| Self-Play | ![selfplay](https://img.shields.io/badge/-selfplay-blue) ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/selfplay/) | +| [DQN](https://arxiv.org/abs/1312.5602) | ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![value](https://img.shields.io/badge/-value-orange) ![offpolicy](https://img.shields.io/badge/-offpolicy-blue) | [code](./examples/toy_env) [code](./examples/gridworld/) | +| [MAT](https://arxiv.org/abs/2205.14953) | ![MARL](https://img.shields.io/badge/-MARL-yellow) ![Transformer](https://img.shields.io/badge/-Transformer-blue) | [code](./examples/mpe/) | +| [VDN](https://arxiv.org/abs/1706.05296) | ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | +| [SAC](https://arxiv.org/abs/1812.05905) | ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/toy_env) [code](./examples/sac/) | +| [DDPG](https://arxiv.org/abs/1509.02971) | ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/toy_env) [code](./examples/ddpg/) |
@@ -54,19 +54,21 @@ Users are also welcome to contribute their own training examples and demos to th
-| Environment/Demo | Tags | Refs | -|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-------------------------------:| -| [MuJoCo](https://github.com/deepmind/mujoco)
| ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/mujoco/) | -| [CartPole](https://gymnasium.farama.org/environments/classic_control/cart_pole/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/cartpole/) | -| [MPE: Simple Spread](https://pettingzoo.farama.org/environments/mpe/simple_spread/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | -| [StarCraft II](https://github.com/oxwhirl/smac)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/smac/) | -| [Chat Bot](https://openrl-docs.readthedocs.io/en/latest/quick_start/train_nlp.html)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![NLP](https://img.shields.io/badge/-NLP-green) ![Transformer](https://img.shields.io/badge/-Transformer-blue) | [code](./examples/nlp/) | -| [Atari Pong](https://gymnasium.farama.org/environments/atari/pong/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![image](https://img.shields.io/badge/-image-red) | [code](./examples/atari/) | -| [PettingZoo: Tic-Tac-Toe](https://pettingzoo.farama.org/environments/classic/tictactoe/)
| ![selfplay](https://img.shields.io/badge/-selfplay-blue) ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/selfplay/) | -| [DeepMind Control](https://shimmy.farama.org/environments/dm_control/)
| ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/dm_control/) | -| [Omniverse Isaac Gym](https://github.com/NVIDIA-Omniverse/OmniIsaacGymEnvs)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/isaac/) | -| [Snake](http://www.jidiai.cn/env_detail?envid=1)
| ![selfplay](https://img.shields.io/badge/-selfplay-blue) ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/snake/) | -| [GridWorld](./examples/gridworld/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/gridworld/) | -| [Super Mario Bros](https://github.com/Kautenja/gym-super-mario-bros)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![image](https://img.shields.io/badge/-image-red) | [code](./examples/super_mario/) | -| [Gym Retro](https://github.com/openai/retro)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![image](https://img.shields.io/badge/-image-red) | [code](./examples/retro/) | +| Environment/Demo | Tags | Refs | +|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:---------------------------------------:| +| [MuJoCo](https://github.com/deepmind/mujoco)
| ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/mujoco/) | +| [CartPole](https://gymnasium.farama.org/environments/classic_control/cart_pole/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/cartpole/) | +| [MPE: Simple Spread](https://pettingzoo.farama.org/environments/mpe/simple_spread/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/mpe/) | +| [StarCraft II](https://github.com/oxwhirl/smac)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![MARL](https://img.shields.io/badge/-MARL-yellow) | [code](./examples/smac/) | +| [Chat Bot](https://openrl-docs.readthedocs.io/en/latest/quick_start/train_nlp.html)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![NLP](https://img.shields.io/badge/-NLP-green) ![Transformer](https://img.shields.io/badge/-Transformer-blue) | [code](./examples/nlp/) | +| [Atari Pong](https://gymnasium.farama.org/environments/atari/pong/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![image](https://img.shields.io/badge/-image-red) | [code](./examples/atari/) | +| [PettingZoo: Tic-Tac-Toe](https://pettingzoo.farama.org/environments/classic/tictactoe/)
| ![selfplay](https://img.shields.io/badge/-selfplay-blue) ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/selfplay/) | +| [DeepMind Control](https://shimmy.farama.org/environments/dm_control/)
| ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/dm_control/) | +| [Omniverse Isaac Gym](https://github.com/NVIDIA-Omniverse/OmniIsaacGymEnvs)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/isaac/) | +| [Snake](http://www.jidiai.cn/env_detail?envid=1)
| ![selfplay](https://img.shields.io/badge/-selfplay-blue) ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/snake/) | +| [gym-pybullet-drones](https://github.com/utiasDSL/gym-pybullet-drones)
| ![continuous](https://img.shields.io/badge/-continous-green) | [code](./examples/gym_pybullet_drones/) | +| [GridWorld](./examples/gridworld/)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) | [code](./examples/gridworld/) | +| [Super Mario Bros](https://github.com/Kautenja/gym-super-mario-bros)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![image](https://img.shields.io/badge/-image-red) | [code](./examples/super_mario/) | +| [Gym Retro](https://github.com/openai/retro)
| ![discrete](https://img.shields.io/badge/-discrete-brightgreen) ![image](https://img.shields.io/badge/-image-red) | [code](./examples/retro/) | +
\ No newline at end of file diff --git a/README.md b/README.md index af5b6fbe..4ab403eb 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Environments currently supported by OpenRL (for more details, please refer to [G - [Omniverse Isaac Gym](https://github.com/NVIDIA-Omniverse/OmniIsaacGymEnvs) - [DeepMind Control](https://shimmy.farama.org/environments/dm_control/) - [Snake](http://www.jidiai.cn/env_detail?envid=1) +- [gym-pybullet-drones](https://github.com/utiasDSL/gym-pybullet-drones) - [GridWorld](./examples/gridworld/) - [Super Mario Bros](https://github.com/Kautenja/gym-super-mario-bros) - [Gym Retro](https://github.com/openai/retro) diff --git a/README_zh.md b/README_zh.md index 93be1e1b..8b381027 100644 --- a/README_zh.md +++ b/README_zh.md @@ -93,6 +93,7 @@ OpenRL目前支持的环境（更多详情请参考 [Gallery](Gallery.md)）： - [Omniverse Isaac Gym](https://github.com/NVIDIA-Omniverse/OmniIsaacGymEnvs) - [DeepMind Control](https://shimmy.farama.org/environments/dm_control/) - [Snake](http://www.jidiai.cn/env_detail?envid=1) +- [gym-pybullet-drones](https://github.com/utiasDSL/gym-pybullet-drones) - [GridWorld](./examples/gridworld/) - [Super Mario Bros](https://github.com/Kautenja/gym-super-mario-bros) - [Gym Retro](https://github.com/openai/retro) diff --git a/docs/images/drone.png b/docs/images/drone.png new file mode 100644 index 0000000000000000000000000000000000000000..f3e5f3a0a6d8d34be6bb824124df6deca7e38131 GIT binary patch literal 18935 zcmV(?K-a&CP)Px#KTu3mMd#<}uf5c~#nhs&!=}2~;N|M4xY41y+~()z zq`TeL+vJ|7w6M9x%Fx^0;QtXEa-p>Ua&v%GR#_4i;6=+|c``Eh001BWNklH`Ks;CuZ@i(k-O;kdEz-UEGtpPiGsBP2HR+_=7MG(e^!_ooLHMIwnCr!2etNO1)h994gNQ>QW z%Wi3D=q?R$e7l7KpBI1GmT-nJzDKLtzL$0kp>~qCc7ihp34Ll$l)GokvmhQYE(CFg ze>n)+IsasejPu+uR^|8sWOMoA7&0l-c@K(aNRh2QlZ)aVpux`;euVBONBb?o6>~=} z&|Uz{krK^CZ}mx%;Bu))IPF79wHrv7y%FO1WSkkN+@qL_BL4)qCkjHKedL1@ZpDO) zEo9;n&x=SvekV^uHhXA83A32SR6(3!`H+YNgM`6LpD~mgejiPM88&HZ5z&Tvj(K(uRLrB6Eul0CD#vq0ECsOA*Q697nciB zF1hr}G>zCY2e<*^LV!2}P)_}^(FZUS6F!71aJ^%vS**Y_Q;s0a7|uIW><<4Xte#vI z@atgjCc2reu|DA}Xx&PX?ol~@C;*rO&IaldB?~nr>8|j;FlnlwT!wS@5$X}1|5zzP z{6zQ+GxNO?{^|Ih(<{INXU}}kY&6>UI3qoLte;)AmCAr?!>k`8-_{K{Bk8A*UG5i( zSVa|Ky#!ZX=wEJtP;O5y4|hC4*2D?3I37@DX8$=bF4ps=LdqeZ%d9XztIEqaGQ$ky zO%XE#SxFlSO4A5&h3wV!7ShKkC6Wt7(s{L5g-YRtYQPS28Zhg$+LFAl5llbEq;yja1AXcG1i_5 zCdk^e{*E&KbQF}GQY%(X9jw6m1T6{QCWW^bVxlC82=hS;NQCSKygab~di=hAN0TXM z3+Ih-T}Y|GGvo;!m0uPZAP%ZOSs2YbRP1dqhQ6!0q%s`JwEXQ@2-j%B4?`yTjv1?J z+6N(;=q+scptmBJrFUfP7%i4cB|I?Z*CiBRC$(3lWsf&@ZMRc@R-(IemETh+lcE5( zVO^s1JS+hLD&IaL2Jg@&YJt?Opk#~JqAe`r8cFt3+r~?oYR^x#?~r#wo|`*dZAO`* zx%3I!#aDSF2&qcE?3YQ?R_G&{5I2K7Evn_2B(vkk&_V>#eO50v+v?8;7TY%EmWpj+`1^eT zBD>A!lr%#CJ%DPmCZgW4*|?o;I@Dzo)S&kRUy-CMEvaajOTr~DaAk-Kn{+TCr)8a zy=S>14X8xCJgj-*Dd8H%b)`_AIslU>#&@kz-KWvND>bi{)lDq5FVU{J)oVjw!BcTh z!`b*O_3B=3E}TPoGgbzGu{R3E>kPGCZsiMiwO!*CsC1ormcA=!=guXRM@0cy)Pi|? zkj|7Ssu(|HV-h3n+=Y1xVJiqY<6>ir19rVFVxzCBruumq3ElbxS+i9vwNhrjY^Q42 z@j@D*RnORtl8B{HrplMpm=J4V72R#^0yi|_WnHa5pz4|tWjLiikA(1B6tg{C4WP7l z{7fbc9MTXjtsb0PX`dZO!Xcmo_!UQcYOHj*xQL&&7- z>4D|STqeTw?Z`B#gpu9FD%fIqH>5v6jCYl&rR)dNtX(gxvx^$S2&(wNL7T0|g;WfY zB0L`UZwT)*i|K5Rrx9fs>yt4AQVwnug|;7QIQz;q7VqdzMpcr&M$M@CJ`NYUS6ic;Y89f8Zdudww8|?+-a!MoLVXVE}uEo|_^5|#fa%9mgZ`KbJwdXj}rZd>lqdW$t1111d9tgOBNP z452mli0j2e(SBs}q}e08eG8$4Us@~Gq!1~7Kg{gtd7gwIOg>AH&1hkWR=$w~Kg==I z`5>Sm!5_x5uoktJne|8SfSRN(;5cUqr z4&ZJFDVg*<_D&$Rm+PeRMZj)0j|bx(+TA_vvrS-M?--z6Ooa0N@SpSCMI23Xm*TRY z?Y^Zj;J|~a3u_uwfb4hs^YG*@o@2ZDI)E!c#Yv+ipNT_@*9!M^4aw1-Qy*cFC;e!{al zk~3Mn4(dPuGl(e`VeVob;sU^)5J&iky#KB6zJuci+={v5z&+R~&02O3CC z&KJpj-yueq6bG#hf~+^@C)*fgh-5I2{`y}auH9rtxPWlQko7h}o{YHRfff|;oxS_d zk=xx^)!kO;XdOxb-G(mdY!)@T++RrSLV}{9y0X5j_-|LxZbZ$8+hsa&w4Q%Z;Wx&&7Z|V?;(CfA-^B7d9NDG0CVV# zyeYzkWR6y15+v*u^*ZO>dk90B@i;eD-um5#|9~)uS<>BG)pi!F@CQiUghuv4{FWT` z|4t{`KgXPrX@ARHN=bGsG^SRTOud1xK~=S`m_ytTwzF&Yl6gV8~LX=qirO zC?n-3E)-es*~~A~vU=V*R*cVAM?uG|t02}4;A&}1T%HGMss4!+$r=dm=HInBlpTq^{2ExDZ)W_qPljVWLzah&xWFY?PO=c?OG z2GLUu2$wH$SE;OOqrUr%l%MF0EeL|7K$^aP$LOZlb zvh71sNr|8C`OJSNIz!F}8AE6(yyLxA3X-~^))9G4FILhA3=wJ8NUIuAfwRa=)Yt7x z6UA-d#tQX1uDj;Ltgvx53$uddA7leTmN%4xjZti~Q3=@RGo{bZJU$2JT8`n+39$$=Qad%d;7d(T9-;T*21wq(5lUDRZ~P~?}$V0 zZD(HJ<3@qj`yD%Uh;ghl1^mwE^W*F5>*M2DL}tz~f}m0^qwHTtvCj;kq7Idy-7#vp z#9@7R{i(`C>H%Q{`~h7{`N2Cg@^PJmm82rS1S1QZZSd4!fj>>iW(eU&98;cO0L)yK z=NEkYcm@}{da*eiGs>hd8#aBk&=;Zkstfa(@|ut)T+k|~7Ggf*0cOaHKJy;k6~;Zb z>WI@Zi`W9P3vHSwQA)C{0SmfJ=*uYI-|`{G z34x%c(>S=orpwQ->)2iS0D?1@pP=)-s|8FOr+rW7x@@4v-V<|JHL!-#o}qj^ zqPq;_*qjCWmvR)hG|&!VnQg0B=C93ce72>W#+gLb@DoGL{)*kehZl^U=4%Q;UlpZ$mtssuh9QrBkO2}*pWY<>qMVYXC zc0E`jtR*4XUssTYZ05SyMi8Ry2=d`BuF*cmmamOG>;Q$qAtm;*=n3L_;gn%@qGu={ zN9fjU6;EoEDxS~(0OJ3i6(^z{5H!D(`x1<*RnD;$SV_AoA)R6SVx-;fpl^m0PV*zr z6L6Q+jIkugy)8to>=PD}Qx*S&OA5sB0H3hH$jU0D(ubyd?~mO`McY|Zzz^!Ib#YLi z2>F(ed+h1Y0)fvlgqFR0Zms4>i5^GOa{g@k130+j5jtsoj#*sZv3Zr0)gFn={x_pw zNEq-6p%emPc-Tcnc6H%|IH)0%B#bMGEXE4IBN!}z;7X~Qf3x|#V=Dz)k3+Od2f9~K zHdy}A+IFJ-eQlyuNoRisGjt=s7@>nH_JqoAv`5MBgt(DJG-q?poPiHzf|j2`6oM{r z=oV6x8*Pc+1u;CeK#zf#l>r>->Pax3ZcKWAH01Ui%7jyhu03OQ)9!dJ78qh}aG}tZ zvo1ie9K^imYG?9UpF@Z4@o?8q{d*vG05NI-iPPiPvOr)tWqP@Y|M9wpYULx;{8n>B z9or~Y9Z*zuN}LZ4B-BW?16tfvjr&kjJmcp})V1(;AXDuD{q>S5GWDSEa!7^zpPyrl z`2sA-9&Mg5G6e z$KPK;oDYidm}b=1mrcMQY#L4dD^Dn>9fLP0yWiN$@pA+< zr289)_0VLW7a^MbDZG}+#TiYQmAkJG`UCW50%*6-502VZr2XmDnenRMq(YsJzrTeT zXW3_G>A#;Le#=xK2R&dB??=#nMv`keA7$mu6} z^{3U_CU$-}OaxC%f);D_$t!a`fOu>7!@qoj_&d|pAu&_Ja<(aLEVByzllLJ;R`v<< zx4!n4GIis(H#8MwhN1=Jjtkv+&$Gu(3DQy%#qZ%$;aFaom-syLVFOn3kbVD+-!OhSkyh{vV|KZ;^sYYv%+A}!I&yc|ZM z9H?+_bSatMLuInOwHsA|J$PdyWiwYp$m8RR?T_KNe9%3Dv>EeSekWua?Zwvr#pye^8b z>>1+9tS7xWWr)ue_1F~4O5J{~q(|1|0d)zDN${Ay@Ogy6#l3USGy}`2b`=f~M(w<_ zjT7=znngv++Iq~WhswUM2(c#ey)!STO%qA8uts8eXm>%m1O&4OVJlq`GQWJfG}FXZ zB)t-DAa7O2m9jMxXwUCNTN^rI*~)DUt94Mli5@uDq^uxFwe9h0$)fSx2tfr3_ahq1vr~r&Wjz#zK`1T z`yuKaSDDgwZrKAT5BnR2^7%@VDAN0L@kV;gR5|k%ha{T20u=|lPB^wF5qzi0ovXIo z^UhsfN`q%4If;^Azb+6s*RC^#?QG=Qs;Oz*l)LL%t{omID`R(8#yLaWR(NA@Q@;j-i1 zTjnXT%@;5Ry1CEdt}We8!#+17nh#$}czHe_Gc`upTv%qm^)(7xNfc3SZmg<=w&Z6B z`E@GL&P+q7NF1YsE1JwhYnN|1wd>9D^Ydd_YdfFeXnXWg_^UmyJf*Cooiet8G)O!x z0|eG36F*b-)9hcb(jhbuXF?nPjr+eDO=<~R?8k!mGgeM`9sjN8fLrXu-dMlaN5KUu zA2WQ~iUoi9@ZZW5C^s$lI1o?!fC1l}y=$0u;?$I3o|)+?nqF_JI+cr!YrI9zzXrqg zAcn{JsyiO2&1RzFw2y8eu!yt<1+$aw)S41rEKyeWrM^l7uQe)kjUe`0yU1($dZ>qn zM%TipbAqy%ac(AITru5p!U%5WEyF?+FuBPNsFg=U+-{!oaTJ>YJLLZKni91h4K%?( z1!cN6nj8Ic(&<(;_NpMVucvFSfH`}*iDQ}kt1XE0HGf!2W6&ySN!Ue=3VOTjbI2N{ zB39Do_!h>dn89MN^#qEF)F$gu>`2bC)UQ;@8ga~u4L``mg#|B%3t0ul*1guMX0`4f zI%kMwHsMI!44r8Ryc*lNmiB8VAgfUq*tAWMEkb{9TfCDeY{8Qm!Lj2HYKVEC3js8( zelxCF)b`~yErG9bTT5$yJjxY5p_bWSG3E3Hh!42CarM3`;soLuHYrPi7QbZAMPkdS zqBaLWj`d|XhCXCWzc-WdAZ@hLGv-kgnJQ#uFA)1`>xo%c6c;k1&e(F3B=;;vea(|~ zWn~xHwYRxC{Y2>@b@7=G62E$~=sY}rn;oA>IlsT}rt*NCd5jFd5Qk&*jPH<6Y7w|0p{UX6BfWq{lVX8(hs zfasc)I*Rk1Z6%cJgrNJRrNpf{?6cTA7b3XkgqPwrmvX2h;-cNL0f|@tyzJyzw{diO zX%`JKS0&tt;~&O^!GdrV#k3itqwjoQsoZ9#_SnD=U%jrDO-XR?`Deb4`gv;9mPt`fo}_Q0vDfRL}Hut z=A3XP%=*bCdO3YjEW2tzbZ5Y#HnsWOGvyh51AU{xv;J3?q&gQX1qGcnj~S@a$K%l% zOvmW#6kvU6Z^0OqGKjJ>JayG(r6x$U)|v$JVb^8GVR&5`+r>mv^s6W)k$Qy9%8b)- zvO|PVJ%mQV6x}`of_aY2}Y7#*fmlXxP+Szz zq&>=vuIW|ShJdq0AmUJtTgCrCXbIx%7cfW^vYz%U7Z`U_MZCoGWziW;Gnh9C3HTON zY^vMcecOs=Kj)~8-5l*9cPem|lXEpn*AR~oR<;x1lMY4*^ATSnY-Y1zt}t4EohRR( zC`z_~TlT`Z?rNN9X=y<1?lDmfHi;3JNm^DAUnLB!Pr!S9=W!^rx?Vj5bD*ll-$;oZbvf zlVgkp9}W;EI#W#mv#4dO-Eh6a9f_ac;}G+n68Qy@?ofa1T2)XdQ(6`)g4&$?DfJXELN$IPi<#*Twmra74MUZ8CapRyK|$es_ySF{IPJ`q zV0wKVA#*2C<9QX?o8}<%$hAS-L-$hgEW}>3HA*io^xno0R9Q8af6@mlRi%#dTaz}? zZzLlb8P<5Vk^gy@3l#4{^;J;Y_6yguMf;{9lJa>iR$~nPg$2O&L!71oF}izv=dYJHgSV8(n}okgiF8qb6l1%inm)U41i*73 zu46UG#xS*U$?Vk%$ z7oqb2N-@Ls-BXFlJGKX}32SX<-a_kE3~~WDd;a}bvUo!kj5xuVio~R8oi?J3AbguO z3CpD!!5TC7t_wC=klq2Mi001BWNkllx!U7cvxR8+DG5O7!e#Ify-I_vh0d<*G(xs=xkhUpq z9FePYkb1iocmp9U$HUbXw2EsBG3KtLK$1AF*om!CUlek;Toatf)u z=e&q@A-rfigZlvEF+S-zO>V;WM1xw_Upks{ei%EPLBsd_X^_8Ai|LLP?$$vC>JIB} z9LVY1brZC386+k5*UE!9921_Y=(3@g zXN1!SA!dg4k5rr9h%f{Hxf11|EBpg%$IsVdPpneXEbD+#R=5TqGf?&A8*hKQU57Yz z2Gnx!*CB4vUWo}DgKYiZuf=(Sj5C4M{sTemf|`a|Y=v>@lX}NRvR2~!`U_P0t#?ykzUCKM?1aH<9b*jg zTXo_=h&?;Ms>@c=NV&uF0td5H7ZS9-3PX66(HMBqrv4AISn~anWZgwf9g%oy(Z#KQ z9%Br}yU2{z(T^hRA7n8zZRE$1r#gHwbHg?~azY66)=)7~G!iZM(jOGJ3gbspxPrTI zkQ_I$5qwW1bAuW|pPxhf7#i^;#GT_wV1g(aTYfcO^JCmU32@$J&U*K1EcH; zd!TVChK*BG1=&olF;VyZQ59X@EEG^&$}sjW47mB)H7mmmm8W_zk#%;#`o_N-;OWPEh?qHlf|ZE z_uSxC#*qdz5k7QHW67yq;O(t|qq00|W(3U&bNMWnqc!!VFr|yy&XX(OL+K_W@x4bU&zwEV$#p;l%#hD+rdS9cIqiz(a~ zAeE*W9A+Z z+|>vpRU+5PyQdnPB9U5+pbC$@@rGQF6w=rV_*9b_Cv}8bRx@QO>T4QD-^aWx#QoLYYUO23G$%RS;*G>+<7ZiIP$w?5jf0%FP6t6N zbe={Yt5BHYbSs>Di?2I|=yDs_qanCZ7p}CqZIx2ErRHGtD)0%MG^JtYyeP*WQyHX{ z*SM;jDg26nnd=Hn{w(sJ^?x1Z?(iGDs;@_` z81fxTd>bgo$e==|^vcZo8rvTAH5liNn%M27#C2H1^?#~1l_hb}eF-%#I)MeJnv}Uw zk76~g3^JV+(&A&phDd3Jx;4@nrYy;y!e|wLKYO+a^Cg*5uLh(ilr$zl^ysbJL8R)C zbBLi_%&8f})NxVrW6jzTRZDBy8B0gfSJH5CO>-oyZd`L4Ocv(Hk#U1rPPRiVYX=MU zk=1FD6)7+-zLGR+4NLCbh5qY|o3wC~aDv(p`2_DEEn%eQl`66zyzf_HF3=1u^}n~q~;%fM*wx-w5|ohaUUWoir5 zRVs1}bS$cc?=i}YEoE7J)|HanvPV0onZZH9-nAJk z^g5dXZ~Orf*EPajjhvHXHW?w6ZCdxK6|1k8^}R;;YH3{Ws&6{R#Z)dfwZSSy_1R!yZj?{YgE z>cSMGyDS9;ov0>J=Hi;ybZO1vy3)a>tDZPx^lJBPK<-R4UeWzB92|Y|)~8Nwq=Nf! zu$j9s>9eDm8-sg-fyF{NI|-0lbPca#FIil_OE;t&YU`KL09Ty7G!ag;G1f*xDhJ+& z*2}1cVK|s8%G|_=h>WR1Lf)s?@tO+6d}*nY+m08`>nhr59XTA*fl&oN8+1Qz4yxuRfSzwPC$4~Ev^Xqg*&?QIhA&S;YmDaeU?Rs zM~(Ky-W?o-*a;(3{iS!g%tIX&ZSyF~{n06g%PdmU9VU!gVAtM_4F%5Q;}U{<*M$?E zv6ot$Q{U=ip-48u)4@V)1eC4QMIy8nSiew+`}3wQ*Vhv#%JYsY#8A?iq_HW%4aQH< zi@lQmcgxbnix5@j+*%6$eeV79-@irPsl9)7N~k<^7#!kqV~O85=iS zpeN6~8w|MQ%VIr&lcem@B(5f;sE;kZl(A>KYTz=Ho zFBvzZ-mk@X?D|rghEIe7jOCpWa-6pTa zv+V(jzt=tuKB0EvQ+;~${KXpUt;HBq%a8EusMZl6iOKYCmjoI3qqVqP4r4ys5ag8$ zVTcG6H!c*N?rOlwkZ^KX7`7lfd&e06US#thc?WEtG-2N@#X>F4SM20gAI`lBmFart zkUTMXAQP=`UIi_KG}4a7Wz!MWQ%L4Q!(b%>=O6x3vKUqLvJue>f?TU$>f9>btIVl| zq66)tQ0aV)*h7zCVh)5nW|_OF9@T$7$aO8|6OANI9G=B>b#4IR=wNL6f8;tna-Lq6 zT}*U6hmabJP|^$bl%!qQG>ot4x$HOp$NFOG`{dK<7dp7O-&1ZL19<5uu}Aw8QiZX> z4L-^tp5}=3_3)?3n7avN>U}}qaTDZJ$$)qBU$4c@4Ai4Mgq^07imeSs6+^t+5K-1{ z(@Y94sM3YkY?8c^S@%TR%et0Lw3D5tLTdNh7m``dVy>Q>$c&-6n9C)qgS;81rwT9> z?$HqA-G#I4!XC^LJ&Mc$*k{7w^i>42=MMR8`QCn}f%&sqOh)&4hn>lfcXM&BY~RqG zLzsc}tUl7GU2$r|rq_ueV^RzZ@ijDej2BFaB?rrIfA4!d58BvOhcHL_NRo$}a ztF+jkLvhZFJA_JhDiY!zgJPjw$!zG=1m-4#*NHpmE?Gh>_slS^(T=MX!=>@svn$(Eg`#X8Gl=V~JvA@M@DX#dJ@h-xJLClhq8-x@cR)<@oeK9n$0y?NWfaAB zr`6Qjyn-PVix;J@MMUPdQ5LKlMiOXqzkQ!>MnP>0DU&)DzsDEP4~08qW81N}MpjjW zsMzSigbmTWp9_LGdSy#+wM?G$)a_wd1GBrjm_qzrhmO`VnY1d)5zGakp9~!&aWY6 za4qwQvFl@FTq;FEI<@1SpGPsCkvLy5X#M|yVh_UEvcea;>-Jgo)}6C`Y?HK2;rxY(ju`fk_th;~&UVka!lCmV++ax9$d5ZZiirZdv{1kz5O8){B z&(E6JyJXwBFpk?|{E4F86dlUTkR{!sxq{kt-t?bG@$C7JtyHm%d!q|$STp^7{!m!> ziri_F*57Of-Sow4W#Bn5=2?N6X&?8&BM0evzt6f-qqZuQR`pk zi`UkmrJq=%+}S)N@WzDH0eSMAR3AVwyT_axPhf2F_8m7tXB3UnL6Z+Sxnw(Fns&$O z)|^>6BhOyJ@+~VilB2+!2a#09(U5&7=n*K!s}oCkWU@vAW9J(7%>*JLgTsqgy-;R! z6zlnz@|})?9Lb{DpgpYHzf*)zzBua5r^#8=gm@gC-X*@Zya;hi2N_Of2q^ z17c^ziN)7Z?0e%&CHS4s}1mCuaE|0v6`%$y)<2P)~WfXz*+N-JWPRlmZ zjb+LeUX9Ds%*N#6VY#QfD5m#2?L@`|sp0wKzJkao(l)MFryEWKPn)6GWQ`)+Mac%{ zl^MhA^PNeXbnc#Y$~_VJmr)#qd@OjQYcQ}?5-5Iu3w=;=LJ2_>nmV2klyqXb)4VKb8)bcdL0!CZXea{xC}tnZH-EK- z(WGGwW6Bd_MeBqZsvMRrzGFrATne$-?WiCjd%jbZnm~aLJo~ zqlsOBi`hhMl?@@TIkrJ)Yz&B-sI_tLunyC0_~e4R6-ozEcYQGkr13SYw+#*YO^zaF zV~Y+MZ;osB{XN?tfnr7*LL%;J<$XPZVsi096oYsV(ME6d|7aY-S?4ZcS8#mYp+gT~| zunbjRpgq%R6RzBwEGQ*!H;)0WhnepT7!U3=~pZYGK7`q;e<@MRjT8 zXEO?kE{D)W%3Yy*NOp%L$joz&YmQPNYJDqH>VQMwFK+avSrn5e;RL%>O0JQSw{eV9 zkwxFXjlC%2TGjuN|8AVGS9xnxx(;nJ9WHNuTyyo>;hX3}n>QqFSm&1%nI$dEoRSvo zn}9>9k9+fzg_gP?1V%XvU+Eq5eEB^Crz7TaopW3}3{#91M93zhSOIW0uI4qe9uzc6 zw7z{5Wk*gU@7P`3GVvvPcv7b`*jQgi@tO`umY6o|ql;^q<3QB&$|x>%X?Ru04zCr- zX6HvD=~=aLPd`&=DM?H~NR6XTk5j(A9OAM9YiOb$TV1Fr0J5(T)_@a=2_i@0u$I$3 zAT&l*E2kAYodL*qoiWEK)N>RlgS_l3vTq2Ew;J)Nie9-n=BiLk2_aSI1;V{3O9Kmj z0+HW;b8&BZ35t1EdCA6DB%2F<8N7o}8>lH7HVb3psE7arw>gCgk<5zGxr;qzliuIe zd%429c9QgknN+ zLJ^ATU2+umyJ_@MoKGBw$16CNNFumg_y%Ril2QnxK)G= z+!&nTsTG74YZO<79ifZSGG{AUZJDrf)liO`EyDvYS+$J6yht6;dmr0z9QHiWvJn|5 zx^W82;#Rz<3KUNS+<=ThF^5N>4+>?qHQrB@qFlCP@4Cr1QOpkzu$py956)-iIPAMQ zuAJFlXCnNv8<8^4IAi117x#Et`|_e<&kOOHr;n|2duY~IwptW0+GN1(QA9Eq`%NB) z&D4hAnJm@ceh_^egZ^8KTo7uUKz4Yl0=lJ0QM?pD4s0Y~cJc0-9I&eJ%a?xIhk z&FOX%PmSMmG$(W&cu(WbwHl3c7F`=F+L$29`8CZ

L6JWksn$z%dnjvhW6wdvMlC zc6_oVwO6Cqeruz14ZiQB>kIeUE=HX;KJKAd4E1dr2xx&~=pnhIy-cH+tiZ0tJtZiH z6c!#2Z{Dbmz|H7a=a6L!3&j>l+cIU3LR5)olZc))C28Syww;nsS5%9>LbVBGBWCJ6 z@AMPmnim1H$NeqSybi?fvW*3{YnfQ_3W|GbY@8q>Sw!P56`5f9vAA=|)hL*lKtrJ< zmVVml8aC&7_aSo3uUrrf1l_uecb^o_Pv82L!h4`7hBdSqcTNtOy-d$*XsD?Xn znO+ad`7NB*J-lm(Gq)gqy{^4QA3cjU?x-R%k1vN}5S2JC@xc==iu4pj+dSHM3rG&r zbv*vTUB-ANb^fy8^(k*y)p%?ZB^DvfMV7iEdj>-3Q3ys#xq)ELcGH;i6HGU`95edl zB9IEDL}1M(&i{^oj$mF-mXqksA6pQ%I^`Q@;wvgr#BTw`^e9g10d@B)s@g-CVVbZE z=GOsqbwc5L6#gG=9CpWT5A~Wx?8Og{1Z~s!OrMz&T2d$gz6(^g6jtwi~CnRlB{viFx&L?WO z1lX;eqRAVIYMqk`MH>ET*rnpU+C`gO?c_paLKKwa%M z73Y{BQ+m~Wjn{5u6AM>B?6X}Mxr#d5)L@XE?NuXGCshUc&h{85wIJCz`s(+#KMt~$ zXBFZTCZ^uciV{4=F&@B3rpul}^2L(;_|-KE zagUw@wNgr64nyC=@);YSo+Iv9bXu8Az?b%m#ZmKBXGGq8u#u2ZE97#y512SOn_NwI zye?DXvKa*xMEee(FY+$@b=ADNVB^oA*q0&0S?U?8Cof)XaeEYe!&s>KL}ngwnnyWf z;C6#QZQ_)f*ec{O3N8pRlaH<7DA>Nog*Bs&Xd@%yMz8vR+r)Mw3wMR2P)|OacW^qF<$ijtclOpkmmo=^$fYXPW6kaQFVv; zf$I{B18dG1Gu?VT3jUahQxSr9g;dWNW%)$jdhj!61>Iq@aB@Rix)5+sxE|^BN5wX zJF&E@Po~8I#ZGkf7&k@UJ-0`X!j?QY@8na2|Fw5D>T#nm5U&A)f0Eko|FDN7zybr_ zw0l`+vrfJ?z4rQbC$mBl(o6-;^YQs(Q%acDjgtaVh*@{?y8}g#R`J22tU6VWk)=!i z^NWmj&fFsHacrP~an>tg_`-M!4!eH`#H{b^yA@PSgGR2Rh0?GNWNJE>jK!J8^LfYN zUXyrJ&$6>puyQ?0-FOFYx9A>MVQ`LO`0giMs}RPZ8OA(UMS1H!gSa|gd?NaA8e*Fr zdOqP+?n=O`!RQc;g`NX(nDoRIvL%|Eh)jgIu%yKV9m^@jVZ5!8AJ=!KTj!k#yITrB z(q$uxzCF!Wu@Ua!X%AowS`}=pyQDT#2o&vJ<~`DE)#d)y@3KrnWtniKFS+j~L+?+7 z7^@JMSd>P)i;Lgp7%1);-)ifsoaJ|-U?)TL4L^!VUzv#gg+%J95W`hS2R`E}QzI5N zrR*|`n**Pv4rP7X%A8t6_&5>%6wl9*fY0tdE|JI?` z$g6AfR}pCV&p=;Ixfz%39{4`Z6Cw6PLzMy9(oAoO#iu1(vt%^wxLdeLBCj4r$piS9 zcB#Ln&CA-SveBeuxY*3|HErDy4Y^7NZ)zWU->QgaxXG%cn6bJtAADzH@=G zC7M&Jc>;aqUc3c}Ig4u~z5v92QE$*Q$2gmj2I68Zg)RAY1?8uuvbk{=rz>iTOtarB zow(DDa^Hilt{pNKS5^w19w?&z0yN%kMN@1*cpHhuc-(E9IgB5glXddOKwHIE48j8{ zWNKQsKP11hHj;B4h?5>NzVW&n7%G`eOwoPeK*e8w0K{vA*o=k6#AvYe2 zJN)J((TJj6fy6-HnEWO^l-FXh_fY8c4G+Ej@_f7D`u2waj5ol0Mp|#HFs)h3>TcpL zoCeW%np{p!(%RNOeC7U15Yzip!qD3d4!yl69NQNzd)uhy@SGkSLeB?7l#uo^;}>+b z(>nUaANFYN>&0Tan#QhBGG-bNKIAT7G{SLLyNf^6Z~U4&B)_)_lV9{Rl%z0c{R&&M z=QpS!??7}v5IP^pYgEg6+sDEfA!uZp*AatM3h|Gb}kafr$9 zYVMUvt7wE6+CW5MxPVxMI32eXnX?Zn-}l{3qxP`H1cHU(YzFT&Sxm1M6ALABh(A8L zv7!;8q4RtgU*eM2RE<*-8o&HW}7^RTd94>D{xgPy(?qW6KA+5HdpYK=cKC z`6-LyUSumd$YkF95oz01A;v;ayTl@$ecaZ95A<0^0JNUpsT2&3eqwWedV)W z#I+&Da(arfVrOv~VeV@Q2V%jv3Vm9RMOhwzD& z=mMeo3FzB}>?|=Lc|)8RdA?15o4myXjSYHTzrCd%KV9yN)+2r)VDLh}QVtJG!d@+- zpfLsQ0t6)A^PsuLNUPf z`=g)u?-xG+;X(Q#Y4l(wBOaF%Wl*JOw^d>KsV0?I3 zh`nK8PlgCVQtrLW-k$&O{{bwH4)iWV?glKNRiJMd&7%t68SOaS7v2|NkyTD-hWUmp5SZ!hWMn{7JB8-qu0^`eSPPQ-4OeX0eI`8gZuCg{vuFW z_x(Fi48uNV8orJydU~E6XyXLdd(9lI_df!S;S&P$^MdkKA!c8b*`7P<9K3KT?TR?v zEG2xDa1`zdWcu0h`3-R?4#WJFtJ?@)g}3`^>@`7rlJ2q?fidJcRK9o{2;;9KAWDP8 zt0Vy!pZ?!{mS^Ai-fbtLDVzrc)d*DR_-F6h9|tj27QVI+$~2y%kPOYPK`~wX%kgu+z^e*I zyfDNFFD1VfPi|8)L*Ex-M)04khY7EIg7+alApahSJ-j!iN&t^7g!y4;;Lx+jw9HbF zNbaERQwAx%8)D1S2Txz|6$%@kV+7-6Ox_O7H-FbeX`Cee2>T)#g=ErY;oc|I7T7r> z`i@E%Cb3Zm^Ppe84-}(p1Yt}viV47f=j(ZWh$;SAFY+?%H^QF0x^o4skG@Cufb@tf zICi949ez7JVm#!&lNV?N{SPemnmKzu$~zB4-(*w&;2&;MZ|;`+Mv#yojAhQ@!M-HK zp2cwy(^{XLfxx6Y-HyHi`-(=_L8)0=ZxAplVuPp+Ee8K%9={g=dz6Zp||u}4zCF@ z%{efPsVt^zAy?qeenk;xR(WZ5kP2&fWt)N{8dEF9>W?tC_9Y>Xil?EVirE9T9!!$? z@{J==ljk=LTZp5{3Qf%WL$BG={OPe6htCk;j!w7R=nW6JIoTmpw7t2`+{EGvW55{a zUFGni5J&ntc>9|Q!X;TlM;v;SrtOj0u>DG^5W<*BI{A7K$FnWam&xyxx@PyVxUQO8 z`mLC}*;k9jHeMPNd)-G=3^cUDX$mwxh;BEu+#7z)9B|Alu9sMd2gKhnQZk3|uwP4< zOwi=G#C1aBV!~Yf7Q{CF)i8bl#K_+sii?@AEAJv3Ut@7oiM$qx6EXDhS{RGJMm9ve z5Fc*wT8ME|l|pew;xkcwapGrpQ0lH-P5&&2_mLe9lOXe`*ocE~%`p55G&W6)8=B3g zW-Gi;0YI#{4w3JG*k%%km=*2ETR$stl&j?~s98I>= 3.10 +- Fellow the installation instruction of [gym-pybullet-drones](https://github.com/utiasDSL/gym-pybullet-drones#installation). + +### Train PPO + diff --git a/examples/gym_pybullet_drones/ppo.yaml b/examples/gym_pybullet_drones/ppo.yaml new file mode 100644 index 00000000..07f59d94 --- /dev/null +++ b/examples/gym_pybullet_drones/ppo.yaml @@ -0,0 +1,10 @@ +episode_length: 500 +lr: 1e-3 +critic_lr: 1e-3 +gamma: 0.1 +ppo_epoch: 5 +use_valuenorm: true +entropy_coef: 0.0 +hidden_size: 128 +layer_N: 4 +use_recurrent_policy: true \ No newline at end of file diff --git a/examples/gym_pybullet_drones/test_env.py b/examples/gym_pybullet_drones/test_env.py new file mode 100644 index 00000000..7059ce98 --- /dev/null +++ b/examples/gym_pybullet_drones/test_env.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2023 The OpenRL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""""" +import time + +import numpy as np +import gym_pybullet_drones +import gymnasium as gym + +from openrl.envs.common import make +def test_env(): + env = gym.make("hover-aviary-v0",gui=False,record=False) + print("obs space:",env.observation_space) + print("action space:",env.action_space) + obs, info = env.reset(seed=42, options={}) + totoal_step =0 + totol_reward = 0. + while True: + obs, reward, done, truncated, info = env.step(env.action_space.sample()) + totoal_step+=1 + totol_reward+=reward + # env.render() + # time.sleep(1) + if done: + break + print("total step:",totoal_step) + print("total reward:",totol_reward) + +def test_vec_env(): + env = make("pybullet_drones/hover-aviary-v0",env_num=2,gui=False,record=False,asynchronous=True) + info,obs = env.reset(seed=0) + totoal_step = 0 + totol_reward = 0. + while True: + obs, reward, done, info = env.step(env.random_action()) + totoal_step+=1 + totol_reward+=np.mean(reward) + if np.any(done) or totoal_step>100: + break + env.close() + print("total step:", totoal_step) + print("total reward:", totol_reward) + + + + +if __name__ == '__main__': + test_env() + # test_vec_env() \ No newline at end of file diff --git a/examples/gym_pybullet_drones/train_ppo.py b/examples/gym_pybullet_drones/train_ppo.py new file mode 100644 index 00000000..c7eeea9b --- /dev/null +++ b/examples/gym_pybullet_drones/train_ppo.py @@ -0,0 +1,90 @@ +import numpy as np +import torch + +from openrl.configs.config import create_config_parser +from openrl.envs.common import make +from openrl.modules.common import PPONet as Net +from openrl.runners.common import PPOAgent as Agent + +env_name = "pybullet_drones/hover-aviary-v0" + + +def train(): + # create the neural network + cfg_parser = create_config_parser() + cfg = cfg_parser.parse_args(["--config", "ppo.yaml"]) + + # create environment, set environment parallelism to 64 + env_num = 20 + # env_num = 1 + + env = make( + env_name, + env_num=env_num, + cfg=cfg, + asynchronous=True, + env_wrappers=[], + gui=False, + ) + + net = Net(env, cfg=cfg, device="cuda" if torch.cuda.is_available() else "cpu") + # initialize the trainer + agent = Agent( + net, + ) + # start training, set total number of training steps to 100000 + agent.train(total_time_steps=1000000) + + agent.save("./ppo_agent") + env.close() + return agent + + +def evaluation(): + cfg_parser = create_config_parser() + cfg = cfg_parser.parse_args(["--config", "ppo.yaml"]) + # begin to test + # Create an environment for testing and set the number of environments to interact with to 4. Set rendering mode to group_rgb_array. + + env = make( + env_name, + env_num=1, + asynchronous=False, + env_wrappers=[], + cfg=cfg, + gui=False, + record=False, + ) + + + net = Net(env, cfg=cfg, device="cuda" if torch.cuda.is_available() else "cpu") + # initialize the trainer + agent = Agent( + net, + ) + agent.load("./ppo_agent") + + # The trained agent sets up the interactive environment it needs. + agent.set_env(env) + # Initialize the environment and get initial observations and environmental information. + obs, info = env.reset() + done = False + step = 0 + total_reward = 0.0 + while not np.any(done): + # Based on environmental observation input, predict next action. + action, _ = agent.act(obs, deterministic=True) + print("action:",action) + obs, r, done, info = env.step(action) + step += 1 + total_reward += np.mean(r) + # if step % 50 == 0: + # print(f"{step}: reward:{np.mean(r)}") + print("total step:", step) + print("total reward:", total_reward) + env.close() + + +if __name__ == "__main__": + # train() + evaluation() diff --git a/openrl/envs/common/registration.py b/openrl/envs/common/registration.py index 4b852954..0648f6fe 100644 --- a/openrl/envs/common/registration.py +++ b/openrl/envs/common/registration.py @@ -65,7 +65,12 @@ def make( id=id, env_num=env_num, render_mode=convert_render_mode, **kwargs ) else: - if id.startswith("snakes_"): + if id.startswith("pybullet_drones/"): + + from openrl.envs.gym_pybullet_drones import make_single_agent_drone_envs + env_fns = make_single_agent_drone_envs(id=id,env_num=env_num,render_mode=convert_render_mode,**kwargs) + + elif id.startswith("snakes_"): from openrl.envs.snake import make_snake_envs env_fns = make_snake_envs( diff --git a/openrl/envs/gym_pybullet_drones/__init__.py b/openrl/envs/gym_pybullet_drones/__init__.py new file mode 100644 index 00000000..1ff64fcf --- /dev/null +++ b/openrl/envs/gym_pybullet_drones/__init__.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2023 The OpenRL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""""" +import copy +from typing import Callable, List, Optional, Union + +import gymnasium as gym +from gymnasium import Env + +from openrl.envs.common import build_envs + + +def make_single_agent_drone_env(id:str, + render_mode, + disable_env_checker, + **kwargs): + import gym_pybullet_drones + prefix = "pybullet_drones/" + assert id.startswith(prefix), "id must start with pybullet_drones/" + kwargs.pop("cfg") + + env = gym.envs.registration.make(id[len(prefix):],**kwargs) + return env + + +def make_single_agent_drone_envs( + id: str, + env_num: int = 1, + render_mode: Optional[Union[str, List[str]]] = None, + **kwargs, +) -> List[Callable[[], Env]]: + from openrl.envs.wrappers import ( # AutoReset,; DictWrapper, + RemoveTruncated, + Single2MultiAgentWrapper, + ) + + env_wrappers = copy.copy(kwargs.pop("env_wrappers", [])) + env_wrappers += [ + Single2MultiAgentWrapper, + RemoveTruncated, + ] + + env_fns = build_envs( + make=make_single_agent_drone_env, + id=id, + env_num=env_num, + render_mode=render_mode, + wrappers=env_wrappers, + **kwargs, + ) + return env_fns From e1e1a79382a78351b9581d1745c7ceb3a1791b50 Mon Sep 17 00:00:00 2001 From: huangshiyu Date: Tue, 5 Sep 2023 17:24:03 +0800 Subject: [PATCH 2/2] add gym_pybullet_drones env --- examples/gym_pybullet_drones/test_env.py | 49 ++++++++++++--------- examples/gym_pybullet_drones/train_ppo.py | 3 +- openrl/envs/common/registration.py | 6 ++- openrl/envs/gym_pybullet_drones/__init__.py | 16 +++---- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/examples/gym_pybullet_drones/test_env.py b/examples/gym_pybullet_drones/test_env.py index 7059ce98..fd1eb334 100644 --- a/examples/gym_pybullet_drones/test_env.py +++ b/examples/gym_pybullet_drones/test_env.py @@ -17,47 +17,54 @@ """""" import time -import numpy as np import gym_pybullet_drones import gymnasium as gym +import numpy as np from openrl.envs.common import make + + def test_env(): - env = gym.make("hover-aviary-v0",gui=False,record=False) - print("obs space:",env.observation_space) - print("action space:",env.action_space) + env = gym.make("hover-aviary-v0", gui=False, record=False) + print("obs space:", env.observation_space) + print("action space:", env.action_space) obs, info = env.reset(seed=42, options={}) - totoal_step =0 - totol_reward = 0. + totoal_step = 0 + totol_reward = 0.0 while True: obs, reward, done, truncated, info = env.step(env.action_space.sample()) - totoal_step+=1 - totol_reward+=reward + totoal_step += 1 + totol_reward += reward # env.render() # time.sleep(1) if done: break - print("total step:",totoal_step) - print("total reward:",totol_reward) + print("total step:", totoal_step) + print("total reward:", totol_reward) + def test_vec_env(): - env = make("pybullet_drones/hover-aviary-v0",env_num=2,gui=False,record=False,asynchronous=True) - info,obs = env.reset(seed=0) + env = make( + "pybullet_drones/hover-aviary-v0", + env_num=2, + gui=False, + record=False, + asynchronous=True, + ) + info, obs = env.reset(seed=0) totoal_step = 0 - totol_reward = 0. + totol_reward = 0.0 while True: - obs, reward, done, info = env.step(env.random_action()) - totoal_step+=1 - totol_reward+=np.mean(reward) - if np.any(done) or totoal_step>100: + obs, reward, done, info = env.step(env.random_action()) + totoal_step += 1 + totol_reward += np.mean(reward) + if np.any(done) or totoal_step > 100: break env.close() print("total step:", totoal_step) print("total reward:", totol_reward) - - -if __name__ == '__main__': +if __name__ == "__main__": test_env() - # test_vec_env() \ No newline at end of file + # test_vec_env() diff --git a/examples/gym_pybullet_drones/train_ppo.py b/examples/gym_pybullet_drones/train_ppo.py index c7eeea9b..52cfdc7a 100644 --- a/examples/gym_pybullet_drones/train_ppo.py +++ b/examples/gym_pybullet_drones/train_ppo.py @@ -56,7 +56,6 @@ def evaluation(): record=False, ) - net = Net(env, cfg=cfg, device="cuda" if torch.cuda.is_available() else "cpu") # initialize the trainer agent = Agent( @@ -74,7 +73,7 @@ def evaluation(): while not np.any(done): # Based on environmental observation input, predict next action. action, _ = agent.act(obs, deterministic=True) - print("action:",action) + print("action:", action) obs, r, done, info = env.step(action) step += 1 total_reward += np.mean(r) diff --git a/openrl/envs/common/registration.py b/openrl/envs/common/registration.py index 0648f6fe..11685c5d 100644 --- a/openrl/envs/common/registration.py +++ b/openrl/envs/common/registration.py @@ -66,9 +66,11 @@ def make( ) else: if id.startswith("pybullet_drones/"): - from openrl.envs.gym_pybullet_drones import make_single_agent_drone_envs - env_fns = make_single_agent_drone_envs(id=id,env_num=env_num,render_mode=convert_render_mode,**kwargs) + + env_fns = make_single_agent_drone_envs( + id=id, env_num=env_num, render_mode=convert_render_mode, **kwargs + ) elif id.startswith("snakes_"): from openrl.envs.snake import make_snake_envs diff --git a/openrl/envs/gym_pybullet_drones/__init__.py b/openrl/envs/gym_pybullet_drones/__init__.py index 1ff64fcf..6e57d40d 100644 --- a/openrl/envs/gym_pybullet_drones/__init__.py +++ b/openrl/envs/gym_pybullet_drones/__init__.py @@ -24,24 +24,22 @@ from openrl.envs.common import build_envs -def make_single_agent_drone_env(id:str, - render_mode, - disable_env_checker, - **kwargs): +def make_single_agent_drone_env(id: str, render_mode, disable_env_checker, **kwargs): import gym_pybullet_drones + prefix = "pybullet_drones/" assert id.startswith(prefix), "id must start with pybullet_drones/" kwargs.pop("cfg") - env = gym.envs.registration.make(id[len(prefix):],**kwargs) + env = gym.envs.registration.make(id[len(prefix) :], **kwargs) return env def make_single_agent_drone_envs( - id: str, - env_num: int = 1, - render_mode: Optional[Union[str, List[str]]] = None, - **kwargs, + id: str, + env_num: int = 1, + render_mode: Optional[Union[str, List[str]]] = None, + **kwargs, ) -> List[Callable[[], Env]]: from openrl.envs.wrappers import ( # AutoReset,; DictWrapper, RemoveTruncated,