hw_submission(吴振锋):add hw7_20230629

chapter7_tricks/mujoco_hopper_onppo_seed0/formatted_total_config.py

@@ -0,0 +1,156 @@
+from easydict import EasyDict
+
+main_config = dict(
+    exp_name='hopper_onppo_seed0',
+    env=dict(
+        manager=dict(
+            episode_num=float('inf'),
+            max_retry=5,
+            step_timeout=None,
+            auto_reset=True,
+            reset_timeout=None,
+            retry_type='reset',
+            retry_waiting_time=0.1,
+            shared_memory=True,
+            copy_on_get=True,
+            context='fork',
+            wait_num=float('inf'),
+            step_wait_timeout=None,
+            connect_timeout=60,
+            reset_inplace=False,
+            cfg_type='SyncSubprocessEnvManagerDict',
+            type='subprocess',
+        ),
+        stop_value=4000,
+        n_evaluator_episode=10,
+        action_clip=False,
+        delay_reward_step=0,
+        replay_path=None,
+        save_replay_gif=False,
+        replay_path_gif=None,
+        action_bins_per_branch=None,
+        cfg_type='MujocoEnvDict',
+        env_id='Hopper-v3',
+        norm_obs={'use_norm': False},
+        norm_reward={'use_norm': False},
+        collector_env_num=8,
+        evaluator_env_num=10,
+    ),
+    policy=dict(
+        model=dict(
+            obs_shape=11,
+            action_shape=3,
+            action_space='continuous',
+        ),
+        learn=dict(
+            learner=dict(
+                train_iterations=1000000000,
+                dataloader=dict(
+                    num_workers=0,
+                ),
+                log_policy=True,
+                hook=dict(
+                    load_ckpt_before_run='',
+                    log_show_after_iter=100,
+                    save_ckpt_after_iter=10000,
+                    save_ckpt_after_run=True,
+                ),
+                cfg_type='BaseLearnerDict',
+            ),
+            epoch_per_collect=10,
+            batch_size=320,
+            learning_rate=0.0003,
+            value_weight=0.5,
+            entropy_weight=0.001,
+            clip_ratio=0.2,
+            adv_norm=True,
+            value_norm=True,
+            ppo_param_init=True,
+            grad_clip_type='clip_norm',
+            grad_clip_value=0.5,
+            ignore_done=False,
+            update_per_collect=1,
+        ),
+        collect=dict(
+            collector=dict(
+                deepcopy_obs=False,
+                transform_obs=False,
+                collect_print_freq=100,
+                cfg_type='SampleSerialCollectorDict',
+                type='sample',
+            ),
+            unroll_len=1,
+            discount_factor=0.99,
+            gae_lambda=0.95,
+            n_sample=3200,
+        ),
+        eval=dict(
+            evaluator=dict(
+                eval_freq=500,
+                render={'render_freq': -1, 'mode': 'train_iter'},
+                cfg_type='InteractionSerialEvaluatorDict',
+                stop_value=4000,
+                n_episode=10,
+            ),
+        ),
+        other=dict(
+            replay_buffer=dict(
+                type='advanced',
+                replay_buffer_size=4096,
+                max_use=float('inf'),
+                max_staleness=float('inf'),
+                alpha=0.6,
+                beta=0.4,
+                anneal_step=100000,
+                enable_track_used_data=False,
+                deepcopy=False,
+                thruput_controller=dict(
+                    push_sample_rate_limit=dict(
+                        max=float('inf'),
+                        min=0,
+                    ),
+                    window_seconds=30,
+                    sample_min_limit_ratio=1,
+                ),
+                monitor=dict(
+                    sampled_data_attr=dict(
+                        average_range=5,
+                        print_freq=200,
+                    ),
+                    periodic_thruput=dict(
+                        seconds=60,
+                    ),
+                ),
+                cfg_type='AdvancedReplayBufferDict',
+            ),
+        ),
+        on_policy=True,
+        cuda=True,
+        multi_gpu=False,
+        bp_update_sync=True,
+        traj_len_inf=False,
+        priority=False,
+        priority_IS_weight=False,
+        recompute_adv=True,
+        action_space='continuous',
+        nstep_return=False,
+        multi_agent=False,
+        transition_with_policy_data=True,
+        cfg_type='PPOCommandModePolicyDict',
+    ),
+)
+main_config = EasyDict(main_config)
+main_config = main_config
+create_config = dict(
+    env=dict(
+        type='mujoco',
+        import_names=['dizoo.mujoco.envs.mujoco_env'],
+    ),
+    env_manager=dict(
+        cfg_type='SyncSubprocessEnvManagerDict',
+        type='subprocess',
+    ),
+    policy=dict(type='ppo'),
+)
+create_config = EasyDict(create_config)
+create_config = create_config

chapter7_tricks/mujoco_hopper_onppo_seed0/git_log.txt

@@ -0,0 +1,61 @@
+d90e789
+
+commit d90e78973c8b9b77e1d7175b59d1a50eca358f4c
+Author: niuyazhe <niuyazhe@sensetime.com>
+Date:   Wed Jun 21 19:19:34 2023 +0800
+
+    polish(nyz): polish ch7 supp details
+
+ chapter7_tricks/chapter7_supp_entropy.pdf | Bin 1198004 -> 1224744 bytes
+ 1 file changed, 0 insertions(+), 0 deletions(-)
+
+commit a3659ba60a5c7e2cc092c6cac6e6f48db8da25f1
+Author: niuyazhe <niuyazhe@sensetime.com>
+Date:   Wed Jun 21 19:02:16 2023 +0800
+
+    feature(nyz): add ch7 entropy supp
+
+ README.md                                 |   2 +-
+ chapter7_tricks/chapter7_supp_entropy.pdf | Bin 0 -> 1198004 bytes
+ 2 files changed, 1 insertion(+), 1 deletion(-)
+
+commit bf18d93d6e8f5f06e6850b912ef2662ae3e92297
+Author: niuyazhe <niuyazhe@sensetime.com>
+Date:   Sun Jun 11 18:35:31 2023 +0800
+
+    polish(nyz): polish grad clip codes and move other tricks
+
+ README.md                                          |  4 +--
+ .../{ppo_dual_clip_code_demo.py => dual_clip.py}   |  0
+ chapter7_tricks/{gae_code_demo.py => gae.py}       |  0
+ .../grad_clip_norm.py                              | 30 ++++++++++++----------
+ .../grad_clip_norm_zh.py                           | 21 +++++++--------
+ .../grad_clip_value.py                             | 28 +++++++++++---------
+ .../grad_ignore.py                                 | 28 ++++++++++----------
+ .../{ppo_value_clip_code_demo.py => value_clip.py} |  0
+ 8 files changed, 61 insertions(+), 50 deletions(-)
+
+commit 424c2b2fa5ee71820b71ffdf3d4c0fe38432d27e
+Author: karroyan <karroyan@sjtu.edu.cn>
+Date:   Sun Jun 11 16:24:55 2023 +0800
+
+    feature(lxy): add clip_grad_value and grad_ignore (#78)
+
+ chapter1_overview/clip_grad_value.py |  53 +++++++++++++++
+ chapter1_overview/grad_ignore.py     | 122 +++++++++++++++++++++++++++++++++++
+ 2 files changed, 175 insertions(+)
+
+commit 439d510a50e96ad38d7a7b994aad4b2d3e96f7d7
+Author: Wang hl <59834623+kxzxvbk@users.noreply.github.com>
+Date:   Sun Jun 11 04:24:01 2023 -0400
+
+    feature(whl): add demo code for GAE and dual/value clip (#77)
+
+    * feature(whl): Add demo code for GAE and dual/value clip
+
+    * polish
+
+ chapter7_tricks/gae_code_demo.py            | 70 +++++++++++++++++++++++++++++
+ chapter7_tricks/ppo_dual_clip_code_demo.py  | 51 +++++++++++++++++++++
+ chapter7_tricks/ppo_value_clip_code_demo.py | 48 ++++++++++++++++++++
+ 3 files changed, 169 insertions(+)