Pixel observation with recurrent SAC-Discrete

.gitignore

@@ -15,3 +15,5 @@ scripts/tmp_configs/
 
 # singularity
 *.sif
+
+third_party/

configs/credit_assign/catch/rnn.yml

@@ -0,0 +1,55 @@
+seed: 73
+cuda: 0 # use_gpu
+# RAM: ~10G
+env:
+  env_type: pomdp
+  env_name: Catch-40-v0
+
+  num_eval_tasks: 20 # num of eval episodes
+
+train:
+  # 20000*(7*n) = 5M steps
+  num_iters: 20000 # number meta-training iterates
+  num_init_rollouts_pool: 5 # before training
+  num_rollouts_per_iter: 1
+
+  num_updates_per_iter: 0.25 # 1.0
+
+  # buffer params
+  buffer_type: seq_efficient
+  buffer_size: 1e6
+  batch_size: 32 # to tune based on sampled_seq_len
+  sampled_seq_len: -1 # -1 is all, or positive integer
+  sample_weight_baseline: 0.0
+
+eval:
+  eval_stochastic: false # also eval stochastic policy
+  log_interval: 50 # num of iters
+  save_interval: -1
+  log_tensorboard: true
+
+policy:
+  separate: True
+  arch: lstm # [lstm, gru]
+  algo: sacd # only support sac-discrete
+
+  action_embedding_size: 0 # no need for catch
+  state_embedding_size: 0 # use image encoder instead
+  image_encoder: 
+    from_flattened: True
+
+  reward_embedding_size: 0
+  rnn_hidden_size: 128
+
+  dqn_layers: [128, 128]
+  policy_layers: [128, 128]
+  lr: 0.0003
+  gamma: 0.99
+  tau: 0.005
+
+  # sacd alpha
+  entropy_alpha: 0.1
+  automatic_entropy_tuning: False
+  target_entropy: None # the ratio: target_entropy = ratio * log(|A|)
+  alpha_lr: 0.0003
+

configs/credit_assign/keytodoor/HighVar/rnn.yml

@@ -0,0 +1,56 @@
+seed: 73
+cuda: 0 # use_gpu
+# RAM: ~10G
+env:
+  env_type: pomdp
+  env_name: KeytoDoor-HighVar-v0 # KeytoDoor-HighVar5-v0
+
+  num_eval_tasks: 20 # num of eval episodes
+
+train:
+  # 200000*60 = 12M steps
+  num_iters: 200000 # number meta-training iterates
+  num_init_rollouts_pool: 5 # before training
+  num_rollouts_per_iter: 1
+
+  num_updates_per_iter: 0.25 # 1.0
+
+  # buffer params
+  buffer_type: seq_efficient
+  buffer_size: 1e6
+  batch_size: 32 # to tune based on sampled_seq_len
+  sampled_seq_len: -1 # -1 is all, or positive integer
+  sample_weight_baseline: 0.0
+
+eval:
+  eval_stochastic: false # also eval stochastic policy
+  log_interval: 50 # num of iters
+  save_interval: -1
+  log_tensorboard: true
+
+policy:
+  separate: True
+  arch: lstm # [lstm, gru]
+  algo: sacd # only support sac-discrete
+
+  action_embedding_size: 0 # no need for catch
+  state_embedding_size: 0 # use image encoder instead
+  image_encoder: 
+    from_flattened: True
+    normalize_pixel: True
+
+  reward_embedding_size: 0
+  rnn_hidden_size: 128
+
+  dqn_layers: [128, 128]
+  policy_layers: [128, 128]
+  lr: 0.0003
+  gamma: 0.99
+  tau: 0.005
+
+  # sacd alpha
+  entropy_alpha: 0.1
+  automatic_entropy_tuning: False
+  target_entropy: None # the ratio: target_entropy = ratio * log(|A|)
+  alpha_lr: 0.0003
+

configs/credit_assign/keytodoor/LowVar/rnn.yml

@@ -0,0 +1,56 @@
+seed: 73
+cuda: 0 # use_gpu
+# RAM: ~10G
+env:
+  env_type: pomdp
+  env_name: KeytoDoor-LowVar-v0 # KeytoDoor-LowVar5-v0
+
+  num_eval_tasks: 20 # num of eval episodes
+
+train:
+  # 150000*60 = 9M steps
+  num_iters: 150000 # number meta-training iterates
+  num_init_rollouts_pool: 5 # before training
+  num_rollouts_per_iter: 1
+
+  num_updates_per_iter: 0.25 # 1.0
+
+  # buffer params
+  buffer_type: seq_efficient
+  buffer_size: 1e6
+  batch_size: 32 # to tune based on sampled_seq_len
+  sampled_seq_len: -1 # -1 is all, or positive integer
+  sample_weight_baseline: 0.0
+
+eval:
+  eval_stochastic: false # also eval stochastic policy
+  log_interval: 50 # num of iters
+  save_interval: -1
+  log_tensorboard: true
+
+policy:
+  separate: True
+  arch: lstm # [lstm, gru]
+  algo: sacd # only support sac-discrete
+
+  action_embedding_size: 0 # no need for catch
+  state_embedding_size: 0 # use image encoder instead
+  image_encoder: 
+    from_flattened: True
+    normalize_pixel: True
+
+  reward_embedding_size: 0
+  rnn_hidden_size: 128
+
+  dqn_layers: [128, 128]
+  policy_layers: [128, 128]
+  lr: 0.0003
+  gamma: 0.99
+  tau: 0.005
+
+  # sacd alpha
+  entropy_alpha: 0.1
+  automatic_entropy_tuning: False
+  target_entropy: None # the ratio: target_entropy = ratio * log(|A|)
+  alpha_lr: 0.0003
+

configs/credit_assign/keytodoor/SR/rnn.yml

@@ -0,0 +1,56 @@
+seed: 73
+cuda: 0 # use_gpu
+# RAM: ~10G
+env:
+  env_type: pomdp
+  env_name: KeytoDoor-SR-v0
+
+  num_eval_tasks: 20 # num of eval episodes
+
+train:
+  # 100000*90 = 9M steps
+  num_iters: 100000 # number meta-training iterates
+  num_init_rollouts_pool: 5 # before training
+  num_rollouts_per_iter: 1
+
+  num_updates_per_iter: 0.25 # 1.0
+
+  # buffer params
+  buffer_type: seq_efficient
+  buffer_size: 1e6
+  batch_size: 32 # to tune based on sampled_seq_len
+  sampled_seq_len: -1 # -1 is all, or positive integer
+  sample_weight_baseline: 0.0
+
+eval:
+  eval_stochastic: false # also eval stochastic policy
+  log_interval: 50 # num of iters
+  save_interval: -1
+  log_tensorboard: true
+
+policy:
+  separate: True
+  arch: lstm # [lstm, gru]
+  algo: sacd # only support sac-discrete
+
+  action_embedding_size: 0 # no need for catch
+  state_embedding_size: 0 # use image encoder instead
+  image_encoder: 
+    from_flattened: True
+    normalize_pixel: True
+
+  reward_embedding_size: 0
+  rnn_hidden_size: 128
+
+  dqn_layers: [128, 128]
+  policy_layers: [128, 128]
+  lr: 0.0003
+  gamma: 0.99
+  tau: 0.005
+
+  # sacd alpha
+  entropy_alpha: 0.1
+  automatic_entropy_tuning: False
+  target_entropy: None # the ratio: target_entropy = ratio * log(|A|)
+  alpha_lr: 0.0003
+

environments.yml

@@ -125,7 +125,6 @@ dependencies:
   - pcre=8.44=he6710b0_0
   - pexpect=4.8.0=pyhd3eb1b0_3
   - pickleshare=0.7.5=pyhd3eb1b0_1003
-  - pip=20.3.3=py38h06a4308_0
   - prometheus_client=0.9.0=pyhd3eb1b0_0
   - prompt-toolkit=3.0.8=py_0
   - ptyprocess=0.7.0=pyhd3eb1b0_2
@@ -149,7 +148,6 @@ dependencies:
   - scipy=1.6.0=py38h91f5cce_0
   - seaborn=0.11.1=pyhd3eb1b0_0
   - send2trash=1.5.0=pyhd3eb1b0_1
-  - setuptools=52.0.0=py38h06a4308_0
   - sip=4.19.13=py38he6710b0_0
   - six=1.15.0=py38h06a4308_0
   - sqlite=3.33.0=h62c20be_0
@@ -222,9 +220,11 @@ dependencies:
     - importlib-resources==5.4.0
     - ipdb==0.13.4
     - jsonpickle==0.9.6
+    - keras==2.8.0
     - keras-nightly==2.5.0.dev2021032900
     - keras-preprocessing==1.1.2
     - labmaze==1.0.3
+    - libclang==13.0.0
     - lockfile==0.12.2
     - lxml==4.6.2
     - markdown==3.3.3
@@ -241,13 +241,16 @@ dependencies:
     - pathspec==0.9.0
     - patsy==0.5.2
     - pillow==7.2.0
+    - pip==22.0.4
     - platformdirs==2.4.0
+    - pot==0.8.1.0
     - protobuf==3.19.4
     - psutil==5.8.0
     - py-cpuinfo==8.0.0
     - pyasn1==0.4.8
     - pyasn1-modules==0.2.8
     - pybullet==3.1.0
+    - pycolab==1.2
     - pyglet==1.5.0
     - pyopengl==3.1.5
     - pywavelets==1.1.1
@@ -259,14 +262,18 @@ dependencies:
     - sacred==0.7.4
     - sacremoses==0.0.45
     - scikit-image==0.18.1
+    - setuptools==60.10.0
     - statsmodels==0.13.2
     - tables==3.6.1
     - tabulate==0.8.9
     - tensorboard==2.8.0
     - tensorboard-data-server==0.6.1
     - tensorboard-plugin-wit==1.8.0
     - tensorboardx==1.8
+    - tensorflow==2.8.0
+    - tensorflow-io-gcs-filesystem==0.24.0
     - termcolor==1.1.0
+    - tf-estimator-nightly==2.8.0.dev2021122109
     - tifffile==2021.2.26
     - tokenizers==0.10.2
     - tomli==1.2.1