Gym fixes - Follow up from #705

.github/workflows/ci.yml

@@ -32,8 +32,6 @@ jobs:
  pip install .[extra,tests,docs]
  # Use headless version
  pip install opencv-python-headless
- # Tmp fix: ROM missing in the newest atari-py version
- pip install atari-py==0.2.5
  - name: Build the doc
  run: |
  make doc

.gitlab-ci.yml

@@ -7,8 +7,6 @@ type-check:
 pytest:
  script:
  - python --version
- # Fix to get atari ROMs
- - pip install atari-py==0.2.5
  # MKL_THREADING_LAYER=GNU to avoid MKL_THREADING_LAYER=INTEL incompatibility error
  - MKL_THREADING_LAYER=GNU make pytest
 

README.md

@@ -87,7 +87,7 @@ Documentation is available online: [https://sb3-contrib.readthedocs.io/](https:/
 **Note:** Stable-Baselines3 supports PyTorch >= 1.8.1.
 
 ### Prerequisites
-Stable Baselines3 requires python 3.7+.
+Stable Baselines3 requires Python 3.7+.
 
 #### Windows 10
 

docs/guide/callbacks.rst

@@ -174,7 +174,7 @@ and optionally a prefix for the checkpoints (``rl_model`` by default).
  checkpoint_callback = CheckpointCallback(save_freq=1000, save_path='./logs/',
  name_prefix='rl_model')
 
- model = SAC('MlpPolicy', 'Pendulum-v0')
+ model = SAC('MlpPolicy', 'Pendulum-v1')
  model.learn(2000, callback=checkpoint_callback)
 
 
@@ -206,13 +206,13 @@ It will save the best model if ``best_model_save_path`` folder is specified and
  from stable_baselines3.common.callbacks import EvalCallback
 
  # Separate evaluation env
- eval_env = gym.make('Pendulum-v0')
+ eval_env = gym.make('Pendulum-v1')
  # Use deterministic actions for evaluation
  eval_callback = EvalCallback(eval_env, best_model_save_path='./logs/',
  log_path='./logs/', eval_freq=500,
  deterministic=True, render=False)
 
- model = SAC('MlpPolicy', 'Pendulum-v0')
+ model = SAC('MlpPolicy', 'Pendulum-v1')
  model.learn(5000, callback=eval_callback)
 
 
@@ -234,13 +234,13 @@ Alternatively, you can pass directly a list of callbacks to the ``learn()`` meth
 
  checkpoint_callback = CheckpointCallback(save_freq=1000, save_path='./logs/')
  # Separate evaluation env
- eval_env = gym.make('Pendulum-v0')
+ eval_env = gym.make('Pendulum-v1')
  eval_callback = EvalCallback(eval_env, best_model_save_path='./logs/best_model',
  log_path='./logs/results', eval_freq=500)
  # Create the callback list
  callback = CallbackList([checkpoint_callback, eval_callback])
 
- model = SAC('MlpPolicy', 'Pendulum-v0')
+ model = SAC('MlpPolicy', 'Pendulum-v1')
  # Equivalent to:
  # model.learn(5000, callback=[checkpoint_callback, eval_callback])
  model.learn(5000, callback=callback)
@@ -263,12 +263,12 @@ It must be used with the :ref:`EvalCallback` and use the event triggered by a ne
  from stable_baselines3.common.callbacks import EvalCallback, StopTrainingOnRewardThreshold
 
  # Separate evaluation env
- eval_env = gym.make('Pendulum-v0')
+ eval_env = gym.make('Pendulum-v1')
  # Stop training when the model reaches the reward threshold
  callback_on_best = StopTrainingOnRewardThreshold(reward_threshold=-200, verbose=1)
  eval_callback = EvalCallback(eval_env, callback_on_new_best=callback_on_best, verbose=1)
 
- model = SAC('MlpPolicy', 'Pendulum-v0', verbose=1)
+ model = SAC('MlpPolicy', 'Pendulum-v1', verbose=1)
  # Almost infinite number of timesteps, but the training will stop
  # early as soon as the reward threshold is reached
  model.learn(int(1e10), callback=eval_callback)
@@ -299,7 +299,7 @@ An :ref:`EventCallback` that will trigger its child callback every ``n_steps`` t
  checkpoint_on_event = CheckpointCallback(save_freq=1, save_path='./logs/')
  event_callback = EveryNTimesteps(n_steps=500, callback=checkpoint_on_event)
 
- model = PPO('MlpPolicy', 'Pendulum-v0', verbose=1)
+ model = PPO('MlpPolicy', 'Pendulum-v1', verbose=1)
 
  model.learn(int(2e4), callback=event_callback)
 
@@ -328,7 +328,7 @@ and in total for ``max_episodes * n_envs`` episodes.
  # Stops training when the model reaches the maximum number of episodes
  callback_max_episodes = StopTrainingOnMaxEpisodes(max_episodes=5, verbose=1)
 
- model = A2C('MlpPolicy', 'Pendulum-v0', verbose=1)
+ model = A2C('MlpPolicy', 'Pendulum-v1', verbose=1)
  # Almost infinite number of timesteps, but the training will stop
  # early as soon as the max number of episodes is reached
  model.learn(int(1e10), callback=callback_max_episodes)

docs/guide/custom_policy.rst

@@ -407,5 +407,5 @@ you only need to specify ``net_arch=[256, 256]`` (here, two hidden layers of 256
  # Custom critic architecture with two layers of 400 and 300 units
  policy_kwargs = dict(net_arch=dict(pi=[64, 64], qf=[400, 300]))
  # Create the agent
- model = SAC("MlpPolicy", "Pendulum-v0", policy_kwargs=policy_kwargs, verbose=1)
+ model = SAC("MlpPolicy", "Pendulum-v1", policy_kwargs=policy_kwargs, verbose=1)
  model.learn(5000)

docs/guide/examples.rst

@@ -321,7 +321,7 @@ Atari Games
 
 Training a RL agent on Atari games is straightforward thanks to ``make_atari_env`` helper function.
 It will do `all the preprocessing <https://danieltakeshi.github.io/2016/11/25/frame-skipping-and-preprocessing-for-deep-q-networks-on-atari-2600-games/>`_
-and multiprocessing for you.
+and multiprocessing for you. To install the Atari environments, run the command ``pip install gym[atari, accept-rom-license]`` to install the Atari environments and ROMs, or install Stable Baselines3 with ``pip install stable-baselines3[extra]`` to install this and other optional dependencies.
 
 .. image:: ../_static/img/colab-badge.svg
  :target: https://colab.research.google.com/github/Stable-Baselines-Team/rl-colab-notebooks/blob/sb3/atari_games.ipynb
@@ -564,7 +564,7 @@ Behind the scene, SB3 uses an :ref:`EvalCallback <callbacks>`.
 
  # Create the model, the training environment
  # and the test environment (for evaluation)
- model = SAC('MlpPolicy', 'Pendulum-v0', verbose=1,
+ model = SAC('MlpPolicy', 'Pendulum-v1', verbose=1,
  learning_rate=1e-3, create_eval_env=True)
 
  # Evaluate the model every 1000 steps on 5 test episodes

docs/guide/export.rst

@@ -62,7 +62,7 @@ For PPO, assuming a shared feature extactor.
  action_hidden, value_hidden = self.extractor(observation)
  return self.action_net(action_hidden), self.value_net(value_hidden)
 
- # Example: model = PPO("MlpPolicy", "Pendulum-v0")
+ # Example: model = PPO("MlpPolicy", "Pendulum-v1")
  model = PPO.load("PathToTrainedModel.zip")
  model.policy.to("cpu")
  onnxable_model = OnnxablePolicy(model.policy.mlp_extractor, model.policy.action_net, model.policy.value_net)

docs/guide/tensorboard.rst

@@ -61,7 +61,7 @@ Here is a simple example on how to log both additional tensor or arbitrary scala
  from stable_baselines3 import SAC
  from stable_baselines3.common.callbacks import BaseCallback
 
- model = SAC("MlpPolicy", "Pendulum-v0", tensorboard_log="/tmp/sac/", verbose=1)
+ model = SAC("MlpPolicy", "Pendulum-v1", tensorboard_log="/tmp/sac/", verbose=1)
 
 
  class TensorboardCallback(BaseCallback):
@@ -104,7 +104,7 @@ Here is an example of how to render an image to TensorBoard at regular intervals
  from stable_baselines3.common.callbacks import BaseCallback
  from stable_baselines3.common.logger import Image
 
- model = SAC("MlpPolicy", "Pendulum-v0", tensorboard_log="/tmp/sac/", verbose=1)
+ model = SAC("MlpPolicy", "Pendulum-v1", tensorboard_log="/tmp/sac/", verbose=1)
 
 
  class ImageRecorderCallback(BaseCallback):
@@ -141,7 +141,7 @@ Here is an example of how to store a plot in TensorBoard at regular intervals:
  from stable_baselines3.common.callbacks import BaseCallback
  from stable_baselines3.common.logger import Figure
 
- model = SAC("MlpPolicy", "Pendulum-v0", tensorboard_log="/tmp/sac/", verbose=1)
+ model = SAC("MlpPolicy", "Pendulum-v1", tensorboard_log="/tmp/sac/", verbose=1)
 
 
  class FigureRecorderCallback(BaseCallback):
@@ -251,7 +251,7 @@ can get direct access to the underlying SummaryWriter in a callback:
 
 
 
- model = SAC("MlpPolicy", "Pendulum-v0", tensorboard_log="/tmp/sac/", verbose=1)
+ model = SAC("MlpPolicy", "Pendulum-v1", tensorboard_log="/tmp/sac/", verbose=1)
 
 
  class SummaryWriterCallback(BaseCallback):

docs/misc/changelog.rst

@@ -119,6 +119,7 @@ Release 1.3.0 (2021-10-23)
 
 Breaking Changes:
 ^^^^^^^^^^^^^^^^^
+- Support for Python 3.6 was removed.
 - ``sde_net_arch`` argument in policies is deprecated and will be removed in a future version.
 - ``_get_latent`` (``ActorCriticPolicy``) was removed
 - All logging keys now use underscores instead of spaces (@timokau). Concretely this changes:
@@ -127,6 +128,7 @@ Breaking Changes:
  - ``rollout/exploration rate`` to ``rollout/exploration_rate`` and
  - ``rollout/success rate`` to ``rollout/success_rate``.
 
+
 New Features:
 ^^^^^^^^^^^^^
 - Added methods ``get_distribution`` and ``predict_values`` for ``ActorCriticPolicy`` for A2C/PPO/TRPO (@cyprienc)
@@ -145,6 +147,7 @@ Bug Fixes:
 
 Deprecations:
 ^^^^^^^^^^^^^
+- Switched minimum Gym version to 0.21.0.
 
 Others:
 ^^^^^^^

docs/modules/ddpg.rst

@@ -67,7 +67,7 @@ This example is only to demonstrate the use of the library and its functions, an
  from stable_baselines3 import DDPG
  from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
 
- env = gym.make("Pendulum-v0")
+ env = gym.make("Pendulum-v1")
 
  # The noise objects for DDPG
  n_actions = env.action_space.shape[-1]

docs/modules/sac.rst

@@ -73,7 +73,7 @@ This example is only to demonstrate the use of the library and its functions, an
 
  from stable_baselines3 import SAC
 
- env = gym.make("Pendulum-v0")
+ env = gym.make("Pendulum-v1")
 
  model = SAC("MlpPolicy", env, verbose=1)
  model.learn(total_timesteps=10000, log_interval=4)

docs/modules/td3.rst

@@ -67,7 +67,7 @@ This example is only to demonstrate the use of the library and its functions, an
  from stable_baselines3 import TD3
  from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
 
- env = gym.make("Pendulum-v0")
+ env = gym.make("Pendulum-v1")
 
  # The noise objects for TD3
  n_actions = env.action_space.shape[-1]

setup.py

@@ -73,7 +73,7 @@
  packages=[package for package in find_packages() if package.startswith("stable_baselines3")],
  package_data={"stable_baselines3": ["py.typed", "version.txt"]},
  install_requires=[
- "gym>=0.17,<0.20", # gym 0.20 breaks atari-py behavior
+ "gym>=0.21", # Remember to also update gym version in "extra" below when this changes
  "numpy",
  "torch>=1.8.1",
  # For saving models
@@ -116,7 +116,7 @@
  # For render
  "opencv-python",
  # For atari games,
- "atari_py==0.2.6",
+ "gym[atari,accept-rom-license]>=0.21",
  "pillow",
  # Tensorboard support
  "tensorboard>=2.2.0",

tests/test_callbacks.py

@@ -75,7 +75,7 @@ def test_callbacks(tmp_path, model_class):
  if model_class in [A2C, PPO]:
  max_episodes = 1
  n_envs = 2
- # Pendulum-v0 has a timelimit of 200 timesteps
+ # Pendulum-v1 has a timelimit of 200 timesteps
  max_episode_length = 200
  envs = make_vec_env(env_name, n_envs=n_envs, seed=0)
 
@@ -99,7 +99,7 @@ def select_env(model_class) -> str:
  if model_class is DQN:
  return "CartPole-v0"
  else:
- return "Pendulum-v0"
+ return "Pendulum-v1"
 
 
 def test_eval_callback_vec_env():

tests/test_custom_policy.py

@@ -25,7 +25,7 @@ def test_flexible_mlp(model_class, net_arch):
 @pytest.mark.parametrize("net_arch", [[], [4], [4, 4], dict(qf=[8], pi=[8, 4])])
 @pytest.mark.parametrize("model_class", [SAC, TD3])
 def test_custom_offpolicy(model_class, net_arch):
- _ = model_class("MlpPolicy", "Pendulum-v0", policy_kwargs=dict(net_arch=net_arch), learning_starts=100).learn(300)
+ _ = model_class("MlpPolicy", "Pendulum-v1", policy_kwargs=dict(net_arch=net_arch), learning_starts=100).learn(300)
 
 
 @pytest.mark.parametrize("model_class", [A2C, PPO, SAC, TD3])
@@ -38,12 +38,12 @@ def test_custom_optimizer(model_class, optimizer_kwargs):
  kwargs = dict(n_steps=64)
 
  policy_kwargs = dict(optimizer_class=th.optim.AdamW, optimizer_kwargs=optimizer_kwargs, net_arch=[32])
- _ = model_class("MlpPolicy", "Pendulum-v0", policy_kwargs=policy_kwargs, **kwargs).learn(300)
+ _ = model_class("MlpPolicy", "Pendulum-v1", policy_kwargs=policy_kwargs, **kwargs).learn(300)
 
 
 def test_tf_like_rmsprop_optimizer():
  policy_kwargs = dict(optimizer_class=RMSpropTFLike, net_arch=[32])
- _ = A2C("MlpPolicy", "Pendulum-v0", policy_kwargs=policy_kwargs).learn(500)
+ _ = A2C("MlpPolicy", "Pendulum-v1", policy_kwargs=policy_kwargs).learn(500)
 
 
 def test_dqn_custom_policy():

tests/test_deterministic.py

@@ -13,7 +13,7 @@ def test_deterministic_training_common(algo):
  rewards = [[], []]
  # Smaller network
  kwargs = {"policy_kwargs": dict(net_arch=[64])}
- env_id = "Pendulum-v0"
+ env_id = "Pendulum-v1"
  if algo in [TD3, SAC]:
  kwargs.update({"action_noise": NormalActionNoise(0.0, 0.1), "learning_starts": 100, "train_freq": 4})
  else:

tests/test_distributions.py

@@ -43,7 +43,7 @@ def test_squashed_gaussian(model_class):
  """
  Test run with squashed Gaussian (notably entropy computation)
  """
- model = model_class("MlpPolicy", "Pendulum-v0", use_sde=True, n_steps=64, policy_kwargs=dict(squash_output=True))
+ model = model_class("MlpPolicy", "Pendulum-v1", use_sde=True, n_steps=64, policy_kwargs=dict(squash_output=True))
  model.learn(500)
 
  gaussian_mean = th.rand(N_SAMPLES, N_ACTIONS)
@@ -57,10 +57,10 @@ def test_squashed_gaussian(model_class):
 @pytest.fixture()
 def dummy_model_distribution_obs_and_actions() -> Tuple[A2C, np.array, np.array]:
  """
- Fixture creating a Pendulum-v0 gym env, an A2C model and sampling 10 random observations and actions from the env
+ Fixture creating a Pendulum-v1 gym env, an A2C model and sampling 10 random observations and actions from the env
  :return: A2C model, random observations, random actions
  """
- env = gym.make("Pendulum-v0")
+ env = gym.make("Pendulum-v1")
  model = A2C("MlpPolicy", env, seed=23)
  random_obs = np.array([env.observation_space.sample() for _ in range(10)])
  random_actions = np.array([env.action_space.sample() for _ in range(10)])

tests/test_env_checker.py

@@ -11,14 +11,14 @@ class ActionDictTestEnv(gym.Env):
  observation_space = Box(low=-1.0, high=2.0, shape=(3,), dtype=np.float32)
 
  def step(self, action):
- observation = np.array([1.0, 1.5, 0.5])
+ observation = np.array([1.0, 1.5, 0.5], dtype=self.observation_space.dtype)
  reward = 1
  done = True
  info = {}
  return observation, reward, done, info
 
  def reset(self):
- return np.array([1.0, 1.5, 0.5])
+ return np.array([1.0, 1.5, 0.5], dtype=self.observation_space.dtype)
 
  def render(self, mode="human"):
  pass

tests/test_envs.py

@@ -27,7 +27,7 @@
 ]
 
 
-@pytest.mark.parametrize("env_id", ["CartPole-v0", "Pendulum-v0"])
+@pytest.mark.parametrize("env_id", ["CartPole-v0", "Pendulum-v1"])
 def test_env(env_id):
  """
  Check that environmnent integrated in Gym pass the test.
@@ -38,9 +38,9 @@ def test_env(env_id):
  with pytest.warns(None) as record:
  check_env(env)
 
- # Pendulum-v0 will produce a warning because the action space is
+ # Pendulum-v1 will produce a warning because the action space is
  # in [-2, 2] and not [-1, 1]
- if env_id == "Pendulum-v0":
+ if env_id == "Pendulum-v1":
  assert len(record) == 1
  else:
  # The other environments must pass without warning

tests/test_predict.py

@@ -43,15 +43,15 @@ def test_auto_wrap(model_class):
  if model_class is DQN:
  env_name = "CartPole-v0"
  else:
- env_name = "Pendulum-v0"
+ env_name = "Pendulum-v1"
  env = gym.make(env_name)
  eval_env = gym.make(env_name)
  model = model_class("MlpPolicy", env)
  model.learn(100, eval_env=eval_env)
 
 
 @pytest.mark.parametrize("model_class", MODEL_LIST)
-@pytest.mark.parametrize("env_id", ["Pendulum-v0", "CartPole-v1"])
+@pytest.mark.parametrize("env_id", ["Pendulum-v1", "CartPole-v1"])
 @pytest.mark.parametrize("device", ["cpu", "cuda", "auto"])
 def test_predict(model_class, env_id, device):
  if device == "cuda" and not th.cuda.is_available():