Do not allow async simulation for test collector (#705)

Makefile

@@ -37,17 +37,20 @@ check-docstyle:
 	$(call check_install, doc8)
 	$(call check_install, sphinx)
 	$(call check_install, sphinx_rtd_theme)
+	$(call check_install, sphinxcontrib.bibtex, sphinxcontrib_bibtex)
 	pydocstyle ${PROJECT_PATH} && doc8 docs && cd docs && make html SPHINXOPTS="-W"
 
 doc:
 	$(call check_install, sphinx)
 	$(call check_install, sphinx_rtd_theme)
+	$(call check_install, sphinxcontrib.bibtex, sphinxcontrib_bibtex)
 	cd docs && make html && cd _build/html && python3 -m http.server
 
 spelling:
 	$(call check_install, sphinx)
 	$(call check_install, sphinx_rtd_theme)
 	$(call check_install_extra, sphinxcontrib.spelling, sphinxcontrib.spelling pyenchant)
+	$(call check_install, sphinxcontrib.bibtex, sphinxcontrib_bibtex)
 	cd docs && make spelling SPHINXOPTS="-W"
 
 doc-clean:

docs/spelling_wordlist.txt

@@ -159,3 +159,6 @@ Qbert
 Seaquest
 subnets
 subprocesses
+isort
+yapf
+pydocstyle

docs/tutorials/cheatsheet.rst

@@ -103,7 +103,9 @@ By default, parallel environment simulation is synchronous: a step is done after
 
 In case the time cost of environments varies a lot (e.g. 90% step cost 1s, but 10% cost 10s) where slow environments lag fast environments behind, async simulation can be used (related to `Issue 103 <https://github.com/thu-ml/tianshou/issues/103>`_). The idea is to start those finished environments without waiting for slow environments.
 
-Asynchronous simulation is a built-in functionality of :class:`~tianshou.env.BaseVectorEnv`. Just provide ``wait_num`` or ``timeout`` (or both) and async simulation works.
+Asynchronous simulation is a built-in functionality of
+:class:`~tianshou.env.BaseVectorEnv`. Just provide ``wait_num`` or ``timeout``
+(or both) and async simulation works.
 
 ::
 
@@ -121,6 +123,13 @@ You can treat the ``timeout`` parameter as a dynamic ``wait_num``. In each vecto
 
 The figure in the right gives an intuitive comparison among synchronous/asynchronous simulation.
 
+.. note::
+
+    The async simulation collector would cause some exceptions when used as
+    ``test_collector`` in :doc:`/api/tianshou.trainer` (related to
+    `Issue 700 <https://github.com/thu-ml/tianshou/issues/700>`_). Please use
+    sync version for ``test_collector`` instead.
+
 .. warning::
 
     If you use your own environment, please make sure the ``seed`` method is set up properly, e.g.,

tianshou/trainer/base.py

@@ -6,7 +6,7 @@
 import numpy as np
 import tqdm
 
-from tianshou.data import Collector, ReplayBuffer
+from tianshou.data import AsyncCollector, Collector, ReplayBuffer
 from tianshou.policy import BasePolicy
 from tianshou.trainer.utils import gather_info, test_episode
 from tianshou.utils import (
@@ -232,6 +232,7 @@ def reset(self) -> None:
 
         if self.test_collector is not None:
             assert self.episode_per_test is not None
+            assert not isinstance(self.test_collector, AsyncCollector)  # Issue 700
             self.test_collector.reset_stat()
             test_result = test_episode(
                 self.policy, self.test_collector, self.test_fn, self.start_epoch,