Merge pull request #589 from ChrisCummins/fix/587

Add a test to demonstrate #587

tests/llvm/BUILD

@@ -148,9 +148,9 @@ py_test(
 )
 
 py_test(
-    name = "gym_interface_compatability",
+    name = "gym_interface_compatability_test",
     timeout = "short",
-    srcs = ["gym_interface_compatability.py"],
+    srcs = ["gym_interface_compatability_test.py"],
     deps = [
         "//compiler_gym/envs/llvm",
         "//tests:test_main",

tests/llvm/CMakeLists.txt

@@ -139,9 +139,9 @@ cg_py_test(
 
 cg_py_test(
   NAME
-    gym_interface_compatability
+    gym_interface_compatability_test
   SRCS
-    "gym_interface_compatability.py"
+    "gym_interface_compatability_test.py"
   DEPS
     compiler_gym::envs::llvm::llvm
     tests::pytest_plugins::llvm

tests/llvm/gym_interface_compatability.py → tests/llvm/gym_interface_compatability_test.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 """Test that LlvmEnv is compatible with OpenAI gym interface."""
 import gym
+import pytest
 
 from compiler_gym.envs.llvm import LlvmEnv
 from tests.test_main import main
@@ -72,5 +73,28 @@ def reward(self, reward):
     assert reward == 1
 
 
+@pytest.mark.xfail(
+    reason="github.com/facebookresearch/CompilerGym/issues/587", strict=True
+)
+def test_env_spec_make(env: LlvmEnv):
+    """Test that demonstrates a failure in gym compatibility: env.spec does
+    not encode mutable state like benchmark, reward space, and observation
+    space.
+    """
+    env.reset(benchmark="cbench-v1/bitcount")
+    with env.spec.make() as new_env:
+        assert new_env.benchmark == env.benchmark
+
+
+def test_env_spec_make_workaround(env: LlvmEnv):
+    """Demonstrate how #587 would be fixed, by updating the 'kwargs' dict."""
+    env.reset(benchmark="cbench-v1/bitcount")
+    env.spec._kwargs[  # pylint: disable=protected-access
+        "benchmark"
+    ] = "cbench-v1/bitcount"
+    with env.spec.make() as new_env:
+        assert new_env.benchmark == env.benchmark
+
+
 if __name__ == "__main__":
     main()