From a14a8292a99d61e8f8de9c6c96f0872ee82ab381 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 17 Nov 2023 02:28:24 -0800 Subject: [PATCH] Fix tests --- coconut/_pyparsing.py | 47 +++++++++++++------ coconut/compiler/util.py | 17 +++++-- coconut/constants.py | 8 ++-- coconut/requirements.py | 2 +- coconut/root.py | 2 +- coconut/tests/constants_test.py | 3 ++ coconut/tests/main_test.py | 6 ++- .../tests/src/cocotest/agnostic/specific.coco | 4 +- 8 files changed, 61 insertions(+), 28 deletions(-) diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index ce6aa061d..1c7344735 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -37,6 +37,7 @@ default_whitespace_chars, varchars, min_versions, + max_versions, pure_python_env_var, enable_pyparsing_warnings, use_left_recursion_if_available, @@ -92,32 +93,48 @@ PYPARSING_PACKAGE = "cPyparsing" if CPYPARSING else "pyparsing" -min_ver = min(min_versions["pyparsing"], min_versions["cPyparsing"][:3]) # inclusive -max_ver = get_next_version(max(min_versions["pyparsing"], min_versions["cPyparsing"][:3])) # exclusive -cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) +if CPYPARSING: + min_ver = min_versions["cPyparsing"] # inclusive + max_ver = get_next_version(min_versions["cPyparsing"], point_to_increment=len(max_versions["cPyparsing"]) - 1) # exclusive +else: + min_ver = min_versions["pyparsing"] # inclusive + max_ver = get_next_version(min_versions["pyparsing"]) # exclusive -min_ver_str = ver_tuple_to_str(min_ver) -max_ver_str = ver_tuple_to_str(max_ver) +cur_ver = None if __version__ is None else ver_str_to_tuple(__version__) if cur_ver is None or cur_ver < min_ver: raise ImportError( - "This version of Coconut requires pyparsing/cPyparsing version >= " + min_ver_str - + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") - + " (run '{python} -m pip install --upgrade {package}' to fix)".format(python=sys.executable, package=PYPARSING_PACKAGE), + ( + "This version of Coconut requires {package} version >= {min_ver}" + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run '{python} -m pip install --upgrade {package}' to fix)" + ).format( + python=sys.executable, + package=PYPARSING_PACKAGE, + min_ver=ver_tuple_to_str(min_ver), + ) ) elif cur_ver >= max_ver: warn( - "This version of Coconut was built for pyparsing/cPyparsing versions < " + max_ver_str - + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") - + " (run '{python} -m pip install {package}<{max_ver}' to fix)".format(python=sys.executable, package=PYPARSING_PACKAGE, max_ver=max_ver_str), + ( + "This version of Coconut was built for {package} versions < {max_ver}" + + ("; got " + PYPARSING_INFO if PYPARSING_INFO is not None else "") + + " (run '{python} -m pip install {package}<{max_ver}' to fix)" + ).format( + python=sys.executable, + package=PYPARSING_PACKAGE, + max_ver=ver_tuple_to_str(max_ver), + ) ) MODERN_PYPARSING = cur_ver >= (3,) if MODERN_PYPARSING: warn( - "This version of Coconut is not built for pyparsing v3; some syntax features WILL NOT WORK" - + " (run either '{python} -m pip install cPyparsing<{max_ver}' or '{python} -m pip install pyparsing<{max_ver}' to fix)".format(python=sys.executable, max_ver=max_ver_str), + "This version of Coconut is not built for pyparsing v3; some syntax features WILL NOT WORK (run either '{python} -m pip install cPyparsing<{max_ver}' or '{python} -m pip install pyparsing<{max_ver}' to fix)".format( + python=sys.executable, + max_ver=ver_tuple_to_str(max_ver), + ) ) @@ -164,7 +181,6 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True): raise value return value[0], value[1].copy() - ParserElement.packrat_context = frozenset() ParserElement._parseCache = _parseCache # [CPYPARSING] fix append @@ -197,6 +213,9 @@ def append(self, other): return self ParseExpression.append = append +if SUPPORTS_PACKRAT_CONTEXT: + ParserElement.packrat_context = frozenset() + if hasattr(ParserElement, "enableIncremental"): SUPPORTS_INCREMENTAL = sys.version_info >= (3, 8) # avoids stack overflows on py<=37 else: diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 05c0365e5..33a861343 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -885,11 +885,17 @@ def pickle_cache(original, cache_path, include_incremental=True, protocol=pickle "pickleable_cache_items": pickleable_cache_items, "all_adaptive_stats": all_adaptive_stats, } - with univ_open(cache_path, "wb") as pickle_file: - pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) - - # clear the packrat cache when we're done so we don't interfere with anything else happening in this process - clear_packrat_cache(force=True) + try: + with univ_open(cache_path, "wb") as pickle_file: + pickle.dump(pickle_info_obj, pickle_file, protocol=protocol) + except Exception: + logger.log_exc() + return False + else: + return True + finally: + # clear the packrat cache when we're done so we don't interfere with anything else happening in this process + clear_packrat_cache(force=True) def unpickle_cache(cache_path): @@ -979,6 +985,7 @@ def load_cache_for(inputstring, codepath): ) if ( + # only load the cache if we're using anything that makes use of it incremental_enabled or use_adaptive_any_of or use_adaptive_if_available diff --git a/coconut/constants.py b/coconut/constants.py index 45113ba35..a581e5fd6 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -134,7 +134,7 @@ def get_path_env_var(env_var, default): streamline_grammar_for_len = 1536 use_cache_file = True -disable_incremental_for_len = 45875 +disable_incremental_for_len = 46080 # this is disabled by default for now because it doesn't improve performance # by very much but is very hard to test, so it's hard to be confident in it use_adaptive_any_of = get_bool_env_var("COCONUT_ADAPTIVE_ANY_OF", False) @@ -154,7 +154,7 @@ def get_path_env_var(env_var, default): incremental_mode_cache_size = None incremental_cache_limit = 2097152 # clear cache when it gets this large incremental_mode_cache_successes = False -require_cache_clear_frac = 0.25 # require that at least this much of the cache must be cleared on each cache clear +require_cache_clear_frac = 0.3125 # require that at least this much of the cache must be cleared on each cache clear use_left_recursion_if_available = False @@ -483,8 +483,7 @@ def get_path_env_var(env_var, default): "itertools.zip_longest": ("itertools./izip_longest", (3,)), "math.gcd": ("fractions./gcd", (3, 5)), "time.process_time": ("time./clock", (3, 3)), - # # _dummy_thread was removed in Python 3.9, so this no longer works - # "_dummy_thread": ("dummy_thread", (3,)), + "_dummy_thread": ("dummy_thread", (3,)), # third-party backports "asyncio": ("trollius", (3, 4)), @@ -1125,6 +1124,7 @@ def get_path_env_var(env_var, default): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Other", "Programming Language :: Other Scripting Engines", "Programming Language :: Python :: Implementation :: CPython", diff --git a/coconut/requirements.py b/coconut/requirements.py index 3035c8440..56b92cee7 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -130,7 +130,7 @@ def get_req_str(req): max_ver = get_next_version(min_versions[req]) if None in max_ver: assert all(v is None for v in max_ver), "invalid max version " + repr(max_ver) - max_ver = get_next_version(min_versions[req], len(max_ver) - 1) + max_ver = get_next_version(min_versions[req], point_to_increment=len(max_ver) - 1) req_str += ",<" + ver_tuple_to_str(max_ver) return req_str diff --git a/coconut/root.py b/coconut/root.py index e0d21ddb9..d5c3fe2a9 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 32 +DEVELOP = 33 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index 65ae8beea..fc32ed6c8 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -31,6 +31,7 @@ from coconut.constants import ( WINDOWS, PYPY, + PY39, fixpath, ) @@ -100,6 +101,8 @@ def test_imports(self): or PYPY and old_imp in ("trollius", "aenum") # don't test typing_extensions, async_generator or old_imp.startswith(("typing_extensions", "async_generator")) + # don't test _dummy_thread on Py3.9 + or PY39 and new_imp == "_dummy_thread" ): pass elif sys.version_info >= ver_cutoff: diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 20916c79e..3804f1fc8 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -31,7 +31,6 @@ import imp import pytest -import pexpect from coconut.util import noop_ctx, get_target_info from coconut.terminal import ( @@ -64,6 +63,8 @@ get_bool_env_var, coconut_cache_dir, default_use_cache_dir, + base_dir, + fixpath, ) from coconut.api import ( @@ -412,6 +413,8 @@ def comp(path=None, folder=None, file=None, args=[], **kwargs): def rm_path(path, allow_keep=False): """Delete a path.""" + path = os.path.abspath(fixpath(path)) + assert not base_dir.startswith(path), "refusing to delete Coconut itself: " + repr(path) if allow_keep and get_bool_env_var("COCONUT_KEEP_TEST_FILES"): return if os.path.isdir(path): @@ -536,6 +539,7 @@ def add_test_func_names(cls): def spawn_cmd(cmd): """Version of pexpect.spawn that prints the command being run.""" + import pexpect # hide import since not always available print("\n>", cmd) return pexpect.spawn(cmd) diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index cbb1eefbe..e35bb7aad 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -164,7 +164,7 @@ def py36_spec_test(tco: bool) -> bool: def py37_spec_test() -> bool: """Tests for any py37+ version.""" - import asyncio, typing + import asyncio, typing, typing_extensions assert py_breakpoint # type: ignore ns: typing.Dict[str, typing.Any] = {} exec("""async def toa(it): @@ -180,7 +180,7 @@ def py37_spec_test() -> bool: assert l == list(range(10)) class HasVarGen[*Ts] # type: ignore assert HasVarGen `issubclass` object - assert typing.Protocol.__module__ == "typing_extensions" + assert typing.Protocol is typing_extensions.Protocol assert_raises((def -> raise ExceptionGroup("derp", [Exception("herp")])), ExceptionGroup) assert_raises((def -> raise BaseExceptionGroup("derp", [BaseException("herp")])), BaseExceptionGroup) return True