diff --git a/Makefile b/Makefile index a664fa94e..1f789a68d 100644 --- a/Makefile +++ b/Makefile @@ -152,6 +152,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging +# regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving).* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean @@ -183,6 +184,22 @@ test-mypy-all: clean python ./coconut/tests/dest/runner.py python ./coconut/tests/dest/extras.py +# same as test-univ-tests, but forces recompilation for testing --incremental +.PHONY: test-incremental +test-incremental: export COCONUT_USE_COLOR=TRUE +test-incremental: clean-no-tests + python ./coconut/tests --strict --keep-lines --incremental --force + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + +# same as test-incremental, but uses --verbose +.PHONY: test-incremental-verbose +test-incremental-verbose: export COCONUT_USE_COLOR=TRUE +test-incremental-verbose: clean-no-tests + python ./coconut/tests --strict --keep-lines --incremental --force --verbose + python ./coconut/tests/dest/runner.py + python ./coconut/tests/dest/extras.py + # same as test-univ but also tests easter eggs .PHONY: test-easter-eggs test-easter-eggs: export COCONUT_USE_COLOR=TRUE @@ -267,13 +284,16 @@ clean-no-tests: clean: clean-no-tests rm -rf ./coconut/tests/dest +.PHONY: clean-cache +clean-cache: clean + -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + + .PHONY: wipe -wipe: clean +wipe: clean-cache rm -rf ./coconut/tests/dest vprof.json profile.log *.egg-info -find . -name "__pycache__" -type d -prune -exec rm -rf '{}' + -C:/GnuWin32/bin/find.exe . -name "__pycache__" -type d -prune -exec rm -rf '{}' + - -find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + - -C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' + -find . -name "*.pyc" -delete -C:/GnuWin32/bin/find.exe . -name "*.pyc" -delete -python -m coconut --site-uninstall diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index de21afa3a..6f290d3cb 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -42,7 +42,7 @@ get_bool_env_var, use_computation_graph_env_var, use_incremental_if_available, - incremental_cache_size, + default_incremental_cache_size, never_clear_incremental_cache, warn_on_multiline_regex, ) @@ -202,7 +202,7 @@ def enableIncremental(*args, **kwargs): if MODERN_PYPARSING and use_left_recursion_if_available: ParserElement.enable_left_recursion() elif SUPPORTS_INCREMENTAL and use_incremental_if_available: - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=not never_clear_incremental_cache) + ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=not never_clear_incremental_cache) elif use_packrat_parser: ParserElement.enablePackrat(packrat_cache_size) diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 47a705639..221268958 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -98,7 +98,6 @@ specific_targets, pseudo_targets, reserved_vars, - use_packrat_parser, packrat_cache_size, temp_grammar_item_ref_count, indchars, @@ -106,10 +105,12 @@ non_syntactic_newline, allow_explicit_keyword_vars, reserved_prefix, - incremental_cache_size, + incremental_mode_cache_size, + default_incremental_cache_size, repeatedly_clear_incremental_cache, py_vers_with_eols, unwrapper, + incremental_cache_limit, ) from coconut.exceptions import ( CoconutException, @@ -357,16 +358,16 @@ def attach(item, action, ignore_no_tokens=None, ignore_one_token=None, ignore_to def should_clear_cache(): """Determine if we should be clearing the packrat cache.""" - return ( - use_packrat_parser - and ( - not ParserElement._incrementalEnabled - or ( - ParserElement._incrementalWithResets - and repeatedly_clear_incremental_cache - ) - ) - ) + if not ParserElement._packratEnabled: + internal_assert(not ParserElement._incrementalEnabled) + return False + if not ParserElement._incrementalEnabled: + return True + if ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache: + return True + if incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit: + return True + return False def final_evaluate_tokens(tokens): @@ -403,7 +404,10 @@ def force_reset_packrat_cache(): """Forcibly reset the packrat cache and all packrat stats.""" if ParserElement._incrementalEnabled: ParserElement._incrementalEnabled = False - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=ParserElement._incrementalWithResets) + if ParserElement._incrementalWithResets: + ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=True) + else: + ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False) else: ParserElement._packratEnabled = False ParserElement.enablePackrat(packrat_cache_size) @@ -553,26 +557,35 @@ def get_highest_parse_loc(original): return highest_loc -def enable_incremental_parsing(force=False): - """Enable incremental parsing mode where prefix parses are reused.""" - if SUPPORTS_INCREMENTAL or force: - try: - ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=False) - except ImportError as err: - raise CoconutException(str(err)) - logger.log("Incremental parsing mode enabled.") - return True - else: +def enable_incremental_parsing(): + """Enable incremental parsing mode where prefix/suffix parses are reused.""" + if not SUPPORTS_INCREMENTAL: return False + if ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets: # incremental mode is already enabled + return True + ParserElement._incrementalEnabled = False + try: + ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False) + except ImportError as err: + raise CoconutException(str(err)) + logger.log("Incremental parsing mode enabled.") + return True def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCOL): - """Pickle the pyparsing cache for original to filename. """ + """Pickle the pyparsing cache for original to filename.""" internal_assert(all_parse_elements is not None, "pickle_incremental_cache requires cPyparsing") + pickleable_cache_items = [] for lookup, value in get_cache_items_for(original): + if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size: + complain("got too large incremental cache: " + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size)) + break + if len(pickleable_cache_items) >= incremental_cache_limit: + break pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:] pickleable_cache_items.append((pickleable_lookup, value)) + logger.log("Saving {num_items} incremental cache items to {filename!r}.".format( num_items=len(pickleable_cache_items), filename=filename, @@ -588,6 +601,7 @@ def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCO def unpickle_incremental_cache(filename): """Unpickle and load the given incremental cache file.""" internal_assert(all_parse_elements is not None, "unpickle_incremental_cache requires cPyparsing") + if not os.path.exists(filename): return False try: @@ -603,6 +617,14 @@ def unpickle_incremental_cache(filename): num_items=len(pickleable_cache_items), filename=filename, )) + + max_cache_size = min( + incremental_mode_cache_size or float("inf"), + incremental_cache_limit or float("inf"), + ) + if max_cache_size != float("inf"): + pickleable_cache_items = pickleable_cache_items[-max_cache_size:] + packrat_cache = ParserElement.packrat_cache for pickleable_lookup, value in pickleable_cache_items: lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:] diff --git a/coconut/constants.py b/coconut/constants.py index 41647ae3b..aee27380b 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -127,11 +127,16 @@ def get_path_env_var(env_var, default): # note that _parseIncremental produces much smaller caches use_incremental_if_available = True -incremental_cache_size = None + # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing() +default_incremental_cache_size = None repeatedly_clear_incremental_cache = True never_clear_incremental_cache = False +# this is what gets used in compiler.util.enable_incremental_parsing() +incremental_mode_cache_size = None +incremental_cache_limit = 524288 # clear cache when it gets this large + use_left_recursion_if_available = False # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/root.py b/coconut/root.py index a1768b59b..714a7124a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 3 +DEVELOP = 4 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/terminal.py b/coconut/terminal.py index 309504385..8a5f7cde0 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -97,22 +97,24 @@ def format_error(err_value, err_type=None, err_trace=None): return "".join(traceback.format_exception(err_type, err_value, err_trace)).strip() -def complain(error): +def complain(error_or_msg, *args, **kwargs): """Raises in develop; warns in release.""" - if callable(error): + if callable(error_or_msg): if DEVELOP: - error = error() + error_or_msg = error_or_msg() else: return - if not isinstance(error, BaseException) or (not isinstance(error, CoconutInternalException) and isinstance(error, CoconutException)): - error = CoconutInternalException(str(error)) + if not isinstance(error_or_msg, BaseException) or (not isinstance(error_or_msg, CoconutInternalException) and isinstance(error_or_msg, CoconutException)): + error_or_msg = CoconutInternalException(str(error_or_msg), *args, **kwargs) + else: + internal_assert(not args and not kwargs, "if error_or_msg is an error, args and kwargs must be empty, not", (args, kwargs)) if not DEVELOP: - logger.warn_err(error) + logger.warn_err(error_or_msg) elif embed_on_internal_exc: - logger.warn_err(error) + logger.warn_err(error_or_msg) embed(depth=1) else: - raise error + raise error_or_msg def internal_assert(condition, message=None, item=None, extra=None, exc_maker=None):