Fix --incremental

Makefile

@@ -152,6 +152,7 @@ test-mypy-tests: clean-no-tests
 	python ./coconut/tests/dest/extras.py
 
 # same as test-univ but includes verbose output for better debugging
+#  regex for getting non-timing lines: ^(?!Time|\s+Packrat|Loaded|Saving).*
 .PHONY: test-verbose
 test-verbose: export COCONUT_USE_COLOR=TRUE
 test-verbose: clean
@@ -183,6 +184,22 @@ test-mypy-all: clean
 	python ./coconut/tests/dest/runner.py
 	python ./coconut/tests/dest/extras.py
 
+# same as test-univ-tests, but forces recompilation for testing --incremental
+.PHONY: test-incremental
+test-incremental: export COCONUT_USE_COLOR=TRUE
+test-incremental: clean-no-tests
+	python ./coconut/tests --strict --keep-lines --incremental --force
+	python ./coconut/tests/dest/runner.py
+	python ./coconut/tests/dest/extras.py
+
+# same as test-incremental, but uses --verbose
+.PHONY: test-incremental-verbose
+test-incremental-verbose: export COCONUT_USE_COLOR=TRUE
+test-incremental-verbose: clean-no-tests
+	python ./coconut/tests --strict --keep-lines --incremental --force --verbose
+	python ./coconut/tests/dest/runner.py
+	python ./coconut/tests/dest/extras.py
+
 # same as test-univ but also tests easter eggs
 .PHONY: test-easter-eggs
 test-easter-eggs: export COCONUT_USE_COLOR=TRUE
@@ -267,13 +284,16 @@ clean-no-tests:
 clean: clean-no-tests
 	rm -rf ./coconut/tests/dest
 
+.PHONY: clean-cache
+clean-cache: clean
+	-find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' +
+	-C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' +
+
 .PHONY: wipe
-wipe: clean
+wipe: clean-cache
 	rm -rf ./coconut/tests/dest vprof.json profile.log *.egg-info
 	-find . -name "__pycache__" -type d -prune -exec rm -rf '{}' +
 	-C:/GnuWin32/bin/find.exe . -name "__pycache__" -type d -prune -exec rm -rf '{}' +
-	-find . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' +
-	-C:/GnuWin32/bin/find.exe . -name "__coconut_cache__" -type d -prune -exec rm -rf '{}' +
 	-find . -name "*.pyc" -delete
 	-C:/GnuWin32/bin/find.exe . -name "*.pyc" -delete
 	-python -m coconut --site-uninstall

coconut/_pyparsing.py

@@ -42,7 +42,7 @@
     get_bool_env_var,
     use_computation_graph_env_var,
     use_incremental_if_available,
-    incremental_cache_size,
+    default_incremental_cache_size,
     never_clear_incremental_cache,
     warn_on_multiline_regex,
 )
@@ -202,7 +202,7 @@ def enableIncremental(*args, **kwargs):
 if MODERN_PYPARSING and use_left_recursion_if_available:
     ParserElement.enable_left_recursion()
 elif SUPPORTS_INCREMENTAL and use_incremental_if_available:
-    ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=not never_clear_incremental_cache)
+    ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=not never_clear_incremental_cache)
 elif use_packrat_parser:
     ParserElement.enablePackrat(packrat_cache_size)
 

coconut/compiler/util.py

@@ -98,18 +98,19 @@
     specific_targets,
     pseudo_targets,
     reserved_vars,
-    use_packrat_parser,
     packrat_cache_size,
     temp_grammar_item_ref_count,
     indchars,
     comment_chars,
     non_syntactic_newline,
     allow_explicit_keyword_vars,
     reserved_prefix,
-    incremental_cache_size,
+    incremental_mode_cache_size,
+    default_incremental_cache_size,
     repeatedly_clear_incremental_cache,
     py_vers_with_eols,
     unwrapper,
+    incremental_cache_limit,
 )
 from coconut.exceptions import (
     CoconutException,
@@ -357,16 +358,16 @@ def attach(item, action, ignore_no_tokens=None, ignore_one_token=None, ignore_to
 
 def should_clear_cache():
     """Determine if we should be clearing the packrat cache."""
-    return (
-        use_packrat_parser
-        and (
-            not ParserElement._incrementalEnabled
-            or (
-                ParserElement._incrementalWithResets
-                and repeatedly_clear_incremental_cache
-            )
-        )
-    )
+    if not ParserElement._packratEnabled:
+        internal_assert(not ParserElement._incrementalEnabled)
+        return False
+    if not ParserElement._incrementalEnabled:
+        return True
+    if ParserElement._incrementalWithResets and repeatedly_clear_incremental_cache:
+        return True
+    if incremental_cache_limit is not None and len(ParserElement.packrat_cache) > incremental_cache_limit:
+        return True
+    return False
 
 
 def final_evaluate_tokens(tokens):
@@ -403,7 +404,10 @@ def force_reset_packrat_cache():
     """Forcibly reset the packrat cache and all packrat stats."""
     if ParserElement._incrementalEnabled:
         ParserElement._incrementalEnabled = False
-        ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=ParserElement._incrementalWithResets)
+        if ParserElement._incrementalWithResets:
+            ParserElement.enableIncremental(default_incremental_cache_size, still_reset_cache=True)
+        else:
+            ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False)
     else:
         ParserElement._packratEnabled = False
         ParserElement.enablePackrat(packrat_cache_size)
@@ -553,26 +557,35 @@ def get_highest_parse_loc(original):
     return highest_loc
 
 
-def enable_incremental_parsing(force=False):
-    """Enable incremental parsing mode where prefix parses are reused."""
-    if SUPPORTS_INCREMENTAL or force:
-        try:
-            ParserElement.enableIncremental(incremental_cache_size, still_reset_cache=False)
-        except ImportError as err:
-            raise CoconutException(str(err))
-        logger.log("Incremental parsing mode enabled.")
-        return True
-    else:
+def enable_incremental_parsing():
+    """Enable incremental parsing mode where prefix/suffix parses are reused."""
+    if not SUPPORTS_INCREMENTAL:
         return False
+    if ParserElement._incrementalEnabled and not ParserElement._incrementalWithResets:  # incremental mode is already enabled
+        return True
+    ParserElement._incrementalEnabled = False
+    try:
+        ParserElement.enableIncremental(incremental_mode_cache_size, still_reset_cache=False)
+    except ImportError as err:
+        raise CoconutException(str(err))
+    logger.log("Incremental parsing mode enabled.")
+    return True
 
 
 def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCOL):
-    """Pickle the pyparsing cache for original to filename. """
+    """Pickle the pyparsing cache for original to filename."""
     internal_assert(all_parse_elements is not None, "pickle_incremental_cache requires cPyparsing")
+
     pickleable_cache_items = []
     for lookup, value in get_cache_items_for(original):
+        if incremental_mode_cache_size is not None and len(pickleable_cache_items) > incremental_mode_cache_size:
+            complain("got too large incremental cache: " + str(len(get_pyparsing_cache())) + " > " + str(incremental_mode_cache_size))
+            break
+        if len(pickleable_cache_items) >= incremental_cache_limit:
+            break
         pickleable_lookup = (lookup[0].parse_element_index,) + lookup[1:]
         pickleable_cache_items.append((pickleable_lookup, value))
+
     logger.log("Saving {num_items} incremental cache items to {filename!r}.".format(
         num_items=len(pickleable_cache_items),
         filename=filename,
@@ -588,6 +601,7 @@ def pickle_incremental_cache(original, filename, protocol=pickle.HIGHEST_PROTOCO
 def unpickle_incremental_cache(filename):
     """Unpickle and load the given incremental cache file."""
     internal_assert(all_parse_elements is not None, "unpickle_incremental_cache requires cPyparsing")
+
     if not os.path.exists(filename):
         return False
     try:
@@ -603,6 +617,14 @@ def unpickle_incremental_cache(filename):
         num_items=len(pickleable_cache_items),
         filename=filename,
     ))
+
+    max_cache_size = min(
+        incremental_mode_cache_size or float("inf"),
+        incremental_cache_limit or float("inf"),
+    )
+    if max_cache_size != float("inf"):
+        pickleable_cache_items = pickleable_cache_items[-max_cache_size:]
+
     packrat_cache = ParserElement.packrat_cache
     for pickleable_lookup, value in pickleable_cache_items:
         lookup = (all_parse_elements[pickleable_lookup[0]],) + pickleable_lookup[1:]

coconut/constants.py

@@ -127,11 +127,16 @@ def get_path_env_var(env_var, default):
 
 # note that _parseIncremental produces much smaller caches
 use_incremental_if_available = True
-incremental_cache_size = None
+
 # these only apply to use_incremental_if_available, not compiler.util.enable_incremental_parsing()
+default_incremental_cache_size = None
 repeatedly_clear_incremental_cache = True
 never_clear_incremental_cache = False
 
+# this is what gets used in compiler.util.enable_incremental_parsing()
+incremental_mode_cache_size = None
+incremental_cache_limit = 524288  # clear cache when it gets this large
+
 use_left_recursion_if_available = False
 
 # -----------------------------------------------------------------------------------------------------------------------

coconut/root.py

@@ -26,7 +26,7 @@
 VERSION = "3.0.3"
 VERSION_NAME = None
 # False for release, int >= 1 for develop
-DEVELOP = 3
+DEVELOP = 4
 ALPHA = False  # for pre releases rather than post releases
 
 assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"

coconut/terminal.py

@@ -97,22 +97,24 @@ def format_error(err_value, err_type=None, err_trace=None):
         return "".join(traceback.format_exception(err_type, err_value, err_trace)).strip()
 
 
-def complain(error):
+def complain(error_or_msg, *args, **kwargs):
     """Raises in develop; warns in release."""
-    if callable(error):
+    if callable(error_or_msg):
         if DEVELOP:
-            error = error()
+            error_or_msg = error_or_msg()
         else:
             return
-    if not isinstance(error, BaseException) or (not isinstance(error, CoconutInternalException) and isinstance(error, CoconutException)):
-        error = CoconutInternalException(str(error))
+    if not isinstance(error_or_msg, BaseException) or (not isinstance(error_or_msg, CoconutInternalException) and isinstance(error_or_msg, CoconutException)):
+        error_or_msg = CoconutInternalException(str(error_or_msg), *args, **kwargs)
+    else:
+        internal_assert(not args and not kwargs, "if error_or_msg is an error, args and kwargs must be empty, not", (args, kwargs))
     if not DEVELOP:
-        logger.warn_err(error)
+        logger.warn_err(error_or_msg)
     elif embed_on_internal_exc:
-        logger.warn_err(error)
+        logger.warn_err(error_or_msg)
         embed(depth=1)
     else:
-        raise error
+        raise error_or_msg
 
 
 def internal_assert(condition, message=None, item=None, extra=None, exc_maker=None):