More performance tuning

Makefile

@@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests
 	python ./coconut/tests/dest/extras.py
 
 # same as test-univ but includes verbose output for better debugging
-#  regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar))[^\n]*\n*
+#  regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed)\s)[^\n]*\n*
 .PHONY: test-verbose
 test-verbose: export COCONUT_USE_COLOR=TRUE
 test-verbose: clean

coconut/_pyparsing.py

@@ -48,7 +48,7 @@
     never_clear_incremental_cache,
     warn_on_multiline_regex,
     num_displayed_timing_items,
-    use_adaptive_if_available,
+    use_cache_file,
 )
 from coconut.util import get_clock_time  # NOQA
 from coconut.util import (
@@ -152,6 +152,7 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True):
                     if isinstance(value, Exception):
                         raise value
                     return value[0], value[1].copy()
+
         ParserElement.packrat_context = []
         ParserElement._parseCache = _parseCache
 
@@ -207,7 +208,8 @@ def enableIncremental(*args, **kwargs):
             + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)
         )
 
-USE_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") and use_adaptive_if_available
+SUPPORTS_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode")
+USE_CACHE = SUPPORTS_INCREMENTAL and use_cache_file
 
 maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None)
 

coconut/command/command.py

@@ -28,10 +28,10 @@
 from subprocess import CalledProcessError
 
 from coconut._pyparsing import (
+    USE_CACHE,
     unset_fast_pyparsing_reprs,
     start_profiling,
     print_profiling_results,
-    SUPPORTS_INCREMENTAL,
 )
 
 from coconut.compiler import Compiler
@@ -130,7 +130,7 @@ class Command(object):
     mypy_args = None  # corresponds to --mypy flag
     argv_args = None  # corresponds to --argv flag
     stack_size = 0  # corresponds to --stack-size flag
-    use_cache = True  # corresponds to --no-cache flag
+    use_cache = USE_CACHE  # corresponds to --no-cache flag
 
     prompt = Prompt()
 
@@ -283,11 +283,6 @@ def execute_args(self, args, interact=True, original_args=None):
                 self.argv_args = list(args.argv)
             if args.no_cache:
                 self.use_cache = False
-            elif SUPPORTS_INCREMENTAL:
-                self.use_cache = True
-            else:
-                logger.log("incremental parsing mode not supported in current environment (try '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable))
-                self.use_cache = False
 
             # execute non-compilation tasks
             if args.docs:
@@ -611,17 +606,9 @@ def callback(compiled):
                         self.execute_file(destpath, argv_source_path=codepath)
 
             parse_kwargs = dict(
-                filename=os.path.basename(codepath),
+                codepath=codepath,
+                use_cache=self.use_cache,
             )
-            if self.use_cache:
-                code_dir, code_fname = os.path.split(codepath)
-
-                cache_dir = os.path.join(code_dir, coconut_cache_dir)
-                ensure_dir(cache_dir)
-
-                pickle_fname = code_fname + ".pickle"
-                parse_kwargs["cache_filename"] = os.path.join(cache_dir, pickle_fname)
-
             if package is True:
                 self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs)
             elif package is False:

coconut/command/util.py

@@ -81,6 +81,7 @@
     kilobyte,
     min_stack_size_kbs,
     coconut_base_run_args,
+    high_proc_prio,
 )
 
 if PY26:
@@ -130,6 +131,11 @@
         ),
     )
     prompt_toolkit = None
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
 
 # -----------------------------------------------------------------------------------------------------------------------
 # UTILITIES:
@@ -222,9 +228,7 @@ def handling_broken_process_pool():
 
 def kill_children():
     """Terminate all child processes."""
-    try:
-        import psutil
-    except ImportError:
+    if psutil is None:
         logger.warn(
             "missing psutil; --jobs may not properly terminate",
             extra="run '{python} -m pip install psutil' to fix".format(python=sys.executable),
@@ -709,6 +713,19 @@ def was_run_code(self, get_all=True):
             return self.stored[-1]
 
 
+def highten_process():
+    """Set the current process to high priority."""
+    if high_proc_prio and psutil is not None:
+        try:
+            p = psutil.Process()
+            if WINDOWS:
+                p.nice(psutil.HIGH_PRIORITY_CLASS)
+            else:
+                p.nice(-10)
+        except Exception:
+            logger.log_exc()
+
+
 class multiprocess_wrapper(pickleable_obj):
     """Wrapper for a method that needs to be multiprocessed."""
     __slots__ = ("base", "method", "stack_size", "rec_limit", "logger", "argv")
@@ -728,6 +745,7 @@ def __reduce__(self):
 
     def __call__(self, *args, **kwargs):
         """Call the method."""
+        highten_process()
         sys.setrecursionlimit(self.rec_limit)
         logger.copy_from(self.logger)
         sys.argv = self.argv

coconut/compiler/compiler.py

@@ -30,6 +30,7 @@
 from coconut.root import *  # NOQA
 
 import sys
+import os
 import re
 from contextlib import contextmanager
 from functools import partial, wraps
@@ -38,6 +39,7 @@
 
 from coconut._pyparsing import (
     USE_COMPUTATION_GRAPH,
+    USE_CACHE,
     ParseBaseException,
     ParseResults,
     col as getcol,
@@ -174,6 +176,7 @@
     pickle_cache,
     handle_and_manage,
     sub_all,
+    get_cache_path,
 )
 from coconut.compiler.header import (
     minify_header,
@@ -1257,7 +1260,7 @@ def inner_parse_eval(
         if outer_ln is None:
             outer_ln = self.adjust(lineno(loc, original))
         with self.inner_environment(ln=outer_ln):
-            self.streamline(parser, inputstring)
+            self.streamline(parser, inputstring, inner=True)
             pre_procd = self.pre(inputstring, **preargs)
             parsed = parse(parser, pre_procd)
             return self.post(parsed, **postargs)
@@ -1270,7 +1273,7 @@ def parsing(self, keep_state=False, filename=None):
             self.current_compiler[0] = self
             yield
 
-    def streamline(self, grammar, inputstring="", force=False):
+    def streamline(self, grammar, inputstring="", force=False, inner=False):
         """Streamline the given grammar for the given inputstring."""
         if force or (streamline_grammar_for_len is not None and len(inputstring) > streamline_grammar_for_len):
             start_time = get_clock_time()
@@ -1282,7 +1285,7 @@ def streamline(self, grammar, inputstring="", force=False):
                     length=len(inputstring),
                 ),
             )
-        else:
+        elif not inner:
             logger.log("No streamlining done for input of length {length}.".format(length=len(inputstring)))
 
     def run_final_checks(self, original, keep_state=False):
@@ -1309,20 +1312,25 @@ def parse(
         postargs,
         streamline=True,
         keep_state=False,
-        filename=None,
-        cache_filename=None,
+        codepath=None,
+        use_cache=None,
     ):
         """Use the parser to parse the inputstring with appropriate setup and teardown."""
+        if use_cache is None:
+            use_cache = codepath is not None and USE_CACHE
+        if use_cache:
+            cache_path = get_cache_path(codepath)
+        filename = os.path.basename(codepath) if codepath is not None else None
         with self.parsing(keep_state, filename):
             if streamline:
                 self.streamline(parser, inputstring)
             # unpickling must happen after streamlining and must occur in the
             #  compiler so that it happens in the same process as compilation
-            if cache_filename is not None:
+            if use_cache:
                 incremental_enabled = load_cache_for(
                     inputstring=inputstring,
                     filename=filename,
-                    cache_filename=cache_filename,
+                    cache_path=cache_path,
                 )
             pre_procd = parsed = None
             try:
@@ -1343,8 +1351,8 @@ def parse(
                             + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)",
                         )
             finally:
-                if cache_filename is not None and pre_procd is not None:
-                    pickle_cache(pre_procd, cache_filename, include_incremental=incremental_enabled)
+                if use_cache and pre_procd is not None:
+                    pickle_cache(pre_procd, cache_path, include_incremental=incremental_enabled)
             self.run_final_checks(pre_procd, keep_state)
         return out
 

coconut/compiler/util.py

@@ -28,6 +28,7 @@
 from coconut.root import *  # NOQA
 
 import sys
+import os
 import re
 import ast
 import inspect
@@ -48,7 +49,7 @@
     MODERN_PYPARSING,
     USE_COMPUTATION_GRAPH,
     SUPPORTS_INCREMENTAL,
-    USE_ADAPTIVE,
+    SUPPORTS_ADAPTIVE,
     replaceWith,
     ZeroOrMore,
     OneOrMore,
@@ -82,6 +83,7 @@
     get_target_info,
     memoize,
     univ_open,
+    ensure_dir,
 )
 from coconut.terminal import (
     logger,
@@ -120,6 +122,8 @@
     adaptive_reparse_usage_weight,
     use_adaptive_any_of,
     disable_incremental_for_len,
+    coconut_cache_dir,
+    use_adaptive_if_available,
 )
 from coconut.exceptions import (
     CoconutException,
@@ -398,7 +402,7 @@ def adaptive_manager(item, original, loc, reparse=False):
 
 def final(item):
     """Collapse the computation graph upon parsing the given item."""
-    if USE_ADAPTIVE:
+    if SUPPORTS_ADAPTIVE and use_adaptive_if_available:
         item = Wrap(item, adaptive_manager, greedy=True)
     # evaluate_tokens expects a computation graph, so we just call add_action directly
     return add_action(trace(item), final_evaluate_tokens)
@@ -774,8 +778,8 @@ def unpickle_cache(filename):
     return True
 
 
-def load_cache_for(inputstring, filename, cache_filename):
-    """Load cache_filename (for the given inputstring and filename)."""
+def load_cache_for(inputstring, filename, cache_path):
+    """Load cache_path (for the given inputstring and filename)."""
     if not SUPPORTS_INCREMENTAL:
         raise CoconutException("incremental parsing mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable))
     if len(inputstring) < disable_incremental_for_len:
@@ -793,16 +797,27 @@ def load_cache_for(inputstring, filename, cache_filename):
             input_len=len(inputstring),
             max_len=disable_incremental_for_len,
         )
-    did_load_cache = unpickle_cache(cache_filename)
-    logger.log("{Loaded} cache for {filename!r} from {cache_filename!r} ({incremental_info}).".format(
+    did_load_cache = unpickle_cache(cache_path)
+    logger.log("{Loaded} cache for {filename!r} from {cache_path!r} ({incremental_info}).".format(
         Loaded="Loaded" if did_load_cache else "Failed to load",
         filename=filename,
-        cache_filename=cache_filename,
+        cache_path=cache_path,
         incremental_info=incremental_info,
     ))
     return incremental_enabled
 
 
+def get_cache_path(codepath):
+    """Get the cache filename to use for the given codepath."""
+    code_dir, code_fname = os.path.split(codepath)
+
+    cache_dir = os.path.join(code_dir, coconut_cache_dir)
+    ensure_dir(cache_dir)
+
+    pickle_fname = code_fname + ".pkl"
+    return os.path.join(cache_dir, pickle_fname)
+
+
 # -----------------------------------------------------------------------------------------------------------------------
 # TARGETS:
 # -----------------------------------------------------------------------------------------------------------------------
@@ -886,7 +901,6 @@ def get_target_info_smart(target, mode="lowest"):
 
 class MatchAny(MatchFirst):
     """Version of MatchFirst that always uses adaptive parsing."""
-    adaptive_mode = True
     all_match_anys = []
 
     def __init__(self, *args, **kwargs):
@@ -903,9 +917,13 @@ def __or__(self, other):
         return self
 
 
+if SUPPORTS_ADAPTIVE:
+    MatchAny.setAdaptiveMode(True)
+
+
 def any_of(*exprs, **kwargs):
     """Build a MatchAny of the given MatchFirst."""
-    use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of)
+    use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) and SUPPORTS_ADAPTIVE
     internal_assert(not kwargs, "excess keyword arguments passed to any_of", kwargs)
 
     AnyOf = MatchAny if use_adaptive else MatchFirst

coconut/constants.py

@@ -121,6 +121,9 @@ def get_path_env_var(env_var, default):
 
 # below constants are experimentally determined to maximize performance
 
+use_packrat_parser = True  # True also gives us better error messages
+packrat_cache_size = None  # only works because final() clears the cache
+
 streamline_grammar_for_len = 4096
 
 # Current problems with this:
@@ -130,14 +133,12 @@ def get_path_env_var(env_var, default):
 # disable_incremental_for_len = streamline_grammar_for_len
 disable_incremental_for_len = 0
 
-use_packrat_parser = True  # True also gives us better error messages
-packrat_cache_size = None  # only works because final() clears the cache
+use_cache_file = True
+use_adaptive_any_of = True
 
 # note that _parseIncremental produces much smaller caches
 use_incremental_if_available = False
 
-use_adaptive_any_of = True
-
 use_adaptive_if_available = False  # currently broken
 adaptive_reparse_usage_weight = 10
 
@@ -716,6 +717,8 @@ def get_path_env_var(env_var, default):
 
 base_default_jobs = "sys" if not PY26 else 0
 
+high_proc_prio = True
+
 mypy_install_arg = "install"
 jupyter_install_arg = "install"
 
@@ -993,7 +996,7 @@ def get_path_env_var(env_var, default):
 
 # min versions are inclusive
 unpinned_min_versions = {
-    "cPyparsing": (2, 4, 7, 2, 2, 4),
+    "cPyparsing": (2, 4, 7, 2, 2, 5),
     ("pre-commit", "py3"): (3,),
     ("psutil", "py>=27"): (5,),
     "jupyter": (1, 0),