From 5753c2b924a441f43e98589fc2a4af1a6a228684 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 12 Nov 2023 19:40:27 -0800 Subject: [PATCH] More performance tuning --- Makefile | 2 +- coconut/_pyparsing.py | 6 ++++-- coconut/command/command.py | 21 ++++----------------- coconut/command/util.py | 24 +++++++++++++++++++++--- coconut/compiler/compiler.py | 26 +++++++++++++++++--------- coconut/compiler/util.py | 36 +++++++++++++++++++++++++++--------- coconut/constants.py | 13 ++++++++----- coconut/root.py | 2 +- 8 files changed, 83 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 229aae742..beceb6df8 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ test-mypy-tests: clean-no-tests python ./coconut/tests/dest/extras.py # same as test-univ but includes verbose output for better debugging -# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar))[^\n]*\n* +# regex for getting non-timing lines: ^(?!\s*(Time|Packrat|Loaded|Saving|Adaptive|Errorless|Grammar|Failed)\s)[^\n]*\n* .PHONY: test-verbose test-verbose: export COCONUT_USE_COLOR=TRUE test-verbose: clean diff --git a/coconut/_pyparsing.py b/coconut/_pyparsing.py index 9d7ed9b2b..2c288ece3 100644 --- a/coconut/_pyparsing.py +++ b/coconut/_pyparsing.py @@ -48,7 +48,7 @@ never_clear_incremental_cache, warn_on_multiline_regex, num_displayed_timing_items, - use_adaptive_if_available, + use_cache_file, ) from coconut.util import get_clock_time # NOQA from coconut.util import ( @@ -152,6 +152,7 @@ def _parseCache(self, instring, loc, doActions=True, callPreParse=True): if isinstance(value, Exception): raise value return value[0], value[1].copy() + ParserElement.packrat_context = [] ParserElement._parseCache = _parseCache @@ -207,7 +208,8 @@ def enableIncremental(*args, **kwargs): + " (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable) ) -USE_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") and use_adaptive_if_available +SUPPORTS_ADAPTIVE = hasattr(MatchFirst, "setAdaptiveMode") +USE_CACHE = SUPPORTS_INCREMENTAL and use_cache_file maybe_make_safe = getattr(_pyparsing, "maybe_make_safe", None) diff --git a/coconut/command/command.py b/coconut/command/command.py index f2ca9b635..84abc4cf4 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -28,10 +28,10 @@ from subprocess import CalledProcessError from coconut._pyparsing import ( + USE_CACHE, unset_fast_pyparsing_reprs, start_profiling, print_profiling_results, - SUPPORTS_INCREMENTAL, ) from coconut.compiler import Compiler @@ -130,7 +130,7 @@ class Command(object): mypy_args = None # corresponds to --mypy flag argv_args = None # corresponds to --argv flag stack_size = 0 # corresponds to --stack-size flag - use_cache = True # corresponds to --no-cache flag + use_cache = USE_CACHE # corresponds to --no-cache flag prompt = Prompt() @@ -283,11 +283,6 @@ def execute_args(self, args, interact=True, original_args=None): self.argv_args = list(args.argv) if args.no_cache: self.use_cache = False - elif SUPPORTS_INCREMENTAL: - self.use_cache = True - else: - logger.log("incremental parsing mode not supported in current environment (try '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) - self.use_cache = False # execute non-compilation tasks if args.docs: @@ -611,17 +606,9 @@ def callback(compiled): self.execute_file(destpath, argv_source_path=codepath) parse_kwargs = dict( - filename=os.path.basename(codepath), + codepath=codepath, + use_cache=self.use_cache, ) - if self.use_cache: - code_dir, code_fname = os.path.split(codepath) - - cache_dir = os.path.join(code_dir, coconut_cache_dir) - ensure_dir(cache_dir) - - pickle_fname = code_fname + ".pickle" - parse_kwargs["cache_filename"] = os.path.join(cache_dir, pickle_fname) - if package is True: self.submit_comp_job(codepath, callback, "parse_package", code, package_level=package_level, **parse_kwargs) elif package is False: diff --git a/coconut/command/util.py b/coconut/command/util.py index 2f57f7fd3..1758b399b 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -81,6 +81,7 @@ kilobyte, min_stack_size_kbs, coconut_base_run_args, + high_proc_prio, ) if PY26: @@ -130,6 +131,11 @@ ), ) prompt_toolkit = None +try: + import psutil +except ImportError: + psutil = None + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: @@ -222,9 +228,7 @@ def handling_broken_process_pool(): def kill_children(): """Terminate all child processes.""" - try: - import psutil - except ImportError: + if psutil is None: logger.warn( "missing psutil; --jobs may not properly terminate", extra="run '{python} -m pip install psutil' to fix".format(python=sys.executable), @@ -709,6 +713,19 @@ def was_run_code(self, get_all=True): return self.stored[-1] +def highten_process(): + """Set the current process to high priority.""" + if high_proc_prio and psutil is not None: + try: + p = psutil.Process() + if WINDOWS: + p.nice(psutil.HIGH_PRIORITY_CLASS) + else: + p.nice(-10) + except Exception: + logger.log_exc() + + class multiprocess_wrapper(pickleable_obj): """Wrapper for a method that needs to be multiprocessed.""" __slots__ = ("base", "method", "stack_size", "rec_limit", "logger", "argv") @@ -728,6 +745,7 @@ def __reduce__(self): def __call__(self, *args, **kwargs): """Call the method.""" + highten_process() sys.setrecursionlimit(self.rec_limit) logger.copy_from(self.logger) sys.argv = self.argv diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 53487eb7e..a970e993e 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -30,6 +30,7 @@ from coconut.root import * # NOQA import sys +import os import re from contextlib import contextmanager from functools import partial, wraps @@ -38,6 +39,7 @@ from coconut._pyparsing import ( USE_COMPUTATION_GRAPH, + USE_CACHE, ParseBaseException, ParseResults, col as getcol, @@ -174,6 +176,7 @@ pickle_cache, handle_and_manage, sub_all, + get_cache_path, ) from coconut.compiler.header import ( minify_header, @@ -1257,7 +1260,7 @@ def inner_parse_eval( if outer_ln is None: outer_ln = self.adjust(lineno(loc, original)) with self.inner_environment(ln=outer_ln): - self.streamline(parser, inputstring) + self.streamline(parser, inputstring, inner=True) pre_procd = self.pre(inputstring, **preargs) parsed = parse(parser, pre_procd) return self.post(parsed, **postargs) @@ -1270,7 +1273,7 @@ def parsing(self, keep_state=False, filename=None): self.current_compiler[0] = self yield - def streamline(self, grammar, inputstring="", force=False): + def streamline(self, grammar, inputstring="", force=False, inner=False): """Streamline the given grammar for the given inputstring.""" if force or (streamline_grammar_for_len is not None and len(inputstring) > streamline_grammar_for_len): start_time = get_clock_time() @@ -1282,7 +1285,7 @@ def streamline(self, grammar, inputstring="", force=False): length=len(inputstring), ), ) - else: + elif not inner: logger.log("No streamlining done for input of length {length}.".format(length=len(inputstring))) def run_final_checks(self, original, keep_state=False): @@ -1309,20 +1312,25 @@ def parse( postargs, streamline=True, keep_state=False, - filename=None, - cache_filename=None, + codepath=None, + use_cache=None, ): """Use the parser to parse the inputstring with appropriate setup and teardown.""" + if use_cache is None: + use_cache = codepath is not None and USE_CACHE + if use_cache: + cache_path = get_cache_path(codepath) + filename = os.path.basename(codepath) if codepath is not None else None with self.parsing(keep_state, filename): if streamline: self.streamline(parser, inputstring) # unpickling must happen after streamlining and must occur in the # compiler so that it happens in the same process as compilation - if cache_filename is not None: + if use_cache: incremental_enabled = load_cache_for( inputstring=inputstring, filename=filename, - cache_filename=cache_filename, + cache_path=cache_path, ) pre_procd = parsed = None try: @@ -1343,8 +1351,8 @@ def parse( + str(sys.getrecursionlimit()) + " (you may also need to increase --stack-size)", ) finally: - if cache_filename is not None and pre_procd is not None: - pickle_cache(pre_procd, cache_filename, include_incremental=incremental_enabled) + if use_cache and pre_procd is not None: + pickle_cache(pre_procd, cache_path, include_incremental=incremental_enabled) self.run_final_checks(pre_procd, keep_state) return out diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index b681c082a..989c0df6a 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -28,6 +28,7 @@ from coconut.root import * # NOQA import sys +import os import re import ast import inspect @@ -48,7 +49,7 @@ MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, - USE_ADAPTIVE, + SUPPORTS_ADAPTIVE, replaceWith, ZeroOrMore, OneOrMore, @@ -82,6 +83,7 @@ get_target_info, memoize, univ_open, + ensure_dir, ) from coconut.terminal import ( logger, @@ -120,6 +122,8 @@ adaptive_reparse_usage_weight, use_adaptive_any_of, disable_incremental_for_len, + coconut_cache_dir, + use_adaptive_if_available, ) from coconut.exceptions import ( CoconutException, @@ -398,7 +402,7 @@ def adaptive_manager(item, original, loc, reparse=False): def final(item): """Collapse the computation graph upon parsing the given item.""" - if USE_ADAPTIVE: + if SUPPORTS_ADAPTIVE and use_adaptive_if_available: item = Wrap(item, adaptive_manager, greedy=True) # evaluate_tokens expects a computation graph, so we just call add_action directly return add_action(trace(item), final_evaluate_tokens) @@ -774,8 +778,8 @@ def unpickle_cache(filename): return True -def load_cache_for(inputstring, filename, cache_filename): - """Load cache_filename (for the given inputstring and filename).""" +def load_cache_for(inputstring, filename, cache_path): + """Load cache_path (for the given inputstring and filename).""" if not SUPPORTS_INCREMENTAL: raise CoconutException("incremental parsing mode requires cPyparsing (run '{python} -m pip install --upgrade cPyparsing' to fix)".format(python=sys.executable)) if len(inputstring) < disable_incremental_for_len: @@ -793,16 +797,27 @@ def load_cache_for(inputstring, filename, cache_filename): input_len=len(inputstring), max_len=disable_incremental_for_len, ) - did_load_cache = unpickle_cache(cache_filename) - logger.log("{Loaded} cache for {filename!r} from {cache_filename!r} ({incremental_info}).".format( + did_load_cache = unpickle_cache(cache_path) + logger.log("{Loaded} cache for {filename!r} from {cache_path!r} ({incremental_info}).".format( Loaded="Loaded" if did_load_cache else "Failed to load", filename=filename, - cache_filename=cache_filename, + cache_path=cache_path, incremental_info=incremental_info, )) return incremental_enabled +def get_cache_path(codepath): + """Get the cache filename to use for the given codepath.""" + code_dir, code_fname = os.path.split(codepath) + + cache_dir = os.path.join(code_dir, coconut_cache_dir) + ensure_dir(cache_dir) + + pickle_fname = code_fname + ".pkl" + return os.path.join(cache_dir, pickle_fname) + + # ----------------------------------------------------------------------------------------------------------------------- # TARGETS: # ----------------------------------------------------------------------------------------------------------------------- @@ -886,7 +901,6 @@ def get_target_info_smart(target, mode="lowest"): class MatchAny(MatchFirst): """Version of MatchFirst that always uses adaptive parsing.""" - adaptive_mode = True all_match_anys = [] def __init__(self, *args, **kwargs): @@ -903,9 +917,13 @@ def __or__(self, other): return self +if SUPPORTS_ADAPTIVE: + MatchAny.setAdaptiveMode(True) + + def any_of(*exprs, **kwargs): """Build a MatchAny of the given MatchFirst.""" - use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) + use_adaptive = kwargs.pop("use_adaptive", use_adaptive_any_of) and SUPPORTS_ADAPTIVE internal_assert(not kwargs, "excess keyword arguments passed to any_of", kwargs) AnyOf = MatchAny if use_adaptive else MatchFirst diff --git a/coconut/constants.py b/coconut/constants.py index d34b931ba..32ea113be 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -121,6 +121,9 @@ def get_path_env_var(env_var, default): # below constants are experimentally determined to maximize performance +use_packrat_parser = True # True also gives us better error messages +packrat_cache_size = None # only works because final() clears the cache + streamline_grammar_for_len = 4096 # Current problems with this: @@ -130,14 +133,12 @@ def get_path_env_var(env_var, default): # disable_incremental_for_len = streamline_grammar_for_len disable_incremental_for_len = 0 -use_packrat_parser = True # True also gives us better error messages -packrat_cache_size = None # only works because final() clears the cache +use_cache_file = True +use_adaptive_any_of = True # note that _parseIncremental produces much smaller caches use_incremental_if_available = False -use_adaptive_any_of = True - use_adaptive_if_available = False # currently broken adaptive_reparse_usage_weight = 10 @@ -716,6 +717,8 @@ def get_path_env_var(env_var, default): base_default_jobs = "sys" if not PY26 else 0 +high_proc_prio = True + mypy_install_arg = "install" jupyter_install_arg = "install" @@ -993,7 +996,7 @@ def get_path_env_var(env_var, default): # min versions are inclusive unpinned_min_versions = { - "cPyparsing": (2, 4, 7, 2, 2, 4), + "cPyparsing": (2, 4, 7, 2, 2, 5), ("pre-commit", "py3"): (3,), ("psutil", "py>=27"): (5,), "jupyter": (1, 0), diff --git a/coconut/root.py b/coconut/root.py index b8030fc51..c509a4b28 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.3" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 25 +DEVELOP = 26 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"