From 06b4a5b95c01de44779f0ab01ce484d38fe18968 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 1 May 2023 19:31:18 -0700 Subject: [PATCH 01/57] Reenable develop --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 48e7e69b1..1077766c8 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = 1 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From e6c82c222cb6970ec473f9a1db8778d0ed578abb Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 10 May 2023 18:00:33 -0700 Subject: [PATCH 02/57] Various fmap fixes Resolves #736 and #737. --- DOCS.md | 3 ++- _coconut/__init__.pyi | 1 + coconut/compiler/templates/header.py_template | 23 ++++++++++++++----- coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 1 + .../tests/src/cocotest/agnostic/suite.coco | 2 ++ coconut/tests/src/cocotest/agnostic/util.coco | 7 ++++++ 7 files changed, 31 insertions(+), 8 deletions(-) diff --git a/DOCS.md b/DOCS.md index ebacc5ea4..d2a938efd 100644 --- a/DOCS.md +++ b/DOCS.md @@ -747,7 +747,7 @@ Coconut uses a `$` sign right after an iterator before a slice to perform iterat Iterator slicing works just like sequence slicing, including support for negative indices and slices, and support for `slice` objects in the same way as can be done with normal slicing. Iterator slicing makes no guarantee, however, that the original iterator passed to it be preserved (to preserve the iterator, use Coconut's [`reiterable`](#reiterable) built-in). -Coconut's iterator slicing is very similar to Python's `itertools.islice`, but unlike `itertools.islice`, Coconut's iterator slicing supports negative indices, and will preferentially call an object's `__iter_getitem__` (always used if available) or `__getitem__` (only used if the object is a collections.abc.Sequence). Coconut's iterator slicing is also optimized to work well with all of Coconut's built-in objects, only computing the elements of each that are actually necessary to extract the desired slice. +Coconut's iterator slicing is very similar to Python's `itertools.islice`, but unlike `itertools.islice`, Coconut's iterator slicing supports negative indices, and will preferentially call an object's `__iter_getitem__` (always used if available) or `__getitem__` (only used if the object is a `collections.abc.Sequence`). Coconut's iterator slicing is also optimized to work well with all of Coconut's built-in objects, only computing the elements of each that are actually necessary to extract the desired slice. ##### Example @@ -3013,6 +3013,7 @@ The new methods provided by `multiset` on top of `collections.Counter` are: - multiset.**isdisjoint**(_other_): Return True if two multisets have a null intersection. - multiset.**\_\_xor\_\_**(_other_): Return the symmetric difference of two multisets as a new multiset. Specifically: `a ^ b = (a - b) | (b - a)` - multiset.**count**(_item_): Return the number of times an element occurs in a multiset. Equivalent to `multiset[item]`, but additionally verifies the count is non-negative. +- multiset.**\_\_fmap\_\_**(_func_): Apply a function to the contents of the multiset; magic method for [`fmap`](#fmap). Coconut also ensures that `multiset` supports [rich comparisons and `Counter.total()`](https://docs.python.org/3/library/collections.html#collections.Counter) on all Python versions. diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi index e60765ee8..ed242669c 100644 --- a/_coconut/__init__.pyi +++ b/_coconut/__init__.pyi @@ -135,6 +135,7 @@ pandas_numpy_modules: _t.Any = ... jax_numpy_modules: _t.Any = ... tee_type: _t.Any = ... reiterables: _t.Any = ... +fmappables: _t.Any = ... Ellipsis = Ellipsis NotImplemented = NotImplemented diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 347eb1178..12ee3842c 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -35,6 +35,7 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} jax_numpy_modules = {jax_numpy_modules} tee_type = type(itertools.tee((), 1)[0]) reiterables = abc.Sequence, abc.Mapping, abc.Set + fmappables = list, tuple, dict, set, frozenset abc.Sequence.register(collections.deque) Ellipsis, NotImplemented, NotImplementedError, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, RuntimeError, all, any, bool, bytes, callable, classmethod, complex, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, globals, map, min, max, next, object, property, range, reversed, set, setattr, slice, str, sum, super, tuple, type, vars, zip, repr, print{comma_bytearray} = Ellipsis, NotImplemented, NotImplementedError, Exception, AttributeError, ImportError, IndexError, KeyError, NameError, TypeError, ValueError, StopIteration, RuntimeError, all, any, bool, bytes, callable, classmethod, complex, dict, enumerate, filter, float, frozenset, getattr, hasattr, hash, id, int, isinstance, issubclass, iter, len, list, locals, globals, map, min, max, next, object, property, range, reversed, set, setattr, slice, str, sum, {lstatic}super{rstatic}, tuple, type, vars, zip, {lstatic}repr{rstatic}, {lstatic}print{rstatic}{comma_bytearray} def _coconut_handle_cls_kwargs(**kwargs): @@ -1453,18 +1454,27 @@ class multiset(_coconut.collections.Counter{comma_object}): if result < 0: raise _coconut.ValueError("multiset has negative count for " + _coconut.repr(item)) return result + def __fmap__(self, func): + return self.__class__(_coconut.dict((func(obj), num) for obj, num in self.items())) {def_total_and_comparisons}{assign_multiset_views}_coconut.abc.MutableSet.register(multiset) -def _coconut_base_makedata(data_type, args): +def _coconut_base_makedata(data_type, args, from_fmap=False, fallback_to_init=False): if _coconut.hasattr(data_type, "_make") and _coconut.issubclass(data_type, _coconut.tuple): return data_type._make(args) if _coconut.issubclass(data_type, (_coconut.range, _coconut.abc.Iterator)): return args if _coconut.issubclass(data_type, _coconut.str): return "".join(args) - return data_type(args) -def makedata(data_type, *args): + if fallback_to_init or _coconut.issubclass(data_type, _coconut.fmappables): + return data_type(args) + if from_fmap: + raise _coconut.TypeError("no known __fmap__ implementation for " + _coconut.repr(data_type) + " (pass fallback_to_init=True to fall back on __init__ and __iter__)") + raise _coconut.TypeError("no known makedata implementation for " + _coconut.repr(data_type) + " (pass fallback_to_init=True to fall back on __init__)") +def makedata(data_type, *args, **kwargs): """Construct an object of the given data_type containing the given arguments.""" - return _coconut_base_makedata(data_type, args) + fallback_to_init = kwargs.pop("fallback_to_init", False) + if kwargs: + raise _coconut.TypeError("makedata() got unexpected keyword arguments " + _coconut.repr(kwargs)) + return _coconut_base_makedata(data_type, args, fallback_to_init=fallback_to_init) {def_datamaker} {class_amap} def fmap(func, obj, **kwargs): @@ -1474,6 +1484,7 @@ def fmap(func, obj, **kwargs): Override by defining obj.__fmap__(func). """ starmap_over_mappings = kwargs.pop("starmap_over_mappings", False) + fallback_to_init = kwargs.pop("fallback_to_init", False) if kwargs: raise _coconut.TypeError("fmap() got unexpected keyword arguments " + _coconut.repr(kwargs)) obj_fmap = _coconut.getattr(obj, "__fmap__", None) @@ -1505,9 +1516,9 @@ def fmap(func, obj, **kwargs): if aiter is not _coconut.NotImplemented: return _coconut_amap(func, aiter) if starmap_over_mappings: - return _coconut_base_makedata(obj.__class__, {_coconut_}starmap(func, obj.items()) if _coconut.isinstance(obj, _coconut.abc.Mapping) else {_coconut_}map(func, obj)) + return _coconut_base_makedata(obj.__class__, {_coconut_}starmap(func, obj.items()) if _coconut.isinstance(obj, _coconut.abc.Mapping) else {_coconut_}map(func, obj), from_fmap=True, fallback_to_init=fallback_to_init) else: - return _coconut_base_makedata(obj.__class__, {_coconut_}map(func, obj.items() if _coconut.isinstance(obj, _coconut.abc.Mapping) else obj)) + return _coconut_base_makedata(obj.__class__, {_coconut_}map(func, obj.items() if _coconut.isinstance(obj, _coconut.abc.Mapping) else obj), from_fmap=True, fallback_to_init=fallback_to_init) def _coconut_memoize_helper(maxsize=None, typed=False): return maxsize, typed def memoize(*args, **kwargs): diff --git a/coconut/root.py b/coconut/root.py index 1077766c8..a79f09b37 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 1 +DEVELOP = 2 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 8f61821a0..9d6763eea 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1589,4 +1589,5 @@ def primary_test() -> bool: assert ["abc" ;; "def"] == [['abc'], ['def']] assert {"a":0, "b":1}$[0] == "a" assert (|0, NotImplemented, 2|)$[1] is NotImplemented + assert m{1, 1, 2} |> fmap$(.+1) == m{2, 2, 3} return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 46c2fdd5f..905f6ab16 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1045,6 +1045,8 @@ forward 2""") == 900 assert get_glob() == 0 assert wrong_get_set_glob(20) == 10 assert take_xy(xy("a", "b")) == ("a", "b") + assert InitAndIter(range(3)) |> fmap$(.+1, fallback_to_init=True) == InitAndIter(range(1, 4)) + assert_raises(-> InitAndIter(range(3)) |> fmap$(.+1), TypeError) # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 86aa712a8..69aff8db3 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -530,6 +530,13 @@ def summer(): summer.acc += summer.args.pop() return summer() +class InitAndIter: + def __init__(self, it): + self.it = tuple(it) + def __iter__(self) = self.it + def __eq__(self, other) = + self.__class__ == other.__class__ and self.it == other.it + # Data Blocks: try: From 27794a33916768618698a0c2a3058b7351046889 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 12 May 2023 16:32:47 -0500 Subject: [PATCH 03/57] Fix tests --- DOCS.md | 2 +- coconut/tests/src/cocotest/agnostic/suite.coco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index d2a938efd..b9cba9afe 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3013,7 +3013,7 @@ The new methods provided by `multiset` on top of `collections.Counter` are: - multiset.**isdisjoint**(_other_): Return True if two multisets have a null intersection. - multiset.**\_\_xor\_\_**(_other_): Return the symmetric difference of two multisets as a new multiset. Specifically: `a ^ b = (a - b) | (b - a)` - multiset.**count**(_item_): Return the number of times an element occurs in a multiset. Equivalent to `multiset[item]`, but additionally verifies the count is non-negative. -- multiset.**\_\_fmap\_\_**(_func_): Apply a function to the contents of the multiset; magic method for [`fmap`](#fmap). +- multiset.**\_\_fmap\_\_**(_func_): Apply a function to the contents of the multiset, preserving counts; magic method for [`fmap`](#fmap). Coconut also ensures that `multiset` supports [rich comparisons and `Counter.total()`](https://docs.python.org/3/library/collections.html#collections.Counter) on all Python versions. diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 905f6ab16..e7d47a2ff 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1045,7 +1045,7 @@ forward 2""") == 900 assert get_glob() == 0 assert wrong_get_set_glob(20) == 10 assert take_xy(xy("a", "b")) == ("a", "b") - assert InitAndIter(range(3)) |> fmap$(.+1, fallback_to_init=True) == InitAndIter(range(1, 4)) + assert InitAndIter(range(3)) |> fmap$((.+1), fallback_to_init=True) == InitAndIter(range(1, 4)) assert_raises(-> InitAndIter(range(3)) |> fmap$(.+1), TypeError) # must come at end From 1a9fc2ad1d668d272566068425169e280bbc572b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 13 May 2023 01:31:16 -0500 Subject: [PATCH 04/57] Further fix tests --- coconut/tests/src/cocotest/agnostic/util.coco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 69aff8db3..59b3ec93c 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -533,7 +533,7 @@ def summer(): class InitAndIter: def __init__(self, it): self.it = tuple(it) - def __iter__(self) = self.it + def __iter__(self) = iter(self.it) def __eq__(self, other) = self.__class__ == other.__class__ and self.it == other.it From 1ad25801a57f1ba32d0b25da7c33edf21227554b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 May 2023 18:50:21 -0700 Subject: [PATCH 05/57] Fix jobs on standalone mode Resolves #739. --- coconut/command/command.py | 24 +++++++++++++----------- coconut/command/util.py | 5 +++++ coconut/root.py | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index ebeeace41..864b606a1 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -96,6 +96,8 @@ can_parse, invert_mypy_arg, run_with_stack_size, + memoized_isdir, + memoized_isfile, ) from coconut.compiler.util import ( should_indent, @@ -302,7 +304,7 @@ def execute_args(self, args, interact=True, original_args=None): ] # disable jobs if we know we're only compiling one file - if len(src_dest_package_triples) <= 1 and not any(package for _, _, package in src_dest_package_triples): + if len(src_dest_package_triples) <= 1 and not any(memoized_isdir(source) for source, dest, package in src_dest_package_triples): self.disable_jobs() # do compilation @@ -363,12 +365,12 @@ def process_source_dest(self, source, dest, args): processed_source = fixpath(source) # validate args - if (args.run or args.interact) and os.path.isdir(processed_source): + if (args.run or args.interact) and memoized_isdir(processed_source): if args.run: raise CoconutException("source path %r must point to file not directory when --run is enabled" % (source,)) if args.interact: raise CoconutException("source path %r must point to file not directory when --run (implied by --interact) is enabled" % (source,)) - if args.watch and os.path.isfile(processed_source): + if args.watch and memoized_isfile(processed_source): raise CoconutException("source path %r must point to directory not file when --watch is enabled" % (source,)) # determine dest @@ -389,9 +391,9 @@ def process_source_dest(self, source, dest, args): package = False else: # auto-decide package - if os.path.isfile(source): + if memoized_isfile(processed_source): package = False - elif os.path.isdir(source): + elif memoized_isdir(processed_source): package = True else: raise CoconutException("could not find source path", source) @@ -442,17 +444,17 @@ def compile_path(self, path, write=True, package=True, **kwargs): """Compile a path and returns paths to compiled files.""" if not isinstance(write, bool): write = fixpath(write) - if os.path.isfile(path): + if memoized_isfile(path): destpath = self.compile_file(path, write, package, **kwargs) return [destpath] if destpath is not None else [] - elif os.path.isdir(path): + elif memoized_isdir(path): return self.compile_folder(path, write, package, **kwargs) else: raise CoconutException("could not find source path", path) def compile_folder(self, directory, write=True, package=True, **kwargs): """Compile a directory and returns paths to compiled files.""" - if not isinstance(write, bool) and os.path.isfile(write): + if not isinstance(write, bool) and memoized_isfile(write): raise CoconutException("destination path cannot point to a file when compiling a directory") filepaths = [] for dirpath, dirnames, filenames in os.walk(directory): @@ -660,7 +662,7 @@ def running_jobs(self, exit_on_error=True): def has_hash_of(self, destpath, code, package_level): """Determine if a file has the hash of the code.""" - if destpath is not None and os.path.isfile(destpath): + if destpath is not None and memoized_isfile(destpath): with univ_open(destpath, "r") as opened: compiled = readfile(opened) hashash = gethash(compiled) @@ -989,7 +991,7 @@ def watch(self, src_dest_package_triples, run=False, force=False): def recompile(path, src, dest, package): path = fixpath(path) - if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts: + if memoized_isfile(path) and os.path.splitext(path)[1] in code_exts: with self.handling_exceptions(): if dest is True or dest is None: writedir = dest @@ -1043,7 +1045,7 @@ def site_uninstall(self): python_lib = self.get_python_lib() pth_file = os.path.join(python_lib, os.path.basename(coconut_pth_file)) - if os.path.isfile(pth_file): + if memoized_isfile(pth_file): os.remove(pth_file) logger.show_sig("Removed %s from %s" % (os.path.basename(coconut_pth_file), python_lib)) else: diff --git a/coconut/command/util.py b/coconut/command/util.py index 8403def86..3f60c82d1 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -46,6 +46,7 @@ pickleable_obj, get_encoding, get_clock_time, + memoize, ) from coconut.constants import ( WINDOWS, @@ -132,6 +133,10 @@ # ----------------------------------------------------------------------------------------------------------------------- +memoized_isdir = memoize(128)(os.path.isdir) +memoized_isfile = memoize(128)(os.path.isfile) + + def writefile(openedfile, newcontents): """Set the contents of a file.""" openedfile.seek(0) diff --git a/coconut/root.py b/coconut/root.py index a79f09b37..07db2b217 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 2 +DEVELOP = 3 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From ef1041a0b63de8c3bcf677832e0b52999b22e03e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 15 May 2023 19:13:23 -0700 Subject: [PATCH 06/57] Improve --and, multiprocessing --- DOCS.md | 2 +- coconut/command/cli.py | 2 +- coconut/command/command.py | 33 +++++++++++++++++++++++---------- coconut/constants.py | 2 ++ coconut/root.py | 2 +- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/DOCS.md b/DOCS.md index b9cba9afe..0c2eb0585 100644 --- a/DOCS.md +++ b/DOCS.md @@ -142,7 +142,7 @@ dest destination directory for compiled files (defaults to ``` -h, --help show this help message and exit --and source [dest ...] - add an additional source/dest pair to compile + add an additional source/dest pair to compile (dest is optional) -v, -V, --version print Coconut and Python version information -t version, --target version specify target Python version (defaults to universal) diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 5e9c930a1..73af5fde9 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -77,7 +77,7 @@ type=str, nargs="+", action="append", - help="add an additional source/dest pair to compile", + help="add an additional source/dest pair to compile (dest is optional)", ) arguments.add_argument( diff --git a/coconut/command/command.py b/coconut/command/command.py index 864b606a1..56177d9ec 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -23,6 +23,7 @@ import os import time import shutil +import random from contextlib import contextmanager from subprocess import CalledProcessError @@ -68,6 +69,7 @@ error_color_code, jupyter_console_commands, default_jobs, + create_package_retries, ) from coconut.util import ( univ_open, @@ -295,13 +297,14 @@ def execute_args(self, args, interact=True, original_args=None): raise CoconutException("cannot compile with --no-write when using --mypy") # process all source, dest pairs - src_dest_package_triples = [ - self.process_source_dest(src, dst, args) - for src, dst in ( - [(args.source, args.dest)] - + (getattr(args, "and") or []) - ) - ] + src_dest_package_triples = [] + for and_args in [(args.source, args.dest)] + (getattr(args, "and") or []): + if len(and_args) == 1: + src, = and_args + dest = None + else: + src, dest = and_args + src_dest_package_triples.append(self.process_source_dest(src, dest, args)) # disable jobs if we know we're only compiling one file if len(src_dest_package_triples) <= 1 and not any(memoized_isdir(source) for source, dest, package in src_dest_package_triples): @@ -583,11 +586,21 @@ def get_package_level(self, codepath): return package_level return 0 - def create_package(self, dirpath): + def create_package(self, dirpath, retries_left=create_package_retries): """Set up a package directory.""" filepath = os.path.join(dirpath, "__coconut__.py") - with univ_open(filepath, "w") as opened: - writefile(opened, self.comp.getheader("__coconut__")) + try: + with univ_open(filepath, "w") as opened: + writefile(opened, self.comp.getheader("__coconut__")) + except OSError: + logger.log_exc() + if retries_left <= 0: + logger.warn("Failed to write header file at", filepath) + else: + # sleep a random amount of time from 0 to 0.1 seconds to + # stagger calls across processes + time.sleep(random.random() / 10) + self.create_package(dirpath, retries_left - 1) def submit_comp_job(self, path, callback, method, *args, **kwargs): """Submits a job on self.comp to be run in parallel.""" diff --git a/coconut/constants.py b/coconut/constants.py index e42f8a8cb..ecf359496 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -647,6 +647,8 @@ def get_bool_env_var(env_var, default=False): jupyter_console_commands = ("console", "qtconsole") +create_package_retries = 1 + # ----------------------------------------------------------------------------------------------------------------------- # HIGHLIGHTER CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/root.py b/coconut/root.py index 07db2b217..c15fa5162 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 3 +DEVELOP = 4 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 9c2ec388e6a883dab2cd54713051e39a2a1de3f8 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 16 May 2023 15:07:07 -0700 Subject: [PATCH 07/57] Improve --jupyter arg processing --- coconut/command/command.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 56177d9ec..f947853c2 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -984,7 +984,10 @@ def start_jupyter(self, args): # pass the kernel to the console or otherwise just launch Jupyter now that we know our kernel is available if args[0] in jupyter_console_commands: - args += ["--kernel", kernel] + if "--kernel" in args: + logger.warn("unable to specify Coconut kernel in 'jupyter " + args[0] + "' command as --kernel was already specified in the given arguments") + else: + args += ["--kernel", kernel] run_args = jupyter + args if newly_installed_kernels: From 9dcceea811efe19de0e8ec7fba8affaa15c02422 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 18 May 2023 23:07:29 -0700 Subject: [PATCH 08/57] Fix setup.cfg Refs #742. --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2e9053c06..7fa9076ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,5 @@ universal = 1 [metadata] -license_file = LICENSE.txt +license_files = + LICENSE.txt From 861fda43c19ac31a0834f51084965dab465a7545 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 18 May 2023 23:22:40 -0700 Subject: [PATCH 09/57] Bump develop version --- coconut/command/command.py | 2 +- coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index f947853c2..3bcd5fd7d 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -984,7 +984,7 @@ def start_jupyter(self, args): # pass the kernel to the console or otherwise just launch Jupyter now that we know our kernel is available if args[0] in jupyter_console_commands: - if "--kernel" in args: + if any(a.startswith("--kernel") for a in args): logger.warn("unable to specify Coconut kernel in 'jupyter " + args[0] + "' command as --kernel was already specified in the given arguments") else: args += ["--kernel", kernel] diff --git a/coconut/root.py b/coconut/root.py index c15fa5162..d23825cae 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 4 +DEVELOP = 5 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From b06181708415e3e51e90ca34a7c20cd40a5ff905 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 19 May 2023 19:19:35 -0700 Subject: [PATCH 10/57] Improve function composition Resolves #744. --- DOCS.md | 2 + __coconut__/__init__.pyi | 23 ++- coconut/compiler/templates/header.py_template | 144 ++++++++++++++---- coconut/root.py | 2 +- coconut/tests/src/cocotest/agnostic/main.coco | 3 + .../tests/src/cocotest/agnostic/primary.coco | 4 + .../tests/src/cocotest/agnostic/specific.coco | 13 ++ 7 files changed, 158 insertions(+), 33 deletions(-) diff --git a/DOCS.md b/DOCS.md index 0c2eb0585..44c83decd 100644 --- a/DOCS.md +++ b/DOCS.md @@ -726,6 +726,8 @@ The `..` operator has lower precedence than `::` but higher precedence than infi All function composition operators also have in-place versions (e.g. `..=`). +Since all forms of function composition always call the first function in the composition (`f` in `f ..> g` and `g` in `f <.. g`) with exactly the arguments passed into the composition, all forms of function composition will preserve all metadata attached to the first function in the composition, including the function's [signature](https://docs.python.org/3/library/inspect.html#inspect.signature) and any of that function's attributes. + ##### Example **Coconut:** diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 4a42bb999..75c660612 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -169,23 +169,29 @@ enumerate = enumerate _coconut_py_str = py_str _coconut_super = super +_coconut_enumerate = enumerate +_coconut_filter = filter +_coconut_range = range +_coconut_reversed = reversed +_coconut_zip = zip zip_longest = _coconut.zip_longest memoize = _lru_cache - - reduce = _coconut.functools.reduce takewhile = _coconut.itertools.takewhile dropwhile = _coconut.itertools.dropwhile -tee = _coconut_tee = _coconut.itertools.tee -starmap = _coconut_starmap = _coconut.itertools.starmap +tee = _coconut.itertools.tee +starmap = _coconut.itertools.starmap cartesian_product = _coconut.itertools.product -multiset = _coconut_multiset = _coconut.collections.Counter - +multiset = _coconut.collections.Counter _coconut_tee = tee _coconut_starmap = starmap +_coconut_cartesian_product = cartesian_product +_coconut_multiset = multiset + + parallel_map = concurrent_map = _coconut_map = map @@ -200,6 +206,7 @@ def scan( iterable: _t.Iterable[_U], initial: _T = ..., ) -> _t.Iterable[_T]: ... +_coconut_scan = scan class MatchError(Exception): @@ -968,6 +975,7 @@ class cycle(_t.Iterable[_T]): def __fmap__(self, func: _t.Callable[[_T], _U]) -> _t.Iterable[_U]: ... def __copy__(self) -> cycle[_T]: ... def __len__(self) -> int: ... +_coconut_cycle = cycle class groupsof(_t.Generic[_T]): @@ -981,6 +989,7 @@ class groupsof(_t.Generic[_T]): def __copy__(self) -> groupsof[_T]: ... def __len__(self) -> int: ... def __fmap__(self, func: _t.Callable[[_t.Tuple[_T, ...]], _U]) -> _t.Iterable[_U]: ... +_coconut_groupsof = groupsof class windowsof(_t.Generic[_T]): @@ -996,6 +1005,7 @@ class windowsof(_t.Generic[_T]): def __copy__(self) -> windowsof[_T]: ... def __len__(self) -> int: ... def __fmap__(self, func: _t.Callable[[_t.Tuple[_T, ...]], _U]) -> _t.Iterable[_U]: ... +_coconut_windowsof = windowsof class flatten(_t.Iterable[_T]): @@ -1228,6 +1238,7 @@ def lift(func: _t.Callable[[_T, _U], _W]) -> _coconut_lifted_2[_T, _U, _W]: ... def lift(func: _t.Callable[[_T, _U, _V], _W]) -> _coconut_lifted_3[_T, _U, _V, _W]: ... @_t.overload def lift(func: _t.Callable[..., _W]) -> _t.Callable[..., _t.Callable[..., _W]]: ... +_coconut_lift = lift def all_equal(iterable: _Iterable) -> bool: ... diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 12ee3842c..5fa1e9760 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -184,7 +184,7 @@ def tee(iterable, n=2): class _coconut_has_iter(_coconut_baseclass): __slots__ = ("lock", "iter") def __new__(cls, iterable): - self = _coconut.object.__new__(cls) + self = _coconut.super(_coconut_has_iter, cls).__new__(cls) self.lock = _coconut.threading.Lock() self.iter = iterable return self @@ -201,7 +201,7 @@ class reiterable(_coconut_has_iter): def __new__(cls, iterable): if _coconut.isinstance(iterable, _coconut.reiterables): return iterable - return _coconut_has_iter.__new__(cls, iterable) + return _coconut.super({_coconut_}reiterable, cls).__new__(cls, iterable) def get_new_iter(self): """Tee the underlying iterator.""" with self.lock: @@ -331,21 +331,28 @@ def _coconut_iter_getitem(iterable, index): return () iterable = _coconut.itertools.islice(iterable, 0, n) return _coconut.tuple(iterable)[i::step] -class _coconut_base_compose(_coconut_baseclass): - __slots__ = ("func", "func_infos") +class _coconut_base_compose(_coconut_baseclass):{COMMENT.no_slots_to_allow_update_wrapper}{COMMENT.must_use_coconut_attrs_to_avoid_interacting_with_update_wrapper} def __init__(self, func, *func_infos): - self.func = func - self.func_infos = [] + try: + _coconut.functools.update_wrapper(self, func) + except _coconut.AttributeError: + pass + if _coconut.isinstance(func, _coconut_base_compose): + self._coconut_func = func._coconut_func + func_infos = func._coconut_func_infos + func_infos + else: + self._coconut_func = func + self._coconut_func_infos = [] for f, stars, none_aware in func_infos: if _coconut.isinstance(f, _coconut_base_compose): - self.func_infos.append((f.func, stars, none_aware)) - self.func_infos += f.func_infos + self._coconut_func_infos.append((f._coconut_func, stars, none_aware)) + self._coconut_func_infos += f._coconut_func_infos else: - self.func_infos.append((f, stars, none_aware)) - self.func_infos = _coconut.tuple(self.func_infos) + self._coconut_func_infos.append((f, stars, none_aware)) + self._coconut_func_infos = _coconut.tuple(self._coconut_func_infos) def __call__(self, *args, **kwargs): - arg = self.func(*args, **kwargs) - for f, stars, none_aware in self.func_infos: + arg = self._coconut_func(*args, **kwargs) + for f, stars, none_aware in self._coconut_func_infos: if none_aware and arg is None: return arg if stars == 0: @@ -358,9 +365,9 @@ class _coconut_base_compose(_coconut_baseclass): raise _coconut.RuntimeError("invalid internal stars value " + _coconut.repr(stars) + " in " + _coconut.repr(self) + " {report_this_text}") return arg def __repr__(self): - return _coconut.repr(self.func) + " " + " ".join(".." + "?"*none_aware + "*"*stars + "> " + _coconut.repr(f) for f, stars, none_aware in self.func_infos) + return _coconut.repr(self._coconut_func) + " " + " ".join(".." + "?"*none_aware + "*"*stars + "> " + _coconut.repr(f) for f, stars, none_aware in self._coconut_func_infos) def __reduce__(self): - return (self.__class__, (self.func,) + self.func_infos) + return (self.__class__, (self._coconut_func,) + self._coconut_func_infos) def __get__(self, obj, objtype=None): if obj is None: return self @@ -501,7 +508,7 @@ class scan(_coconut_has_iter): optionally starting from initial.""" __slots__ = ("func", "initial") def __new__(cls, function, iterable, initial=_coconut_sentinel): - self = _coconut_has_iter.__new__(cls, iterable) + self = _coconut.super({_coconut_}scan, cls).__new__(cls, iterable) self.func = function self.initial = initial return self @@ -532,8 +539,7 @@ class reversed(_coconut_has_iter): if _coconut.isinstance(iterable, _coconut.range): return iterable[::-1] if _coconut.getattr(iterable, "__reversed__", None) is None or _coconut.isinstance(iterable, (_coconut.list, _coconut.tuple)): - self = _coconut_has_iter.__new__(cls, iterable) - return self + return _coconut.super({_coconut_}reversed, cls).__new__(cls, iterable) return _coconut.reversed(iterable) def __repr__(self): return "reversed(%s)" % (_coconut.repr(self.iter),) @@ -574,7 +580,7 @@ class flatten(_coconut_has_iter):{COMMENT.cant_implement_len_else_list_calls_bec raise _coconut.ValueError("flatten: levels cannot be negative") if levels == 0: return iterable - self = _coconut_has_iter.__new__(cls, iterable) + self = _coconut.super({_coconut_}flatten, cls).__new__(cls, iterable) self.levels = levels self._made_reit = False return self @@ -673,7 +679,7 @@ Additionally supports Cartesian products of numpy arrays.""" for i, a in _coconut.enumerate(numpy.ix_(*iterables)): arr[..., i] = a return arr.reshape(-1, _coconut.len(iterables)) - self = _coconut.object.__new__(cls) + self = _coconut.super({_coconut_}cartesian_product, cls).__new__(cls) self.iters = iterables self.repeat = repeat return self @@ -775,7 +781,7 @@ class _coconut_base_parallel_concurrent_map(map): def get_pool_stack(cls): return cls.threadlocal_ns.__dict__.setdefault("pool_stack", [None]) def __new__(cls, function, *iterables, **kwargs): - self = {_coconut_}map.__new__(cls, function, *iterables) + self = _coconut.super(_coconut_base_parallel_concurrent_map, cls).__new__(cls, function, *iterables) self.result = None self.chunksize = kwargs.pop("chunksize", 1) self.strict = kwargs.pop("strict", False) @@ -870,7 +876,7 @@ class zip_longest(zip): __slots__ = ("fillvalue",) __doc__ = getattr(_coconut.zip_longest, "__doc__", "Version of zip that fills in missing values with fillvalue.") def __new__(cls, *iterables, **kwargs): - self = {_coconut_}zip.__new__(cls, *iterables, strict=False) + self = _coconut.super({_coconut_}zip_longest, cls).__new__(cls, *iterables, strict=False) self.fillvalue = kwargs.pop("fillvalue", None) if kwargs: raise _coconut.TypeError(cls.__name__ + "() got unexpected keyword arguments " + _coconut.repr(kwargs)) @@ -1081,7 +1087,7 @@ class cycle(_coconut_has_iter): before stopping.""" __slots__ = ("times",) def __new__(cls, iterable, times=None): - self = _coconut_has_iter.__new__(cls, iterable) + self = _coconut.super({_coconut_}cycle, cls).__new__(cls, iterable) if times is None: self.times = None else: @@ -1136,7 +1142,7 @@ class windowsof(_coconut_has_iter): If that is not the desired behavior, fillvalue can be passed and will be used in place of missing values.""" __slots__ = ("size", "fillvalue", "step") def __new__(cls, size, iterable, fillvalue=_coconut_sentinel, step=1): - self = _coconut_has_iter.__new__(cls, iterable) + self = _coconut.super({_coconut_}windowsof, cls).__new__(cls, iterable) self.size = _coconut.operator.index(size) if self.size < 1: raise _coconut.ValueError("windowsof: size must be >= 1; not %r" % (self.size,)) @@ -1178,7 +1184,7 @@ class groupsof(_coconut_has_iter): """ __slots__ = ("group_size", "fillvalue") def __new__(cls, n, iterable, fillvalue=_coconut_sentinel): - self = _coconut_has_iter.__new__(cls, iterable) + self = _coconut.super({_coconut_}groupsof, cls).__new__(cls, iterable) self.group_size = _coconut.operator.index(n) if self.group_size < 1: raise _coconut.ValueError("group size must be >= 1; not %r" % (self.group_size,)) @@ -1755,7 +1761,7 @@ class lift(_coconut_baseclass): """ __slots__ = ("func",) def __new__(cls, func, *func_args, **func_kwargs): - self = _coconut.object.__new__(cls) + self = _coconut.super({_coconut_}lift, cls).__new__(cls) self.func = func if func_args or func_kwargs: self = self(*func_args, **func_kwargs) @@ -1879,48 +1885,134 @@ def _coconut_call_or_coefficient(func, *args): func = func * x{COMMENT.no_times_equals_to_avoid_modification} return func class _coconut_SupportsAdd(_coconut.typing.Protocol): + """Coconut (+) Protocol. Equivalent to: + + class SupportsAdd[T, U, V](Protocol): + def __add__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __add__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((+) in a typing context is a Protocol)") class _coconut_SupportsMinus(_coconut.typing.Protocol): + """Coconut (-) Protocol. Equivalent to: + + class SupportsMinus[T, U, V](Protocol): + def __sub__(self: T, other: U) -> V: + raise NotImplementedError + def __neg__(self: T) -> V: + raise NotImplementedError + """ def __sub__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") def __neg__(self): raise NotImplementedError("Protocol methods cannot be called at runtime ((-) in a typing context is a Protocol)") class _coconut_SupportsMul(_coconut.typing.Protocol): + """Coconut (*) Protocol. Equivalent to: + + class SupportsMul[T, U, V](Protocol): + def __mul__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __mul__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((*) in a typing context is a Protocol)") class _coconut_SupportsPow(_coconut.typing.Protocol): + """Coconut (**) Protocol. Equivalent to: + + class SupportsPow[T, U, V](Protocol): + def __pow__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __pow__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((**) in a typing context is a Protocol)") class _coconut_SupportsTruediv(_coconut.typing.Protocol): + """Coconut (/) Protocol. Equivalent to: + + class SupportsTruediv[T, U, V](Protocol): + def __truediv__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __truediv__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((/) in a typing context is a Protocol)") class _coconut_SupportsFloordiv(_coconut.typing.Protocol): + """Coconut (//) Protocol. Equivalent to: + + class SupportsFloordiv[T, U, V](Protocol): + def __floordiv__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __floordiv__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((//) in a typing context is a Protocol)") class _coconut_SupportsMod(_coconut.typing.Protocol): + """Coconut (%) Protocol. Equivalent to: + + class SupportsMod[T, U, V](Protocol): + def __mod__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __mod__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((%) in a typing context is a Protocol)") class _coconut_SupportsAnd(_coconut.typing.Protocol): + """Coconut (&) Protocol. Equivalent to: + + class SupportsAnd[T, U, V](Protocol): + def __and__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __and__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((&) in a typing context is a Protocol)") class _coconut_SupportsXor(_coconut.typing.Protocol): + """Coconut (^) Protocol. Equivalent to: + + class SupportsXor[T, U, V](Protocol): + def __xor__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __xor__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((^) in a typing context is a Protocol)") class _coconut_SupportsOr(_coconut.typing.Protocol): + """Coconut (|) Protocol. Equivalent to: + + class SupportsOr[T, U, V](Protocol): + def __or__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __or__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((|) in a typing context is a Protocol)") class _coconut_SupportsLshift(_coconut.typing.Protocol): + """Coconut (<<) Protocol. Equivalent to: + + class SupportsLshift[T, U, V](Protocol): + def __lshift__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __lshift__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((<<) in a typing context is a Protocol)") class _coconut_SupportsRshift(_coconut.typing.Protocol): + """Coconut (>>) Protocol. Equivalent to: + + class SupportsRshift[T, U, V](Protocol): + def __rshift__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __rshift__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((>>) in a typing context is a Protocol)") class _coconut_SupportsMatmul(_coconut.typing.Protocol): + """Coconut (@) Protocol. Equivalent to: + + class SupportsMatmul[T, U, V](Protocol): + def __matmul__(self: T, other: U) -> V: + raise NotImplementedError(...) + """ def __matmul__(self, other): raise NotImplementedError("Protocol methods cannot be called at runtime ((@) in a typing context is a Protocol)") class _coconut_SupportsInv(_coconut.typing.Protocol): + """Coconut (~) Protocol. Equivalent to: + + class SupportsInv[T, V](Protocol): + def __invert__(self: T) -> V: + raise NotImplementedError(...) + """ def __invert__(self): raise NotImplementedError("Protocol methods cannot be called at runtime ((~) in a typing context is a Protocol)") _coconut_self_match_types = {self_match_types} -_coconut_Expected, _coconut_MatchError, _coconut_count, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_ident, _coconut_map, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_starmap, _coconut_tee, _coconut_zip, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, count, enumerate, flatten, filter, ident, map, multiset, range, reiterable, reversed, starmap, tee, zip, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile +_coconut_Expected, _coconut_MatchError, _coconut_cartesian_product, _coconut_count, _coconut_cycle, _coconut_enumerate, _coconut_flatten, _coconut_filter, _coconut_groupsof, _coconut_ident, _coconut_lift, _coconut_map, _coconut_multiset, _coconut_range, _coconut_reiterable, _coconut_reversed, _coconut_scan, _coconut_starmap, _coconut_tee, _coconut_windowsof, _coconut_zip, _coconut_zip_longest, TYPE_CHECKING, reduce, takewhile, dropwhile = Expected, MatchError, cartesian_product, count, cycle, enumerate, flatten, filter, groupsof, ident, lift, map, multiset, range, reiterable, reversed, scan, starmap, tee, windowsof, zip, zip_longest, False, _coconut.functools.reduce, _coconut.itertools.takewhile, _coconut.itertools.dropwhile{COMMENT.anything_added_here_should_be_copied_to_stub_file} diff --git a/coconut/root.py b/coconut/root.py index d23825cae..0ec9c1352 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 5 +DEVELOP = 6 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/main.coco b/coconut/tests/src/cocotest/agnostic/main.coco index f9a5a067d..2e5402122 100644 --- a/coconut/tests/src/cocotest/agnostic/main.coco +++ b/coconut/tests/src/cocotest/agnostic/main.coco @@ -56,6 +56,7 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: non_py26_test, non_py32_test, py3_spec_test, + py33_spec_test, py36_spec_test, py37_spec_test, py38_spec_test, @@ -66,6 +67,8 @@ def run_main(outer_MatchError, test_easter_eggs=False) -> bool: assert non_py32_test() is True if sys.version_info >= (3,): assert py3_spec_test() is True + if sys.version_info >= (3, 3): + assert py33_spec_test() is True if sys.version_info >= (3, 6): assert py36_spec_test(tco=using_tco) is True if sys.version_info >= (3, 7): diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 9d6763eea..1ccd7020f 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1590,4 +1590,8 @@ def primary_test() -> bool: assert {"a":0, "b":1}$[0] == "a" assert (|0, NotImplemented, 2|)$[1] is NotImplemented assert m{1, 1, 2} |> fmap$(.+1) == m{2, 2, 3} + assert (+) ..> ((*) ..> (/)) == (+) ..> (*) ..> (/) == ((+) ..> (*)) ..> (/) + def f(x, y=1) = x, y # type: ignore + f.is_f = True # type: ignore + assert (f ..*> (+)).is_f # type: ignore return True diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index 128f82dcd..9c936dddd 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -44,6 +44,19 @@ def py3_spec_test() -> bool: return True +def py33_spec_test() -> bool: + """Tests for any py33+ version.""" + from inspect import signature + def f(x, y=1) = x, y + def g(a, b=2) = a, b + assert signature(f ..*> g) == signature(f) == signature(f ..> g) + assert signature(f <*.. g) == signature(g) == signature(f <.. g) + assert signature(f$(0) ..> g) == signature(f$(0)) + assert signature(f ..*> (+)) == signature(f) + assert signature((f ..*> g) ..*> g) == signature(f) + return True + + def py36_spec_test(tco: bool) -> bool: """Tests for any py36+ version.""" from dataclasses import dataclass From 060c9a89a82b186535a09a27261fbb0817f0dca4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 19 May 2023 21:02:40 -0700 Subject: [PATCH 11/57] Add f(...=name) syntax Resolves #743. --- coconut/compiler/compiler.py | 4 ++++ coconut/compiler/grammar.py | 9 +++++---- coconut/root.py | 2 +- coconut/tests/src/cocotest/agnostic/primary.coco | 5 +++++ coconut/tests/src/cocotest/agnostic/suite.coco | 3 +++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 6f0ff640c..f3c7953aa 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2333,6 +2333,8 @@ def split_function_call(self, tokens, loc): star_args.append(argstr) elif arg[0] == "**": dubstar_args.append(argstr) + elif arg[0] == "...": + kwd_args.append(arg[1] + "=" + arg[1]) else: kwd_args.append(argstr) else: @@ -3043,6 +3045,8 @@ def anon_namedtuple_handle(self, tokens): types[i] = typedef else: raise CoconutInternalException("invalid anonymous named item", tok) + if name == "...": + name = item names.append(name) items.append(item) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 0c830210e..68d40d616 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1133,6 +1133,7 @@ class Grammar(object): dubstar + test | star + test | unsafe_name + default + | ellipsis_tokens + equals.suppress() + refname | namedexpr_test ) function_call_tokens = lparen.suppress() + ( @@ -1178,11 +1179,11 @@ class Grammar(object): subscriptgrouplist = itemlist(subscriptgroup, comma) anon_namedtuple = Forward() + maybe_typedef = Optional(colon.suppress() + typedef_test) anon_namedtuple_ref = tokenlist( Group( - unsafe_name - + Optional(colon.suppress() + typedef_test) - + equals.suppress() + test, + unsafe_name + maybe_typedef + equals.suppress() + test + | ellipsis_tokens + maybe_typedef + equals.suppress() + refname, ), comma, ) @@ -1288,8 +1289,8 @@ class Grammar(object): Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ | Group(condense(dollar + lbrack + rbrack)) # $[] | Group(condense(lbrack + rbrack)) # [] - | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . | Group(questionmark) # ? + | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . ) + ~questionmark partial_trailer = ( Group(fixto(dollar, "$(") + function_call) # $( diff --git a/coconut/root.py b/coconut/root.py index 0ec9c1352..7b420b7b7 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 6 +DEVELOP = 7 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 1ccd7020f..84f99c2e5 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1594,4 +1594,9 @@ def primary_test() -> bool: def f(x, y=1) = x, y # type: ignore f.is_f = True # type: ignore assert (f ..*> (+)).is_f # type: ignore + really_long_var = 10 + assert (...=really_long_var) == (10,) + assert (...=really_long_var, abc="abc") == (10, "abc") + assert (abc="abc", ...=really_long_var) == ("abc", 10) + assert (...=really_long_var).really_long_var == 10 return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index e7d47a2ff..b542db14e 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1047,6 +1047,9 @@ forward 2""") == 900 assert take_xy(xy("a", "b")) == ("a", "b") assert InitAndIter(range(3)) |> fmap$((.+1), fallback_to_init=True) == InitAndIter(range(1, 4)) assert_raises(-> InitAndIter(range(3)) |> fmap$(.+1), TypeError) + really_long_var = 10 + assert ret_args_kwargs(...=really_long_var) == ((), {"really_long_var": 10}) == ret_args_kwargs$(...=really_long_var)() + assert ret_args_kwargs(123, ...=really_long_var, abc="abc") == ((123,), {"really_long_var": 10, "abc": "abc"}) == ret_args_kwargs$(123, ...=really_long_var, abc="abc")() # must come at end assert fibs_calls[0] == 1 From 77b07b1a3a285106596dd1eb5679771bcaff2811 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 19 May 2023 21:10:28 -0700 Subject: [PATCH 12/57] Document kwd arg name elision --- DOCS.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/DOCS.md b/DOCS.md index 44c83decd..07e175266 100644 --- a/DOCS.md +++ b/DOCS.md @@ -2059,6 +2059,41 @@ print(p1(5)) quad = 5 * x**2 + 3 * x + 1 ``` +### Keyword Argument Name Elision + +When passing in long variable names as keyword arguments of the same name, Coconut supports the syntax +``` +f(...=long_variable_name) +``` +as a shorthand for +``` +f(long_variable_name=long_variable_name) +``` + +Such syntax is also supported in [partial application](#partial-application) and [anonymous `namedtuple`s](#anonymous-namedtuples). + +##### Example + +**Coconut:** +```coconut +really_long_variable_name_1 = get_1() +really_long_variable_name_2 = get_2() +main_func( + ...=really_long_variable_name_1, + ...=really_long_variable_name_2, +) +``` + +**Python:** +```coconut_python +really_long_variable_name_1 = get_1() +really_long_variable_name_2 = get_2() +main_func( + really_long_variable_name_1=really_long_variable_name_1, + really_long_variable_name_2=really_long_variable_name_2, +) +``` + ### Anonymous Namedtuples Coconut supports anonymous [`namedtuple`](https://docs.python.org/3/library/collections.html#collections.namedtuple) literals, such that `(a=1, b=2)` can be used just as `(1, 2)`, but with added names. Anonymous `namedtuple`s are always pickleable. @@ -2069,6 +2104,8 @@ The syntax for anonymous namedtuple literals is: ``` where, if `` is given for any field, [`typing.NamedTuple`](https://docs.python.org/3/library/typing.html#typing.NamedTuple) is used instead of `collections.namedtuple`. +Anonymous `namedtuple`s also support [keyword argument name elision](#keyword-argument-name-elision). + ##### `_namedtuple_of` On Python versions `>=3.6`, `_namedtuple_of` is provided as a built-in that can mimic the behavior of anonymous namedtuple literals such that `_namedtuple_of(a=1, b=2)` is equivalent to `(a=1, b=2)`. Since `_namedtuple_of` is only available on Python 3.6 and above, however, it is generally recommended to use anonymous namedtuple literals instead, as they work on any Python version. From 355014638526ccb0cb98978f6f02132b45485959 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 19 May 2023 21:19:52 -0700 Subject: [PATCH 13/57] Reduce appveyor testing --- coconut/tests/main_test.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index d73a33d0b..444228b19 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -848,14 +848,6 @@ def test_simple_minify(self): @add_test_func_names class TestExternal(unittest.TestCase): - # more appveyor timeout prevention - if not (WINDOWS and PY2): - def test_pyprover(self): - with using_path(pyprover): - comp_pyprover() - if PY38: - run_pyprover() - if not PYPY or PY2: def test_prelude(self): with using_path(prelude): @@ -869,11 +861,19 @@ def test_bbopt(self): if not PYPY and PY38 and not PY310: install_bbopt() - def test_pyston(self): - with using_path(pyston): - comp_pyston(["--no-tco"]) - if PYPY and PY2: - run_pyston() + # more appveyor timeout prevention + if not WINDOWS: + def test_pyprover(self): + with using_path(pyprover): + comp_pyprover() + if PY38: + run_pyprover() + + def test_pyston(self): + with using_path(pyston): + comp_pyston(["--no-tco"]) + if PYPY and PY2: + run_pyston() # ----------------------------------------------------------------------------------------------------------------------- From 10679813eefba5a9ad4c23a9265ac9ba90887219 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 19 May 2023 21:46:59 -0700 Subject: [PATCH 14/57] Improve fmap docs --- DOCS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 07e175266..c5b9199df 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3198,9 +3198,9 @@ _Can't be done without a series of method definitions for each data type. See th #### `fmap` -**fmap**(_func_, _obj_, *, _starmap\_over\_mappings_=`False`) +**fmap**(_func_, _obj_) -In Haskell, `fmap(func, obj)` takes a data type `obj` and returns a new data type with `func` mapped over the contents. Coconut's `fmap` function does the exact same thing in Coconut. +In Haskell, `fmap(func, obj)` takes a data type `obj` and returns a new data type with `func` mapped over the contents. Coconut's `fmap` function does the exact same thing for Coconut's [data types](#data). `fmap` can also be used on built-ins such as `str`, `list`, `set`, and `dict` as a variant of `map` that returns back an object of the same type. The behavior of `fmap` for a given object can be overridden by defining an `__fmap__(self, func)` magic method that will be called whenever `fmap` is invoked on that object. Note that `__fmap__` implementations should always satisfy the [Functor Laws](https://wiki.haskell.org/Functor). @@ -3218,6 +3218,8 @@ async def fmap_over_async_iters(func, async_iter): ``` such that `fmap` can effectively be used as an async map. +_DEPRECATED: `fmap(func, obj, fallback_to_init=True)` will fall back to `obj.__class__(map(func, obj))` if no `fmap` implementation is available rather than raise `TypeError`._ + ##### Example **Coconut:** From 73d6e3bc79f7c9759a8279b5d6131d692fc1792f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 21 May 2023 21:02:33 -0700 Subject: [PATCH 15/57] Fix xonsh line lookup Resolves #745. --- coconut/command/util.py | 4 +-- coconut/compiler/compiler.py | 1 + coconut/compiler/util.py | 15 +++++++++++ coconut/constants.py | 2 +- coconut/icoconut/root.py | 27 +++---------------- coconut/integrations.py | 52 ++++++++++++++++++++++++++++++++---- coconut/root.py | 2 +- coconut/util.py | 23 ++++++++++++++++ 8 files changed, 93 insertions(+), 33 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index 3f60c82d1..85fdaa404 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -133,8 +133,8 @@ # ----------------------------------------------------------------------------------------------------------------------- -memoized_isdir = memoize(128)(os.path.isdir) -memoized_isfile = memoize(128)(os.path.isfile) +memoized_isdir = memoize(64)(os.path.isdir) +memoized_isfile = memoize(64)(os.path.isfile) def writefile(openedfile, newcontents): diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index f3c7953aa..9a7ba1bd6 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1536,6 +1536,7 @@ def ln_comment(self, ln): else: lni = ln - 1 + # line number must be at start of comment for extract_line_num_from_comment if self.line_numbers and self.keep_lines: if self.minify: comment = str(ln) + " " + self.kept_lines[lni] diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 126136543..035d08268 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -1040,6 +1040,21 @@ def split_comment(line, move_indents=False): return line[:i] + indent, line[i:] +def extract_line_num_from_comment(line, default=None): + """Extract the line number from a line with a line number comment, else return default.""" + _, all_comments = split_comment(line) + for comment in all_comments.split("#"): + words = comment.strip().split(None, 1) + if words: + first_word = words[0].strip(":") + try: + return int(first_word) + except ValueError: + pass + logger.log("failed to extract line num comment from", line) + return default + + def rem_comment(line): """Remove a comment from a line.""" base, comment = split_comment(line) diff --git a/coconut/constants.py b/coconut/constants.py index ecf359496..99711adcd 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1141,7 +1141,7 @@ def get_bool_env_var(env_var, default=False): # INTEGRATION CONSTANTS: # ----------------------------------------------------------------------------------------------------------------------- -# must be replicated in DOCS +# must be replicated in DOCS; must include --line-numbers for xonsh line number extraction coconut_kernel_kwargs = dict(target="sys", line_numbers=True, keep_lines=True, no_wrap=True) icoconut_dir = os.path.join(base_dir, "icoconut") diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 45fc6f1ad..6e2c1eb60 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -44,11 +44,8 @@ conda_build_env_var, coconut_kernel_kwargs, ) -from coconut.terminal import ( - logger, - internal_assert, -) -from coconut.util import override +from coconut.terminal import logger +from coconut.util import override, memoize_with_exceptions from coconut.compiler import Compiler from coconut.compiler.util import should_indent from coconut.command.util import Runner @@ -94,25 +91,7 @@ RUNNER = Runner(COMPILER) -parse_block_memo = {} - - -def memoized_parse_block(code): - """Memoized version of parse_block.""" - internal_assert(lambda: code not in parse_block_memo.values(), "attempted recompilation of", code) - success, result = parse_block_memo.get(code, (None, None)) - if success is None: - try: - parsed = COMPILER.parse_block(code, keep_state=True) - except Exception as err: - success, result = False, err - else: - success, result = True, parsed - parse_block_memo[code] = (success, result) - if success: - return result - else: - raise result +memoized_parse_block = memoize_with_exceptions()(COMPILER.parse_block) def syntaxerr_memoized_parse_block(code): diff --git a/coconut/integrations.py b/coconut/integrations.py index bbed00a40..6104dfc4d 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -25,6 +25,7 @@ coconut_kernel_kwargs, disabled_xonsh_modes, ) +from coconut.util import memoize_with_exceptions # ----------------------------------------------------------------------------------------------------------------------- # IPYTHON: @@ -94,6 +95,14 @@ class CoconutXontribLoader(object): runner = None timing_info = [] + @memoize_with_exceptions() + def _base_memoized_parse_xonsh(self, code, **kwargs): + return self.compiler.parse_xonsh(code, **kwargs) + + def memoized_parse_xonsh(self, code): + """Memoized self.compiler.parse_xonsh.""" + return self._base_memoized_parse_xonsh(code.strip(), keep_state=True) + def new_parse(self, parser, code, mode="exec", *args, **kwargs): """Coconut-aware version of xonsh's _parse.""" if self.loaded and mode not in disabled_xonsh_modes: @@ -106,7 +115,7 @@ def new_parse(self, parser, code, mode="exec", *args, **kwargs): parse_start_time = get_clock_time() quiet, logger.quiet = logger.quiet, True try: - code = self.compiler.parse_xonsh(code, keep_state=True) + code = self.memoized_parse_xonsh(code) except CoconutException as err: err_str = format_error(err).splitlines()[0] code += " #" + err_str @@ -115,17 +124,49 @@ def new_parse(self, parser, code, mode="exec", *args, **kwargs): self.timing_info.append(("parse", get_clock_time() - parse_start_time)) return parser.__class__.parse(parser, code, mode=mode, *args, **kwargs) - def new_try_subproc_toks(self, ctxtransformer, *args, **kwargs): + def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): """Version of try_subproc_toks that handles the fact that Coconut code may have different columns than Python code.""" mode = ctxtransformer.mode if self.loaded: ctxtransformer.mode = "eval" try: - return ctxtransformer.__class__.try_subproc_toks(ctxtransformer, *args, **kwargs) + return ctxtransformer.__class__.try_subproc_toks(ctxtransformer, node, *args, **kwargs) finally: ctxtransformer.mode = mode + def new_ctxvisit(self, ctxtransformer, node, inp, *args, **kwargs): + """Version of ctxvisit that ensures looking up original lines in inp + using Coconut line numbers will work properly.""" + if self.loaded: + from xonsh.tools import get_logical_line + + # hide imports to avoid circular dependencies + from coconut.terminal import logger + from coconut.compiler.util import extract_line_num_from_comment + + compiled = self.memoized_parse_xonsh(inp) + + original_lines = tuple(inp.splitlines()) + used_lines = set() + new_inp_lines = [] + last_ln = 1 + for compiled_line in compiled.splitlines(): + ln = extract_line_num_from_comment(compiled_line, default=last_ln + 1) + try: + line, _, _ = get_logical_line(original_lines, ln - 1) + except IndexError: + logger.log_exc() + line = original_lines[-1] + if line in used_lines: + line = "\n" + else: + used_lines.add(line) + new_inp_lines.append(line) + last_ln = ln + inp = "\n".join(new_inp_lines) + return ctxtransformer.__class__.ctxvisit(ctxtransformer, node, inp, *args, **kwargs) + def __call__(self, xsh, **kwargs): # hide imports to avoid circular dependencies from coconut.util import get_clock_time @@ -147,11 +188,12 @@ def __call__(self, xsh, **kwargs): main_parser.parse = MethodType(self.new_parse, main_parser) ctxtransformer = xsh.execer.ctxtransformer + ctxtransformer.try_subproc_toks = MethodType(self.new_try_subproc_toks, ctxtransformer) + ctxtransformer.ctxvisit = MethodType(self.new_ctxvisit, ctxtransformer) + ctx_parser = ctxtransformer.parser ctx_parser.parse = MethodType(self.new_parse, ctx_parser) - ctxtransformer.try_subproc_toks = MethodType(self.new_try_subproc_toks, ctxtransformer) - self.timing_info.append(("load", get_clock_time() - start_time)) self.loaded = True diff --git a/coconut/root.py b/coconut/root.py index 7b420b7b7..a8051ea2d 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 7 +DEVELOP = 8 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/util.py b/coconut/util.py index 216d0e4e3..98489f5b4 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -205,12 +205,35 @@ def noop_ctx(): def memoize(maxsize=None, *args, **kwargs): """Decorator that memoizes a function, preventing it from being recomputed if it is called multiple times with the same arguments.""" + assert maxsize is None or isinstance(maxsize, int), maxsize if lru_cache is None: return lambda func: func else: return lru_cache(maxsize, *args, **kwargs) +def memoize_with_exceptions(*memo_args, **memo_kwargs): + """Decorator that works like memoize but also memoizes exceptions.""" + def memoizer(func): + @memoize(*memo_args, **memo_kwargs) + def memoized_safe_func(*args, **kwargs): + res = exc = None + try: + res = func(*args, **kwargs) + except Exception as exc: + return res, exc + else: + return res, exc + + def memoized_func(*args, **kwargs): + res, exc = memoized_safe_func(*args, **kwargs) + if exc is not None: + raise exc + return res + return memoized_func + return memoizer + + class keydefaultdict(defaultdict, object): """Version of defaultdict that calls the factory with the key.""" From 654e26ec5a6ce05983519a92a8ac734094e21684 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 21 May 2023 21:16:17 -0700 Subject: [PATCH 16/57] Further improve xonsh line handling --- coconut/icoconut/root.py | 2 +- coconut/integrations.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 6e2c1eb60..a4ccb480f 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -91,7 +91,7 @@ RUNNER = Runner(COMPILER) -memoized_parse_block = memoize_with_exceptions()(COMPILER.parse_block) +memoized_parse_block = memoize_with_exceptions(128)(COMPILER.parse_block) def syntaxerr_memoized_parse_block(code): diff --git a/coconut/integrations.py b/coconut/integrations.py index 6104dfc4d..3eeaf9c47 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -95,7 +95,7 @@ class CoconutXontribLoader(object): runner = None timing_info = [] - @memoize_with_exceptions() + @memoize_with_exceptions(128) def _base_memoized_parse_xonsh(self, code, **kwargs): return self.compiler.parse_xonsh(code, **kwargs) @@ -159,12 +159,12 @@ def new_ctxvisit(self, ctxtransformer, node, inp, *args, **kwargs): logger.log_exc() line = original_lines[-1] if line in used_lines: - line = "\n" + line = "" else: used_lines.add(line) new_inp_lines.append(line) last_ln = ln - inp = "\n".join(new_inp_lines) + inp = "\n".join(new_inp_lines) + "\n" return ctxtransformer.__class__.ctxvisit(ctxtransformer, node, inp, *args, **kwargs) def __call__(self, xsh, **kwargs): From 480f6dcdbaab05a03b86759fa5f31b918474cab5 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 21 May 2023 23:00:30 -0700 Subject: [PATCH 17/57] Ensure existence of kernel logger --- coconut/icoconut/root.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index a4ccb480f..799084415 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -262,6 +262,11 @@ class CoconutKernel(IPythonKernel, object): }, ] + def __init__(self, *args, **kwargs): + super(CoconutKernel, self).__init__(*args, **kwargs) + if self.log is None: + self.log = logger + @override def do_complete(self, code, cursor_pos): # first try with Jedi completions From 6379f2319dce344b1466a06248705b2fb51ca499 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 22 May 2023 01:26:48 -0700 Subject: [PATCH 18/57] Fix kernel trait error --- coconut/icoconut/root.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index 799084415..a078a08f3 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -21,6 +21,7 @@ import os import sys +import logging try: import asyncio @@ -265,7 +266,7 @@ class CoconutKernel(IPythonKernel, object): def __init__(self, *args, **kwargs): super(CoconutKernel, self).__init__(*args, **kwargs) if self.log is None: - self.log = logger + self.log = logging.getLogger(__name__) @override def do_complete(self, code, cursor_pos): From ffaa37112e12b66d7b8b1de960f5048af517c3d2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 22 May 2023 18:09:00 -0700 Subject: [PATCH 19/57] Fix syntax error reconstruction --- coconut/exceptions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/coconut/exceptions.py b/coconut/exceptions.py index c49429cf0..3f37a6d0c 100644 --- a/coconut/exceptions.py +++ b/coconut/exceptions.py @@ -187,14 +187,16 @@ def syntax_err(self): if self.point_to_endpoint and "endpoint" in kwargs: point = kwargs.pop("endpoint") else: - point = kwargs.pop("point") + point = kwargs.pop("point", None) kwargs["point"] = kwargs["endpoint"] = None - ln = kwargs.pop("ln") + ln = kwargs.pop("ln", None) filename = kwargs.pop("filename", None) err = SyntaxError(self.message(**kwargs)) - err.offset = point - err.lineno = ln + if point is not None: + err.offset = point + if ln is not None: + err.lineno = ln if filename is not None: err.filename = filename return err From a77d141a14a96010b539f35d6cc594ed38e122b6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 22 May 2023 20:03:14 -0700 Subject: [PATCH 20/57] Further fix syntax error conversion --- coconut/exceptions.py | 12 ++++++------ coconut/root.py | 2 +- coconut/tests/src/extras.coco | 9 +++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/coconut/exceptions.py b/coconut/exceptions.py index 3f37a6d0c..33e0c40b4 100644 --- a/coconut/exceptions.py +++ b/coconut/exceptions.py @@ -97,7 +97,7 @@ def __init__(self, message, source=None, point=None, ln=None, extra=None, endpoi @property def kwargs(self): """Get the arguments as keyword arguments.""" - return dict(zip(self.args, self.argnames)) + return dict(zip(self.argnames, self.args)) def message(self, message, source, point, ln, extra=None, endpoint=None, filename=None): """Creates a SyntaxError-like message.""" @@ -185,12 +185,12 @@ def syntax_err(self): """Creates a SyntaxError.""" kwargs = self.kwargs if self.point_to_endpoint and "endpoint" in kwargs: - point = kwargs.pop("endpoint") + point = kwargs["endpoint"] else: - point = kwargs.pop("point", None) - kwargs["point"] = kwargs["endpoint"] = None - ln = kwargs.pop("ln", None) - filename = kwargs.pop("filename", None) + point = kwargs.get("point") + ln = kwargs.get("ln") + filename = kwargs.get("filename") + kwargs["point"] = kwargs["endpoint"] = kwargs["ln"] = kwargs["filename"] = None err = SyntaxError(self.message(**kwargs)) if point is not None: diff --git a/coconut/root.py b/coconut/root.py index a8051ea2d..d7fcb0fdc 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 8 +DEVELOP = 9 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index a94313b5a..2ac4d8ede 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -51,8 +51,13 @@ def assert_raises(c, exc, not_exc=None, err_has=None): assert any(has in str(err) for has in err_has), f"{str(err)!r} does not contain any of {err_has!r}" else: assert err_has in str(err), f"{err_has!r} not in {str(err)!r}" - if exc `isinstance` CoconutSyntaxError: - assert "SyntaxError" in str(exc.syntax_err()) + if err `isinstance` CoconutSyntaxError: + syntax_err = err.syntax_err() + assert syntax_err `isinstance` SyntaxError + syntax_err_str = str(syntax_err) + assert syntax_err_str.splitlines()$[0] in str(err), (syntax_err_str, str(err)) + assert "unprintable" not in syntax_err_str, syntax_err_str + assert " Date: Mon, 22 May 2023 22:49:19 -0700 Subject: [PATCH 21/57] Fix kernel errors --- coconut/icoconut/root.py | 5 ++++- coconut/integrations.py | 7 ++++--- coconut/root.py | 2 +- coconut/tests/src/extras.coco | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/coconut/icoconut/root.py b/coconut/icoconut/root.py index a078a08f3..326a2dd62 100644 --- a/coconut/icoconut/root.py +++ b/coconut/icoconut/root.py @@ -92,7 +92,10 @@ RUNNER = Runner(COMPILER) -memoized_parse_block = memoize_with_exceptions(128)(COMPILER.parse_block) + +@memoize_with_exceptions(128) +def memoized_parse_block(code): + return COMPILER.parse_block(code, keep_state=True) def syntaxerr_memoized_parse_block(code): diff --git a/coconut/integrations.py b/coconut/integrations.py index 3eeaf9c47..7636e1e7e 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -96,12 +96,13 @@ class CoconutXontribLoader(object): timing_info = [] @memoize_with_exceptions(128) - def _base_memoized_parse_xonsh(self, code, **kwargs): - return self.compiler.parse_xonsh(code, **kwargs) + def _base_memoized_parse_xonsh(self, code): + return self.compiler.parse_xonsh(code, keep_state=True) def memoized_parse_xonsh(self, code): """Memoized self.compiler.parse_xonsh.""" - return self._base_memoized_parse_xonsh(code.strip(), keep_state=True) + # .strip() outside the memoization + return self._base_memoized_parse_xonsh(code.strip()) def new_parse(self, parser, code, mode="exec", *args, **kwargs): """Coconut-aware version of xonsh's _parse.""" diff --git a/coconut/root.py b/coconut/root.py index d7fcb0fdc..17ec16085 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 9 +DEVELOP = 10 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 2ac4d8ede..49cdbb44a 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -368,8 +368,8 @@ def test_kernel() -> bool: exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) |> unwrap_future$(loop) assert exec_result["status"] == "ok" assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2" - assert k.do_execute("operator ++", False, True, {}, True) |> unwrap_future$(loop) - assert k.do_execute("(++) = 1", False, True, {}, True) |> unwrap_future$(loop) + assert k.do_execute("operator ++", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" + assert k.do_execute("(++) = 1", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" assert k.do_is_complete("if abc:")["status"] == "incomplete" assert k.do_is_complete("f(")["status"] == "incomplete" assert k.do_is_complete("abc")["status"] == "complete" From 9f4a2a58de90c612ace54040329c287c32ed0337 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 22 May 2023 23:17:17 -0700 Subject: [PATCH 22/57] Standardize caseless literals --- coconut/compiler/grammar.py | 26 +++++++++---------- coconut/compiler/util.py | 9 +++++++ .../tests/src/cocotest/agnostic/primary.coco | 2 ++ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 68d40d616..5099a2de5 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -32,7 +32,6 @@ from functools import partial from coconut._pyparsing import ( - CaselessLiteral, Forward, Group, Literal, @@ -115,6 +114,7 @@ boundary, compile_regex, always_match, + caseless_literal, ) @@ -798,17 +798,17 @@ class Grammar(object): octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) - imag_j = CaselessLiteral("j") | fixto(CaselessLiteral("i"), "j") + imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") basenum = combine( integer + dot + Optional(integer) | Optional(integer) + dot + integer, ) | integer - sci_e = combine(CaselessLiteral("e") + Optional(plus | neg_minus)) + sci_e = combine(caseless_literal("e") + Optional(plus | neg_minus)) numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) imag_num = combine(numitem + imag_j) - bin_num = combine(CaselessLiteral("0b") + Optional(underscore.suppress()) + binint) - oct_num = combine(CaselessLiteral("0o") + Optional(underscore.suppress()) + octint) - hex_num = combine(CaselessLiteral("0x") + Optional(underscore.suppress()) + hexint) + bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) + oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) + hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) number = ( bin_num | oct_num @@ -848,10 +848,10 @@ class Grammar(object): u_string = Forward() f_string = Forward() - bit_b = CaselessLiteral("b") - raw_r = CaselessLiteral("r") - unicode_u = CaselessLiteral("u").suppress() - format_f = CaselessLiteral("f").suppress() + bit_b = caseless_literal("b") + raw_r = caseless_literal("r") + unicode_u = caseless_literal("u", suppress=True) + format_f = caseless_literal("f", suppress=True) string = combine(Optional(raw_r) + string_item) # Python 2 only supports br"..." not rb"..." @@ -1236,9 +1236,9 @@ class Grammar(object): set_literal = Forward() set_letter_literal = Forward() - set_s = fixto(CaselessLiteral("s"), "s") - set_f = fixto(CaselessLiteral("f"), "f") - set_m = fixto(CaselessLiteral("m"), "m") + set_s = caseless_literal("s") + set_f = caseless_literal("f") + set_m = caseless_literal("m") set_letter = set_s | set_f | set_m setmaker = Group( (new_namedexpr_test + FollowedBy(rbrace))("test") diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index 035d08268..e6de4537f 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -53,6 +53,7 @@ Regex, Empty, Literal, + CaselessLiteral, Group, ParserElement, _trim_arity, @@ -886,6 +887,14 @@ def any_len_perm_at_least_one(*elems, **kwargs): return any_len_perm_with_one_of_each_group(*groups_and_elems) +def caseless_literal(literalstr, suppress=False): + """Version of CaselessLiteral that always parses to the given literalstr.""" + if suppress: + return CaselessLiteral(literalstr).suppress() + else: + return fixto(CaselessLiteral(literalstr), literalstr) + + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 84f99c2e5..453106920 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1599,4 +1599,6 @@ def primary_test() -> bool: assert (...=really_long_var, abc="abc") == (10, "abc") assert (abc="abc", ...=really_long_var) == ("abc", 10) assert (...=really_long_var).really_long_var == 10 + n = [0] + assert n[0] == 0 return True From 363fe2d2345b54ab2c93b664b65bde84026da62b Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 23 May 2023 22:08:22 -0700 Subject: [PATCH 23/57] Bump reqs, improve tests --- coconut/constants.py | 8 +++--- coconut/root.py | 2 +- coconut/tests/src/extras.coco | 46 ++++++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 99711adcd..8ce53c0e0 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -908,17 +908,17 @@ def get_bool_env_var(env_var, default=False): "argparse": (1, 4), "pexpect": (4,), ("trollius", "py2;cpy"): (2, 2), - "requests": (2, 29), + "requests": (2, 31), ("numpy", "py34"): (1,), ("numpy", "py2;cpy"): (1,), ("dataclasses", "py==36"): (0, 8), ("aenum", "py<34"): (3,), "pydata-sphinx-theme": (0, 13), "myst-parser": (1,), - "mypy[python2]": (1, 2), - ("jupyter-console", "py37"): (6,), + "mypy[python2]": (1, 3), + ("jupyter-console", "py37"): (6, 6), ("typing", "py<35"): (3, 10), - ("typing_extensions", "py37"): (4, 5), + ("typing_extensions", "py37"): (4, 6), ("ipython", "py38"): (8,), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 18), diff --git a/coconut/root.py b/coconut/root.py index 17ec16085..541a7b962 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.0" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 10 +DEVELOP = 11 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 49cdbb44a..ad11e0815 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -29,23 +29,25 @@ if IPY: if PY35: import asyncio from coconut.icoconut import CoconutKernel # type: ignore + from jupyter_client.session import Session else: CoconutKernel = None # type: ignore + Session = object # type: ignore -def assert_raises(c, exc, not_exc=None, err_has=None): - """Test whether callable c raises an exception of type exc.""" - if not_exc is None and exc is CoconutSyntaxError: - not_exc = CoconutParseError +def assert_raises(c, Exc, not_Exc=None, err_has=None): + """Test whether callable c raises an exception of type Exc.""" + if not_Exc is None and Exc is CoconutSyntaxError: + not_Exc = CoconutParseError # we don't check err_has without the computation graph since errors can be quite different if not USE_COMPUTATION_GRAPH: err_has = None try: c() - except exc as err: - if not_exc is not None: - assert not isinstance(err, not_exc), f"{err} instance of {not_exc}" + except Exc as err: + if not_Exc is not None: + assert not isinstance(err, not_Exc), f"{err} instance of {not_Exc}" if err_has is not None: if isinstance(err_has, tuple): assert any(has in str(err) for has in err_has), f"{str(err)!r} does not contain any of {err_has!r}" @@ -59,9 +61,9 @@ def assert_raises(c, exc, not_exc=None, err_has=None): assert "unprintable" not in syntax_err_str, syntax_err_str assert " bool: assert_raises((def -> import \(_coconut)), ImportError, err_has="should never be done at runtime") # NOQA assert_raises((def -> import \_coconut), ImportError, err_has="should never be done at runtime") # NOQA @@ -364,30 +372,50 @@ def test_kernel() -> bool: asyncio.set_event_loop(loop) else: loop = None # type: ignore + k = CoconutKernel() + fake_session = FakeSession() + k.shell.displayhook.session = fake_session + exec_result = k.do_execute("derp = pow$(?, 2)", False, True, {"two": "(+)(1, 1)"}, True) |> unwrap_future$(loop) assert exec_result["status"] == "ok" assert exec_result["user_expressions"]["two"]["data"]["text/plain"] == "2" + assert k.do_execute("operator ++", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" assert k.do_execute("(++) = 1", False, True, {}, True) |> unwrap_future$(loop) |> .["status"] == "ok" + + fail_result = k.do_execute("f([] {})", False, True, {}, True) |> unwrap_future$(loop) + captured_msg_type, captured_msg_content = fake_session.captured_messages[-1] + assert fail_result["status"] == "error" == captured_msg_type, fail_result + assert fail_result["ename"] == "SyntaxError" == captured_msg_content["ename"], fail_result + assert fail_result["traceback"] == captured_msg_content["traceback"], fail_result + assert len(fail_result["traceback"]) == 1, fail_result + assert "parsing failed" in fail_result["traceback"][0], fail_result + assert fail_result["evalue"] == captured_msg_content["evalue"], fail_result + assert "parsing failed" in fail_result["evalue"], fail_result + assert k.do_is_complete("if abc:")["status"] == "incomplete" assert k.do_is_complete("f(")["status"] == "incomplete" assert k.do_is_complete("abc")["status"] == "complete" + inspect_result = k.do_inspect("derp", 4, 0) assert inspect_result["status"] == "ok" assert inspect_result["found"] assert inspect_result["data"]["text/plain"] + complete_result = k.do_complete("der", 1) assert complete_result["status"] == "ok" assert "derp" in complete_result["matches"] assert complete_result["cursor_start"] == 0 assert complete_result["cursor_end"] == 1 + keyword_complete_result = k.do_complete("ma", 1) assert keyword_complete_result["status"] == "ok" assert "match" in keyword_complete_result["matches"] assert "map" in keyword_complete_result["matches"] assert keyword_complete_result["cursor_start"] == 0 assert keyword_complete_result["cursor_end"] == 1 + return True From 54341f32456ca1992becab0bc551ea140b7dc626 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Tue, 23 May 2023 22:16:50 -0700 Subject: [PATCH 24/57] Fix py37 --- coconut/constants.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 8ce53c0e0..f09255863 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -832,7 +832,8 @@ def get_bool_env_var(env_var, default=False): ), "kernel": ( ("ipython", "py2"), - ("ipython", "py3;py<38"), + ("ipython", "py3;py<37"), + ("ipython", "py==37"), ("ipython", "py38"), ("ipykernel", "py2"), ("ipykernel", "py3;py<38"), @@ -928,13 +929,15 @@ def get_bool_env_var(env_var, default=False): # don't upgrade until myst-parser supports the new version "sphinx": (6,), - # don't upgrade this; it breaks on Python 3.6 + # don't upgrade this; it breaks on Python 3.7 + ("ipython", "py==37"): (7, 34), + # don't upgrade these; it breaks on Python 3.6 ("pandas", "py36"): (1,), ("jupyter-client", "py36"): (7, 1, 2), ("typing_extensions", "py==36"): (4, 1), # don't upgrade these; they break on Python 3.5 ("ipykernel", "py3;py<38"): (5, 5), - ("ipython", "py3;py<38"): (7, 9), + ("ipython", "py3;py<37"): (7, 9), ("jupyter-console", "py>=35;py<37"): (6, 1), ("jupyter-client", "py==35"): (6, 1, 12), ("jupytext", "py3"): (1, 8), @@ -967,12 +970,13 @@ def get_bool_env_var(env_var, default=False): # should match the reqs with comments above pinned_reqs = ( "sphinx", + ("ipython", "py==37"), ("pandas", "py36"), ("jupyter-client", "py36"), ("typing_extensions", "py==36"), ("jupyter-client", "py<35"), ("ipykernel", "py3;py<38"), - ("ipython", "py3;py<38"), + ("ipython", "py3;py<37"), ("jupyter-console", "py>=35;py<37"), ("jupyter-client", "py==35"), ("jupytext", "py3"), @@ -1005,7 +1009,7 @@ def get_bool_env_var(env_var, default=False): ("prompt_toolkit", "mark2"): _, ("jedi", "py<39"): _, ("pywinpty", "py2;windows"): _, - ("ipython", "py3;py<38"): _, + ("ipython", "py3;py<37"): _, } classifiers = ( From a36b31432b1c6f115580268c84a3bc2efc1c58be Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 17:16:58 -0700 Subject: [PATCH 25/57] Fix --no-wrap test --- coconut/tests/src/cocotest/non_strict/non_strict_test.coco | 2 +- coconut/tests/src/extras.coco | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco index 099e0dad2..33bea2e47 100644 --- a/coconut/tests/src/cocotest/non_strict/non_strict_test.coco +++ b/coconut/tests/src/cocotest/non_strict/non_strict_test.coco @@ -45,7 +45,7 @@ def non_strict_test() -> bool: assert False match A.CONST in 11: # type: ignore assert False - assert A.CONST == 10 + assert A.CONST == 10 == A.("CONST") match {"a": 1, "b": 2}: # type: ignore case {"a": a}: pass diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index ad11e0815..6411ff8a2 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -82,7 +82,10 @@ def unwrap_future(event_loop, maybe_future): class FakeSession(Session): - captured_messages: list[tuple] = [] + if TYPE_CHECKING: + captured_messages: list[tuple] = [] + else: + captured_messages: list = [] def send(self, stream, msg_or_type, content, *args, **kwargs): self.captured_messages.append((msg_or_type, content)) From deb13b1b8697e29cc75670825f8aeba2630d0b4f Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 17:27:10 -0700 Subject: [PATCH 26/57] Prepare for v3.0.1 release --- coconut/root.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/root.py b/coconut/root.py index 541a7b962..712f277b0 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -23,10 +23,10 @@ # VERSION: # ----------------------------------------------------------------------------------------------------------------------- -VERSION = "3.0.0" +VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 11 +DEVELOP = False ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 4172380cebc0437e9510004be8da4155d8d57ff1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 17:28:42 -0700 Subject: [PATCH 27/57] Improve docs on coconut-develop --- DOCS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DOCS.md b/DOCS.md index c5b9199df..5fe901fc0 100644 --- a/DOCS.md +++ b/DOCS.md @@ -108,6 +108,8 @@ pip install coconut-develop ``` which will install the most recent working version from Coconut's [`develop` branch](https://github.com/evhub/coconut/tree/develop). Optional dependency installation is supported in the same manner as above. For more information on the current development build, check out the [development version of this documentation](http://coconut.readthedocs.io/en/develop/DOCS.html). Be warned: `coconut-develop` is likely to be unstable—if you find a bug, please report it by [creating a new issue](https://github.com/evhub/coconut/issues/new). +_Note: if you have an existing release version of `coconut` installed, you'll need to `pip uninstall coconut` before installing `coconut-develop`._ + ## Compilation ```{contents} From 3ba8e318ccfa75266654364f6f80eb88726898f4 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 18:05:50 -0700 Subject: [PATCH 28/57] Improve unicode operator docs --- DOCS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DOCS.md b/DOCS.md index 5fe901fc0..34cd5eac3 100644 --- a/DOCS.md +++ b/DOCS.md @@ -878,6 +878,8 @@ Custom operators will often need to be surrounded by whitespace (or parentheses If a custom operator that is also a valid name is desired, you can use a backslash before the name to get back the name instead using Coconut's [keyword/variable disambiguation syntax](#handling-keywordvariable-name-overlap). +_Note: redefining existing Coconut operators using custom operator definition syntax is forbidden, including Coconut's built-in [Unicode operator alternatives](#unicode-alternatives)._ + ##### Examples **Coconut:** @@ -1034,6 +1036,8 @@ class CanAddAndSub(Protocol, Generic[T, U, V]): Coconut supports Unicode alternatives to many different operator symbols. The Unicode alternatives are relatively straightforward, and chosen to reflect the look and/or meaning of the original symbol. +_Note: these are only the default, built-in unicode operators. Coconut supports [custom operator definition](#custom-operators) to define your own._ + ##### Full List ``` From 49d114759705e7d4fe29629ba8d929084c61e8c5 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 18:39:43 -0700 Subject: [PATCH 29/57] Fix py37 tests --- coconut/compiler/header.py | 2 ++ coconut/constants.py | 1 + 2 files changed, 3 insertions(+) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index da436a7fa..1ba8b4188 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -579,6 +579,8 @@ def NamedTuple(name, fields): except ImportError: class YouNeedToInstallTypingExtensions{object}: __slots__ = () + def __init__(self): + raise _coconut.TypeError('Protocols cannot be instantiated') Protocol = YouNeedToInstallTypingExtensions typing.Protocol = Protocol '''.format(**format_dict), diff --git a/coconut/constants.py b/coconut/constants.py index f09255863..fc8815357 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -80,6 +80,7 @@ def get_bool_env_var(env_var, default=False): ((PY2 and not PY26) or PY35) and not (PYPY and WINDOWS) and (PY37 or not PYPY) + and sys.version_info[:2] != (3, 7) ) MYPY = ( PY37 From b26781b2cfa1b9634077fab8bf42f61a80f38368 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 20:48:32 -0700 Subject: [PATCH 30/57] Reenable develop --- coconut/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coconut/root.py b/coconut/root.py index 712f277b0..e81f953b7 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = False +DEVELOP = 1 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 2c7c67bcc784085e1362d8876bd5c7b09a0c0149 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 21:34:32 -0700 Subject: [PATCH 31/57] Remove confusing unicode alternatives Resolves #748. --- DOCS.md | 5 ++--- coconut/compiler/grammar.py | 6 +++--- coconut/constants.py | 3 --- coconut/root.py | 2 +- coconut/tests/src/cocotest/agnostic/suite.coco | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/DOCS.md b/DOCS.md index 34cd5eac3..45418ef33 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1052,9 +1052,8 @@ _Note: these are only the default, built-in unicode operators. Coconut supports ≥ (\u2265) or ⊇ (\u2287) => ">=" ⊊ (\u228a) => "<" ⊋ (\u228b) => ">" -∧ (\u2227) or ∩ (\u2229) => "&" -∨ (\u2228) or ∪ (\u222a) => "|" -⊻ (\u22bb) => "^" +∩ (\u2229) => "&" +∪ (\u222a) => "|" « (\xab) => "<<" » (\xbb) => ">>" … (\u2026) => "..." diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 5099a2de5..e77f942ca 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -707,9 +707,9 @@ class Grammar(object): | invalid_syntax("") + ~Literal("|*") + Literal("|") | fixto(Literal("\u2228") | Literal("\u222a"), "|") + amp = ~amp_colon + Literal("&") | fixto(Literal("\u2229"), "&") + caret = Literal("^") + unsafe_bar = ~Literal("|>") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) percent = Literal("%") dollar = Literal("$") diff --git a/coconut/constants.py b/coconut/constants.py index fc8815357..eace256f2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -767,11 +767,8 @@ def get_bool_env_var(env_var, default=False): "\u2260", # != "\u2264", # <= "\u2265", # >= - "\u2227", # & "\u2229", # & - "\u2228", # | "\u222a", # | - "\u22bb", # ^ "\xab", # << "\xbb", # >> "\u2026", # ... diff --git a/coconut/root.py b/coconut/root.py index e81f953b7..3e367e0f1 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 1 +DEVELOP = 2 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index b542db14e..e796a0114 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -148,7 +148,7 @@ def suite_test() -> bool: assert one_to_five([1,2,3,4,5]) == [2,3,4] assert not one_to_five([0,1,2,3,4,5]) assert one_to_five([1,5]) == [] - assert -4 == neg_square_u(2) ≠ 4 ∧ 0 ≤ neg_square_u(0) ≤ 0 + assert -4 == neg_square_u(2) ≠ 4 ∩ 0 ≤ neg_square_u(0) ≤ 0 assert is_null(null1()) assert is_null(null2()) assert empty() |> depth_1 == 0 == empty() |> depth_2 From 7740b58f16313a23c326c6727c03b40dade5b90d Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Wed, 24 May 2023 22:11:00 -0700 Subject: [PATCH 32/57] Fix typo --- DOCS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCS.md b/DOCS.md index 45418ef33..da8e3e136 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3908,7 +3908,7 @@ if group: **windowsof**(_size_, _iterable_, _fillvalue_=`...`, _step_=`1`) -`windowsof` produces an iterable that effectively mimics a sliding window over _iterable_ of size _size_. _step_ determines the spacing between windowsof. +`windowsof` produces an iterable that effectively mimics a sliding window over _iterable_ of size _size_. _step_ determines the spacing between windows. If _size_ is larger than _iterable_, `windowsof` will produce an empty iterable. If that is not the desired behavior, _fillvalue_ can be passed and will be used in place of missing values. Also, if _fillvalue_ is passed and the length of the _iterable_ is not divisible by _step_, _fillvalue_ will be used in that case to pad the last window as well. Note that _fillvalue_ will only ever appear in the last window. From 522a25ca7c93f95e3cc5637b57861e2cadf94388 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 25 May 2023 01:06:55 -0700 Subject: [PATCH 33/57] Rename convenience to api Resolves #750. --- DOCS.md | 42 ++-- HELP.md | 2 +- coconut/api.py | 275 +++++++++++++++++++++++++ coconut/api.pyi | 108 ++++++++++ coconut/command/cli.py | 2 +- coconut/command/resources/zcoconut.pth | 2 +- coconut/command/util.py | 2 +- coconut/compiler/compiler.py | 2 +- coconut/compiler/header.py | 12 +- coconut/constants.py | 4 +- coconut/convenience.py | 257 +---------------------- coconut/convenience.pyi | 95 +-------- coconut/integrations.py | 10 +- coconut/root.py | 2 +- coconut/tests/main_test.py | 12 +- coconut/tests/src/extras.coco | 12 +- 16 files changed, 439 insertions(+), 400 deletions(-) create mode 100644 coconut/api.py create mode 100644 coconut/api.pyi diff --git a/DOCS.md b/DOCS.md index da8e3e136..361259d92 100644 --- a/DOCS.md +++ b/DOCS.md @@ -204,7 +204,7 @@ dest destination directory for compiled files (defaults to run the compiler in a separate thread with the given stack size in kilobytes --site-install, --siteinstall - set up coconut.convenience to be imported on Python start + set up coconut.api to be imported on Python start --site-uninstall, --siteuninstall revert the effects of --site-install --verbose print verbose debug output @@ -220,7 +220,7 @@ coconut-run ``` as an alias for ``` -coconut --run --quiet --target sys --line-numbers --argv +coconut --quiet --target sys --line-numbers --keep-lines --run --argv ``` which will quietly compile and run ``, passing any additional arguments to the script, mimicking how the `python` command works. @@ -391,7 +391,7 @@ Simply installing Coconut should add a `Coconut` kernel to your Jupyter/IPython The Coconut kernel will always compile using the parameters: `--target sys --line-numbers --keep-lines --no-wrap-types`. -Coconut also provides the following convenience commands: +Coconut also provides the following api commands: - `coconut --jupyter notebook` will ensure that the Coconut kernel is available and launch a Jupyter/IPython notebook. - `coconut --jupyter console` will launch a Jupyter/IPython console using the Coconut kernel. @@ -4265,7 +4265,7 @@ Recommended usage is as a debugging tool, where the code `from coconut import em ### Automatic Compilation -If you don't care about the exact compilation parameters you want to use, automatic compilation lets Coconut take care of everything for you. Automatic compilation can be enabled either by importing [`coconut.convenience`](#coconut-convenience) before you import anything else, or by running `coconut --site-install`. Once automatic compilation is enabled, Coconut will check each of your imports to see if you are attempting to import a `.coco` file and, if so, automatically compile it for you. Note that, for Coconut to know what file you are trying to import, it will need to be accessible via `sys.path`, just like a normal import. +If you don't care about the exact compilation parameters you want to use, automatic compilation lets Coconut take care of everything for you. Automatic compilation can be enabled either by importing [`coconut.api`](#coconut-api) before you import anything else, or by running `coconut --site-install`. Once automatic compilation is enabled, Coconut will check each of your imports to see if you are attempting to import a `.coco` file and, if so, automatically compile it for you. Note that, for Coconut to know what file you are trying to import, it will need to be accessible via `sys.path`, just like a normal import. Automatic compilation always compiles modules and packages in-place, and always uses `--target sys`. Automatic compilation is always available in the Coconut interpreter, and, if using the Coconut interpreter, a `reload` built-in is provided to easily reload imported modules. Additionally, the interpreter always allows importing from the current working directory, letting you easily compile and play around with a `.coco` file simply by running the Coconut interpreter and importing it. @@ -4275,15 +4275,17 @@ While automatic compilation is the preferred method for dynamically compiling Co ```coconut # coding: coconut ``` -declaration which can be added to `.py` files to have them treated as Coconut files instead. To use such a coding declaration, you'll need to either run `coconut --site-install` or `import coconut.convenience` at some point before you first attempt to import a file with a `# coding: coconut` declaration. Like automatic compilation, compilation is always done with `--target sys` and is always available from the Coconut interpreter. +declaration which can be added to `.py` files to have them treated as Coconut files instead. To use such a coding declaration, you'll need to either run `coconut --site-install` or `import coconut.api` at some point before you first attempt to import a file with a `# coding: coconut` declaration. Like automatic compilation, compilation is always done with `--target sys` and is always available from the Coconut interpreter. -### `coconut.convenience` +### `coconut.api` -In addition to enabling automatic compilation, `coconut.convenience` can also be used to call the Coconut compiler from code instead of from the command line. See below for specifications of the different convenience functions. +In addition to enabling automatic compilation, `coconut.api` can also be used to call the Coconut compiler from code instead of from the command line. See below for specifications of the different api functions. + +_DEPRECATED: `coconut.convenience` is a deprecated alias for `coconut.api`._ #### `get_state` -**coconut.convenience.get\_state**(_state_=`None`) +**coconut.api.get\_state**(_state_=`None`) Gets a state object which stores the current compilation parameters. State objects can be configured with [**setup**](#setup) or [**cmd**](#cmd) and then used in [**parse**](#parse) or [**coconut\_eval**](#coconut_eval). @@ -4291,9 +4293,9 @@ If _state_ is `None`, gets a new state object, whereas if _state_ is `False`, th #### `parse` -**coconut.convenience.parse**(_code_=`""`, _mode_=`"sys"`, _state_=`False`, _keep\_internal\_state_=`None`) +**coconut.api.parse**(_code_=`""`, _mode_=`"sys"`, _state_=`False`, _keep\_internal\_state_=`None`) -Likely the most useful of the convenience functions, `parse` takes Coconut code as input and outputs the equivalent compiled Python code. _mode_ is used to indicate the context for the parsing and _state_ is the state object storing the compilation parameters to use as obtained from [**get_state**](#get_state) (if `False`, uses the global state object). _keep\_internal\_state_ determines whether the state object will keep internal state (such as what [custom operators](#custom-operators) have been declared)—if `None`, internal state will be kept iff you are not using the global _state_. +Likely the most useful of the api functions, `parse` takes Coconut code as input and outputs the equivalent compiled Python code. _mode_ is used to indicate the context for the parsing and _state_ is the state object storing the compilation parameters to use as obtained from [**get_state**](#get_state) (if `False`, uses the global state object). _keep\_internal\_state_ determines whether the state object will keep internal state (such as what [custom operators](#custom-operators) have been declared)—if `None`, internal state will be kept iff you are not using the global _state_. If _code_ is not passed, `parse` will output just the given _mode_'s header, which can be executed to set up an execution environment in which future code can be parsed and executed without a header. @@ -4340,7 +4342,7 @@ Each _mode_ has two components: what parser it uses, and what header it prepends ##### Example ```coconut_python -from coconut.convenience import parse +from coconut.api import parse exec(parse()) while True: exec(parse(input(), mode="block")) @@ -4348,7 +4350,7 @@ while True: #### `setup` -**coconut.convenience.setup**(_target_=`None`, _strict_=`False`, _minify_=`False`, _line\_numbers_=`False`, _keep\_lines_=`False`, _no\_tco_=`False`, _no\_wrap_=`False`, *, _state_=`False`) +**coconut.api.setup**(_target_=`None`, _strict_=`False`, _minify_=`False`, _line\_numbers_=`False`, _keep\_lines_=`False`, _no\_tco_=`False`, _no\_wrap_=`False`, *, _state_=`False`) `setup` can be used to set up the given state object with the given command-line flags. If _state_ is `False`, the global state object is used. @@ -4364,7 +4366,7 @@ The possible values for each flag argument are: #### `cmd` -**coconut.convenience.cmd**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`None`, _state_=`False`) +**coconut.api.cmd**(_args_=`None`, *, _argv_=`None`, _interact_=`False`, _default\_target_=`None`, _state_=`False`) Executes the given _args_ as if they were fed to `coconut` on the command-line, with the exception that unless _interact_ is true or `-i` is passed, the interpreter will not be started. Additionally, _argv_ can be used to pass in arguments as in `--argv` and _default\_target_ can be used to set the default `--target`. @@ -4372,13 +4374,13 @@ Has the same effect of setting the command-line flags on the given _state_ objec #### `coconut_eval` -**coconut.convenience.coconut_eval**(_expression_, _globals_=`None`, _locals_=`None`, _state_=`False`, _keep\_internal\_state_=`None`) +**coconut.api.coconut_eval**(_expression_, _globals_=`None`, _locals_=`None`, _state_=`False`, _keep\_internal\_state_=`None`) Version of [`eval`](https://docs.python.org/3/library/functions.html#eval) which can evaluate Coconut code. #### `version` -**coconut.convenience.version**(**[**_which_**]**) +**coconut.api.version**(**[**_which_**]**) Retrieves a string containing information about the Coconut version. The optional argument _which_ is the type of version information desired. Possible values of _which_ are: @@ -4390,19 +4392,19 @@ Retrieves a string containing information about the Coconut version. The optiona #### `auto_compilation` -**coconut.convenience.auto_compilation**(_on_=`True`) +**coconut.api.auto_compilation**(_on_=`True`) -Turns [automatic compilation](#automatic-compilation) on or off. This function is called automatically when `coconut.convenience` is imported. +Turns [automatic compilation](#automatic-compilation) on or off. This function is called automatically when `coconut.api` is imported. #### `use_coconut_breakpoint` -**coconut.convenience.use_coconut_breakpoint**(_on_=`True`) +**coconut.api.use_coconut_breakpoint**(_on_=`True`) -Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) which Coconut makes universally available to use [`coconut.embed`](#coconut-embed) instead of [`pdb.set_trace`](https://docs.python.org/3/library/pdb.html#pdb.set_trace) (or undoes that switch if `on=False`). This function is called automatically when `coconut.convenience` is imported. +Switches the [`breakpoint` built-in](https://www.python.org/dev/peps/pep-0553/) which Coconut makes universally available to use [`coconut.embed`](#coconut-embed) instead of [`pdb.set_trace`](https://docs.python.org/3/library/pdb.html#pdb.set_trace) (or undoes that switch if `on=False`). This function is called automatically when `coconut.api` is imported. #### `CoconutException` -If an error is encountered in a convenience function, a `CoconutException` instance may be raised. `coconut.convenience.CoconutException` is provided to allow catching such errors. +If an error is encountered in a api function, a `CoconutException` instance may be raised. `coconut.api.CoconutException` is provided to allow catching such errors. ### `coconut.__coconut__` diff --git a/HELP.md b/HELP.md index e016cb271..99b1a5c4b 100644 --- a/HELP.md +++ b/HELP.md @@ -133,7 +133,7 @@ Compiling single files is not the only way to use the Coconut command-line utili The Coconut compiler supports a large variety of different compilation options, the help for which can always be accessed by entering `coconut -h` into the command line. One of the most useful of these is `--line-numbers` (or `-l` for short). Using `--line-numbers` will add the line numbers of your source code as comments in the compiled code, allowing you to see what line in your source code corresponds to a line in the compiled code where an error occurred, for ease of debugging. -_Note: If you don't need the full control of the Coconut compiler, you can also [access your Coconut code just by importing it](./DOCS.md#automatic-compilation), either from the Coconut interpreter, or in any Python file where you import [`coconut.convenience`](./DOCS.md#coconut-convenience)._ +_Note: If you don't need the full control of the Coconut compiler, you can also [access your Coconut code just by importing it](./DOCS.md#automatic-compilation), either from the Coconut interpreter, or in any Python file where you import [`coconut.api`](./DOCS.md#coconut-api)._ ### Using IPython/Jupyter diff --git a/coconut/api.py b/coconut/api.py new file mode 100644 index 000000000..0e1d42d6e --- /dev/null +++ b/coconut/api.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# ----------------------------------------------------------------------------------------------------------------------- +# INFO: +# ----------------------------------------------------------------------------------------------------------------------- + +""" +Author: Evan Hubinger +License: Apache 2.0 +Description: Coconut's main external API. +""" + +# ----------------------------------------------------------------------------------------------------------------------- +# IMPORTS: +# ----------------------------------------------------------------------------------------------------------------------- + +from __future__ import print_function, absolute_import, unicode_literals, division + +from coconut.root import * # NOQA + +import sys +import os.path +import codecs +try: + from encodings import utf_8 +except ImportError: + utf_8 = None + +from coconut.integrations import embed +from coconut.exceptions import CoconutException +from coconut.command import Command +from coconut.command.cli import cli_version +from coconut.compiler import Compiler +from coconut.constants import ( + version_tag, + code_exts, + coconut_import_hook_args, + coconut_kernel_kwargs, +) + +# ----------------------------------------------------------------------------------------------------------------------- +# COMMAND: +# ----------------------------------------------------------------------------------------------------------------------- + +GLOBAL_STATE = None + + +def get_state(state=None): + """Get a Coconut state object; None gets a new state, False gets the global state.""" + global GLOBAL_STATE + if state is None: + return Command() + elif state is False: + if GLOBAL_STATE is None: + GLOBAL_STATE = Command() + return GLOBAL_STATE + else: + return state + + +def cmd(cmd_args, interact=False, state=False, **kwargs): + """Process command-line arguments.""" + if isinstance(cmd_args, (str, bytes)): + cmd_args = cmd_args.split() + return get_state(state).cmd(cmd_args, interact=interact, **kwargs) + + +VERSIONS = { + "num": VERSION, + "name": VERSION_NAME, + "spec": VERSION_STR, + "tag": version_tag, + "-v": cli_version, +} + + +def version(which="num"): + """Get the Coconut version.""" + if which in VERSIONS: + return VERSIONS[which] + else: + raise CoconutException( + "invalid version type " + repr(which), + extra="valid versions are " + ", ".join(VERSIONS), + ) + + +# ----------------------------------------------------------------------------------------------------------------------- +# COMPILER: +# ----------------------------------------------------------------------------------------------------------------------- + +def setup(*args, **kwargs): + """Set up the given state object.""" + state = kwargs.pop("state", False) + return get_state(state).setup(*args, **kwargs) + + +PARSERS = { + "sys": lambda comp: comp.parse_sys, + "exec": lambda comp: comp.parse_exec, + "file": lambda comp: comp.parse_file, + "package": lambda comp: comp.parse_package, + "block": lambda comp: comp.parse_block, + "single": lambda comp: comp.parse_single, + "eval": lambda comp: comp.parse_eval, + "lenient": lambda comp: comp.parse_lenient, + "xonsh": lambda comp: comp.parse_xonsh, +} + +# deprecated aliases +PARSERS["any"] = PARSERS["debug"] = PARSERS["lenient"] + + +def parse(code="", mode="sys", state=False, keep_internal_state=None): + """Compile Coconut code.""" + if keep_internal_state is None: + keep_internal_state = bool(state) + command = get_state(state) + if command.comp is None: + command.setup() + if mode not in PARSERS: + raise CoconutException( + "invalid parse mode " + repr(mode), + extra="valid modes are " + ", ".join(PARSERS), + ) + return PARSERS[mode](command.comp)(code, keep_state=keep_internal_state) + + +def coconut_eval(expression, globals=None, locals=None, state=False, **kwargs): + """Compile and evaluate Coconut code.""" + command = get_state(state) + if command.comp is None: + setup() + command.check_runner(set_sys_vars=False) + if globals is None: + globals = {} + command.runner.update_vars(globals) + compiled_python = parse(expression, "eval", state, **kwargs) + return eval(compiled_python, globals, locals) + + +# ----------------------------------------------------------------------------------------------------------------------- +# BREAKPOINT: +# ----------------------------------------------------------------------------------------------------------------------- + + +def _coconut_breakpoint(): + """Determine coconut.embed depth based on whether we're being + called by Coconut's breakpoint() or Python's breakpoint().""" + if sys.version_info >= (3, 7): + return embed(depth=1) + else: + return embed(depth=2) + + +def use_coconut_breakpoint(on=True): + """Switches the breakpoint() built-in (universally accessible via + coconut.__coconut__.breakpoint) to use coconut.embed.""" + if on: + sys.breakpointhook = _coconut_breakpoint + else: + sys.breakpointhook = sys.__breakpointhook__ + + +use_coconut_breakpoint() + + +# ----------------------------------------------------------------------------------------------------------------------- +# AUTOMATIC COMPILATION: +# ----------------------------------------------------------------------------------------------------------------------- + + +class CoconutImporter(object): + """Finder and loader for compiling Coconut files at import time.""" + ext = code_exts[0] + command = None + + def run_compiler(self, path): + """Run the Coconut compiler on the given path.""" + if self.command is None: + self.command = Command() + self.command.cmd([path] + list(coconut_import_hook_args)) + + def find_module(self, fullname, path=None): + """Searches for a Coconut file of the given name and compiles it.""" + basepaths = [""] + list(sys.path) + if fullname.startswith("."): + if path is None: + # we can't do a relative import if there's no package path + return + fullname = fullname[1:] + basepaths.insert(0, path) + fullpath = os.path.join(*fullname.split(".")) + for head in basepaths: + path = os.path.join(head, fullpath) + filepath = path + self.ext + dirpath = os.path.join(path, "__init__" + self.ext) + if os.path.exists(filepath): + self.run_compiler(filepath) + # Coconut file was found and compiled, now let Python import it + return + if os.path.exists(dirpath): + self.run_compiler(path) + # Coconut package was found and compiled, now let Python import it + return + + +coconut_importer = CoconutImporter() + + +def auto_compilation(on=True): + """Turn automatic compilation of Coconut files on or off.""" + if on: + if coconut_importer not in sys.meta_path: + sys.meta_path.insert(0, coconut_importer) + else: + try: + sys.meta_path.remove(coconut_importer) + except ValueError: + pass + + +auto_compilation() + + +# ----------------------------------------------------------------------------------------------------------------------- +# ENCODING: +# ----------------------------------------------------------------------------------------------------------------------- + + +if utf_8 is not None: + class CoconutStreamReader(utf_8.StreamReader, object): + """Compile Coconut code from a stream of UTF-8.""" + coconut_compiler = None + + @classmethod + def compile_coconut(cls, source): + """Compile the given Coconut source text.""" + if cls.coconut_compiler is None: + cls.coconut_compiler = Compiler(**coconut_kernel_kwargs) + return cls.coconut_compiler.parse_sys(source) + + @classmethod + def decode(cls, input_bytes, errors="strict"): + """Decode and compile the given Coconut source bytes.""" + input_str, len_consumed = super(CoconutStreamReader, cls).decode(input_bytes, errors) + return cls.compile_coconut(input_str), len_consumed + + class CoconutIncrementalDecoder(utf_8.IncrementalDecoder, object): + """Compile Coconut at the end of incrementally decoding UTF-8.""" + invertible = False + _buffer_decode = CoconutStreamReader.decode + + +def get_coconut_encoding(encoding="coconut"): + """Get a CodecInfo for the given Coconut encoding.""" + if not encoding.startswith("coconut"): + return None + if encoding != "coconut": + raise CoconutException("unknown Coconut encoding: " + repr(encoding)) + if utf_8 is None: + raise CoconutException("coconut encoding requires encodings.utf_8") + return codecs.CodecInfo( + name=encoding, + encode=utf_8.encode, + decode=CoconutStreamReader.decode, + incrementalencoder=utf_8.IncrementalEncoder, + incrementaldecoder=CoconutIncrementalDecoder, + streamreader=CoconutStreamReader, + streamwriter=utf_8.StreamWriter, + ) + + +codecs.register(get_coconut_encoding) diff --git a/coconut/api.pyi b/coconut/api.pyi new file mode 100644 index 000000000..b2845d394 --- /dev/null +++ b/coconut/api.pyi @@ -0,0 +1,108 @@ +#----------------------------------------------------------------------------------------------------------------------- +# INFO: +#----------------------------------------------------------------------------------------------------------------------- + +""" +Author: Evan Hubinger +License: Apache 2.0 +Description: MyPy stub file for api.py. +""" + +#----------------------------------------------------------------------------------------------------------------------- +# IMPORTS: +#----------------------------------------------------------------------------------------------------------------------- + +from typing import ( + Any, + Callable, + Dict, + Iterable, + Optional, + Text, + Union, +) + +from coconut.command.command import Command + +class CoconutException(Exception): + ... + +#----------------------------------------------------------------------------------------------------------------------- +# COMMAND: +#----------------------------------------------------------------------------------------------------------------------- + +GLOBAL_STATE: Optional[Command] = None + + +def get_state(state: Optional[Command]=None) -> Command: ... + + +def cmd(args: Union[Text, bytes, Iterable], interact: bool=False) -> None: ... + + +VERSIONS: Dict[Text, Text] = ... + + +def version(which: Optional[Text]=None) -> Text: ... + + +#----------------------------------------------------------------------------------------------------------------------- +# COMPILER: +#----------------------------------------------------------------------------------------------------------------------- + + +def setup( + target: Optional[str]=None, + strict: bool=False, + minify: bool=False, + line_numbers: bool=False, + keep_lines: bool=False, + no_tco: bool=False, + no_wrap: bool=False, +) -> None: ... + + +PARSERS: Dict[Text, Callable] = ... + + +def parse( + code: Text, + mode: Text=..., + state: Optional[Command]=..., + keep_internal_state: Optional[bool]=None, +) -> Text: ... + + +def coconut_eval( + expression: Text, + globals: Optional[Dict[Text, Any]]=None, + locals: Optional[Dict[Text, Any]]=None, + state: Optional[Command]=..., + keep_internal_state: Optional[bool]=None, +) -> Any: ... + + +# ----------------------------------------------------------------------------------------------------------------------- +# ENABLERS: +# ----------------------------------------------------------------------------------------------------------------------- + + +def use_coconut_breakpoint(on: bool=True) -> None: ... + + +class CoconutImporter: + ext: str + + @staticmethod + def run_compiler(path: str) -> None: ... + + def find_module(self, fullname: str, path: Optional[str]=None) -> None: ... + + +coconut_importer = CoconutImporter() + + +def auto_compilation(on: bool=True) -> None: ... + + +def get_coconut_encoding(encoding: str=...) -> Any: ... diff --git a/coconut/command/cli.py b/coconut/command/cli.py index 73af5fde9..62e9b8050 100644 --- a/coconut/command/cli.py +++ b/coconut/command/cli.py @@ -269,7 +269,7 @@ arguments.add_argument( "--site-install", "--siteinstall", action="store_true", - help="set up coconut.convenience to be imported on Python start", + help="set up coconut.api to be imported on Python start", ) arguments.add_argument( diff --git a/coconut/command/resources/zcoconut.pth b/coconut/command/resources/zcoconut.pth index 8ca5c334e..56fab7383 100644 --- a/coconut/command/resources/zcoconut.pth +++ b/coconut/command/resources/zcoconut.pth @@ -1 +1 @@ -import coconut.convenience +import coconut.api diff --git a/coconut/command/util.py b/coconut/command/util.py index 85fdaa404..7f18d0d36 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -552,7 +552,7 @@ class Runner(object): def __init__(self, comp=None, exit=sys.exit, store=False, path=None): """Create the executor.""" - from coconut.convenience import auto_compilation, use_coconut_breakpoint + from coconut.api import auto_compilation, use_coconut_breakpoint auto_compilation(on=interpreter_uses_auto_compilation) use_coconut_breakpoint(on=interpreter_uses_coconut_breakpoint) self.exit = exit diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 9a7ba1bd6..b79419faf 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -452,7 +452,7 @@ def __init__(self, *args, **kwargs): """Creates a new compiler with the given parsing parameters.""" self.setup(*args, **kwargs) - # changes here should be reflected in __reduce__ and in the stub for coconut.convenience.setup + # changes here should be reflected in __reduce__ and in the stub for coconut.api.setup def setup(self, target=None, strict=False, minify=False, line_numbers=False, keep_lines=False, no_tco=False, no_wrap=False): """Initializes parsing parameters.""" if target is None: diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 1ba8b4188..8ccc2d172 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -305,8 +305,8 @@ def pattern_prepender(func): return pattern_prepender''' if not strict else r'''def prepattern(*args, **kwargs): - """Deprecated built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead.""" - raise _coconut.NameError("deprecated built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead")''' + """Deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'prepattern' disabled by --strict compilation; use 'addpattern' instead")''' ), def_datamaker=( r'''def datamaker(data_type): @@ -314,14 +314,14 @@ def pattern_prepender(func): return _coconut.functools.partial(makedata, data_type)''' if not strict else r'''def datamaker(*args, **kwargs): - """Deprecated built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead.""" - raise _coconut.NameError("deprecated built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead")''' + """Deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'datamaker' disabled by --strict compilation; use 'makedata' instead")''' ), of_is_call=( "of = call" if not strict else r'''def of(*args, **kwargs): - """Deprecated built-in 'of' disabled by --strict compilation; use 'call' instead.""" - raise _coconut.NameError("deprecated built-in 'of' disabled by --strict compilation; use 'call' instead")''' + """Deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead.""" + raise _coconut.NameError("deprecated Coconut built-in 'of' disabled by --strict compilation; use 'call' instead")''' ), return_method_of_self=pycondition( (3,), diff --git a/coconut/constants.py b/coconut/constants.py index eace256f2..3bcc1c6b2 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -598,8 +598,8 @@ def get_bool_env_var(env_var, default=False): ) # always use atomic --xxx=yyy rather than --xxx yyy -coconut_run_args = ("--run", "--target=sys", "--line-numbers", "--quiet") -coconut_run_verbose_args = ("--run", "--target=sys", "--line-numbers") +coconut_run_verbose_args = ("--run", "--target=sys", "--line-numbers", "--keep-lines") +coconut_run_args = coconut_run_verbose_args + ("--quiet",) coconut_import_hook_args = ("--target=sys", "--line-numbers", "--keep-lines", "--quiet") default_mypy_args = ( diff --git a/coconut/convenience.py b/coconut/convenience.py index 917734d60..14a6bed5a 100644 --- a/coconut/convenience.py +++ b/coconut/convenience.py @@ -8,7 +8,7 @@ """ Author: Evan Hubinger License: Apache 2.0 -Description: Convenience functions for using Coconut as a module. +Description: Deprecated alias for coconut.api. """ # ----------------------------------------------------------------------------------------------------------------------- @@ -17,257 +17,4 @@ from __future__ import print_function, absolute_import, unicode_literals, division -from coconut.root import * # NOQA - -import sys -import os.path -import codecs -try: - from encodings import utf_8 -except ImportError: - utf_8 = None - -from coconut.integrations import embed -from coconut.exceptions import CoconutException -from coconut.command import Command -from coconut.command.cli import cli_version -from coconut.compiler import Compiler -from coconut.constants import ( - version_tag, - code_exts, - coconut_import_hook_args, - coconut_kernel_kwargs, -) - -# ----------------------------------------------------------------------------------------------------------------------- -# COMMAND: -# ----------------------------------------------------------------------------------------------------------------------- - -GLOBAL_STATE = None - - -def get_state(state=None): - """Get a Coconut state object; None gets a new state, False gets the global state.""" - global GLOBAL_STATE - if state is None: - return Command() - elif state is False: - if GLOBAL_STATE is None: - GLOBAL_STATE = Command() - return GLOBAL_STATE - else: - return state - - -def cmd(cmd_args, interact=False, state=False, **kwargs): - """Process command-line arguments.""" - if isinstance(cmd_args, (str, bytes)): - cmd_args = cmd_args.split() - return get_state(state).cmd(cmd_args, interact=interact, **kwargs) - - -VERSIONS = { - "num": VERSION, - "name": VERSION_NAME, - "spec": VERSION_STR, - "tag": version_tag, - "-v": cli_version, -} - - -def version(which="num"): - """Get the Coconut version.""" - if which in VERSIONS: - return VERSIONS[which] - else: - raise CoconutException( - "invalid version type " + repr(which), - extra="valid versions are " + ", ".join(VERSIONS), - ) - - -# ----------------------------------------------------------------------------------------------------------------------- -# COMPILER: -# ----------------------------------------------------------------------------------------------------------------------- - -def setup(*args, **kwargs): - """Set up the given state object.""" - state = kwargs.pop("state", False) - return get_state(state).setup(*args, **kwargs) - - -PARSERS = { - "sys": lambda comp: comp.parse_sys, - "exec": lambda comp: comp.parse_exec, - "file": lambda comp: comp.parse_file, - "package": lambda comp: comp.parse_package, - "block": lambda comp: comp.parse_block, - "single": lambda comp: comp.parse_single, - "eval": lambda comp: comp.parse_eval, - "lenient": lambda comp: comp.parse_lenient, - "xonsh": lambda comp: comp.parse_xonsh, -} - -# deprecated aliases -PARSERS["any"] = PARSERS["debug"] = PARSERS["lenient"] - - -def parse(code="", mode="sys", state=False, keep_internal_state=None): - """Compile Coconut code.""" - if keep_internal_state is None: - keep_internal_state = bool(state) - command = get_state(state) - if command.comp is None: - command.setup() - if mode not in PARSERS: - raise CoconutException( - "invalid parse mode " + repr(mode), - extra="valid modes are " + ", ".join(PARSERS), - ) - return PARSERS[mode](command.comp)(code, keep_state=keep_internal_state) - - -def coconut_eval(expression, globals=None, locals=None, state=False, **kwargs): - """Compile and evaluate Coconut code.""" - command = get_state(state) - if command.comp is None: - setup() - command.check_runner(set_sys_vars=False) - if globals is None: - globals = {} - command.runner.update_vars(globals) - compiled_python = parse(expression, "eval", state, **kwargs) - return eval(compiled_python, globals, locals) - - -# ----------------------------------------------------------------------------------------------------------------------- -# BREAKPOINT: -# ----------------------------------------------------------------------------------------------------------------------- - - -def _coconut_breakpoint(): - """Determine coconut.embed depth based on whether we're being - called by Coconut's breakpoint() or Python's breakpoint().""" - if sys.version_info >= (3, 7): - return embed(depth=1) - else: - return embed(depth=2) - - -def use_coconut_breakpoint(on=True): - """Switches the breakpoint() built-in (universally accessible via - coconut.__coconut__.breakpoint) to use coconut.embed.""" - if on: - sys.breakpointhook = _coconut_breakpoint - else: - sys.breakpointhook = sys.__breakpointhook__ - - -use_coconut_breakpoint() - - -# ----------------------------------------------------------------------------------------------------------------------- -# AUTOMATIC COMPILATION: -# ----------------------------------------------------------------------------------------------------------------------- - - -class CoconutImporter(object): - """Finder and loader for compiling Coconut files at import time.""" - ext = code_exts[0] - - @staticmethod - def run_compiler(path): - """Run the Coconut compiler on the given path.""" - cmd([path] + list(coconut_import_hook_args)) - - def find_module(self, fullname, path=None): - """Searches for a Coconut file of the given name and compiles it.""" - basepaths = [""] + list(sys.path) - if fullname.startswith("."): - if path is None: - # we can't do a relative import if there's no package path - return - fullname = fullname[1:] - basepaths.insert(0, path) - fullpath = os.path.join(*fullname.split(".")) - for head in basepaths: - path = os.path.join(head, fullpath) - filepath = path + self.ext - dirpath = os.path.join(path, "__init__" + self.ext) - if os.path.exists(filepath): - self.run_compiler(filepath) - # Coconut file was found and compiled, now let Python import it - return - if os.path.exists(dirpath): - self.run_compiler(path) - # Coconut package was found and compiled, now let Python import it - return - - -coconut_importer = CoconutImporter() - - -def auto_compilation(on=True): - """Turn automatic compilation of Coconut files on or off.""" - if on: - if coconut_importer not in sys.meta_path: - sys.meta_path.insert(0, coconut_importer) - else: - try: - sys.meta_path.remove(coconut_importer) - except ValueError: - pass - - -auto_compilation() - - -# ----------------------------------------------------------------------------------------------------------------------- -# ENCODING: -# ----------------------------------------------------------------------------------------------------------------------- - - -if utf_8 is not None: - class CoconutStreamReader(utf_8.StreamReader, object): - """Compile Coconut code from a stream of UTF-8.""" - coconut_compiler = None - - @classmethod - def compile_coconut(cls, source): - """Compile the given Coconut source text.""" - if cls.coconut_compiler is None: - cls.coconut_compiler = Compiler(**coconut_kernel_kwargs) - return cls.coconut_compiler.parse_sys(source) - - @classmethod - def decode(cls, input_bytes, errors="strict"): - """Decode and compile the given Coconut source bytes.""" - input_str, len_consumed = super(CoconutStreamReader, cls).decode(input_bytes, errors) - return cls.compile_coconut(input_str), len_consumed - - class CoconutIncrementalDecoder(utf_8.IncrementalDecoder, object): - """Compile Coconut at the end of incrementally decoding UTF-8.""" - invertible = False - _buffer_decode = CoconutStreamReader.decode - - -def get_coconut_encoding(encoding="coconut"): - """Get a CodecInfo for the given Coconut encoding.""" - if not encoding.startswith("coconut"): - return None - if encoding != "coconut": - raise CoconutException("unknown Coconut encoding: " + repr(encoding)) - if utf_8 is None: - raise CoconutException("coconut encoding requires encodings.utf_8") - return codecs.CodecInfo( - name=encoding, - encode=utf_8.encode, - decode=CoconutStreamReader.decode, - incrementalencoder=utf_8.IncrementalEncoder, - incrementaldecoder=CoconutIncrementalDecoder, - streamreader=CoconutStreamReader, - streamwriter=utf_8.StreamWriter, - ) - - -codecs.register(get_coconut_encoding) +from coconut.api import * # NOQA diff --git a/coconut/convenience.pyi b/coconut/convenience.pyi index ef9b64194..bfc8f7043 100644 --- a/coconut/convenience.pyi +++ b/coconut/convenience.pyi @@ -12,97 +12,4 @@ Description: MyPy stub file for convenience.py. # IMPORTS: #----------------------------------------------------------------------------------------------------------------------- -from typing import ( - Any, - Callable, - Dict, - Iterable, - Optional, - Text, - Union, -) - -from coconut.command.command import Command - -class CoconutException(Exception): - ... - -#----------------------------------------------------------------------------------------------------------------------- -# COMMAND: -#----------------------------------------------------------------------------------------------------------------------- - -GLOBAL_STATE: Optional[Command] = None - - -def get_state(state: Optional[Command]=None) -> Command: ... - - -def cmd(args: Union[Text, bytes, Iterable], interact: bool=False) -> None: ... - - -VERSIONS: Dict[Text, Text] = ... - - -def version(which: Optional[Text]=None) -> Text: ... - - -#----------------------------------------------------------------------------------------------------------------------- -# COMPILER: -#----------------------------------------------------------------------------------------------------------------------- - - -def setup( - target: Optional[str]=None, - strict: bool=False, - minify: bool=False, - line_numbers: bool=False, - keep_lines: bool=False, - no_tco: bool=False, - no_wrap: bool=False, -) -> None: ... - - -PARSERS: Dict[Text, Callable] = ... - - -def parse( - code: Text, - mode: Text=..., - state: Optional[Command]=..., - keep_internal_state: Optional[bool]=None, -) -> Text: ... - - -def coconut_eval( - expression: Text, - globals: Optional[Dict[Text, Any]]=None, - locals: Optional[Dict[Text, Any]]=None, - state: Optional[Command]=..., - keep_internal_state: Optional[bool]=None, -) -> Any: ... - - -# ----------------------------------------------------------------------------------------------------------------------- -# ENABLERS: -# ----------------------------------------------------------------------------------------------------------------------- - - -def use_coconut_breakpoint(on: bool=True) -> None: ... - - -class CoconutImporter: - ext: str - - @staticmethod - def run_compiler(path: str) -> None: ... - - def find_module(self, fullname: str, path: Optional[str]=None) -> None: ... - - -coconut_importer = CoconutImporter() - - -def auto_compilation(on: bool=True) -> None: ... - - -def get_coconut_encoding(encoding: str=...) -> Any: ... +from coconut.api import * diff --git a/coconut/integrations.py b/coconut/integrations.py index 7636e1e7e..d9bddd2e9 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -57,12 +57,12 @@ def load_ipython_extension(ipython): ipython.push(newvars) # import here to avoid circular dependencies - from coconut import convenience + from coconut import api from coconut.exceptions import CoconutException from coconut.terminal import logger - magic_state = convenience.get_state() - convenience.setup(state=magic_state, **coconut_kernel_kwargs) + magic_state = api.get_state() + api.setup(state=magic_state, **coconut_kernel_kwargs) # add magic function def magic(line, cell=None): @@ -74,9 +74,9 @@ def magic(line, cell=None): # first line in block is cmd, rest is code line = line.strip() if line: - convenience.cmd(line, default_target="sys", state=magic_state) + api.cmd(line, default_target="sys", state=magic_state) code = cell - compiled = convenience.parse(code, state=magic_state) + compiled = api.parse(code, state=magic_state) except CoconutException: logger.print_exc() else: diff --git a/coconut/root.py b/coconut/root.py index 3e367e0f1..bfd6058d5 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 2 +DEVELOP = 3 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 444228b19..b60393986 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -57,7 +57,7 @@ get_bool_env_var, ) -from coconut.convenience import ( +from coconut.api import ( auto_compilation, setup, ) @@ -402,10 +402,10 @@ def using_dest(dest=dest): @contextmanager -def using_coconut(fresh_logger=True, fresh_convenience=False): - """Decorator for ensuring that coconut.terminal.logger and coconut.convenience.* are reset.""" +def using_coconut(fresh_logger=True, fresh_api=False): + """Decorator for ensuring that coconut.terminal.logger and coconut.api.* are reset.""" saved_logger = logger.copy() - if fresh_convenience: + if fresh_api: setup() auto_compilation(False) if fresh_logger: @@ -678,8 +678,8 @@ def test_target_3_snip(self): def test_pipe(self): call('echo ' + escape(coconut_snip) + "| coconut -s", shell=True, assert_output=True) - def test_convenience(self): - call_python(["-c", 'from coconut.convenience import parse; exec(parse("' + coconut_snip + '"))'], assert_output=True) + def test_api(self): + call_python(["-c", 'from coconut.api import parse; exec(parse("' + coconut_snip + '"))'], assert_output=True) def test_import_hook(self): with using_sys_path(src): diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 6411ff8a2..923c74f90 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -281,14 +281,14 @@ def test_convenience() -> bool: assert parse("abc", "lenient") == "abc #1: abc" setup() - assert "Deprecated built-in 'prepattern' disabled by --strict compilation" not in parse("\n", mode="file") - assert "Deprecated built-in 'datamaker' disabled by --strict compilation" not in parse("\n", mode="file") - assert "Deprecated built-in 'of' disabled by --strict compilation" not in parse("\n", mode="file") + assert "Deprecated Coconut built-in 'prepattern' disabled by --strict compilation" not in parse("\n", mode="file") + assert "Deprecated Coconut built-in 'datamaker' disabled by --strict compilation" not in parse("\n", mode="file") + assert "Deprecated Coconut built-in 'of' disabled by --strict compilation" not in parse("\n", mode="file") setup(strict=True) - assert "Deprecated built-in 'prepattern' disabled by --strict compilation" in parse("\n", mode="file") - assert "Deprecated built-in 'datamaker' disabled by --strict compilation" in parse("\n", mode="file") - assert "Deprecated built-in 'of' disabled by --strict compilation" in parse("\n", mode="file") + assert "Deprecated Coconut built-in 'prepattern' disabled by --strict compilation" in parse("\n", mode="file") + assert "Deprecated Coconut built-in 'datamaker' disabled by --strict compilation" in parse("\n", mode="file") + assert "Deprecated Coconut built-in 'of' disabled by --strict compilation" in parse("\n", mode="file") assert_raises(-> parse("def f(x):\n \t pass"), CoconutStyleError) assert_raises(-> parse("lambda x: x"), CoconutStyleError) From 49ba82a18dd34ea99c242425857a530c47414de7 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 25 May 2023 01:28:27 -0700 Subject: [PATCH 34/57] Fix xonsh again Resolves #751. --- coconut/integrations.py | 57 ++++++++++++++++++++++------------------- coconut/root.py | 2 +- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/coconut/integrations.py b/coconut/integrations.py index d9bddd2e9..09678cc82 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -96,34 +96,30 @@ class CoconutXontribLoader(object): timing_info = [] @memoize_with_exceptions(128) - def _base_memoized_parse_xonsh(self, code): + def memoized_parse_xonsh(self, code): return self.compiler.parse_xonsh(code, keep_state=True) - def memoized_parse_xonsh(self, code): + def compile_code(self, code): """Memoized self.compiler.parse_xonsh.""" - # .strip() outside the memoization - return self._base_memoized_parse_xonsh(code.strip()) + # hide imports to avoid circular dependencies + from coconut.exceptions import CoconutException + from coconut.terminal import format_error + from coconut.util import get_clock_time + from coconut.terminal import logger - def new_parse(self, parser, code, mode="exec", *args, **kwargs): - """Coconut-aware version of xonsh's _parse.""" - if self.loaded and mode not in disabled_xonsh_modes: - # hide imports to avoid circular dependencies - from coconut.exceptions import CoconutException - from coconut.terminal import format_error - from coconut.util import get_clock_time - from coconut.terminal import logger + parse_start_time = get_clock_time() + quiet, logger.quiet = logger.quiet, True + try: + # .strip() outside the memoization + code = self.memoized_parse_xonsh(code.strip()) + except CoconutException as err: + err_str = format_error(err).splitlines()[0] + code += " #" + err_str + finally: + logger.quiet = quiet + self.timing_info.append(("parse", get_clock_time() - parse_start_time)) - parse_start_time = get_clock_time() - quiet, logger.quiet = logger.quiet, True - try: - code = self.memoized_parse_xonsh(code) - except CoconutException as err: - err_str = format_error(err).splitlines()[0] - code += " #" + err_str - finally: - logger.quiet = quiet - self.timing_info.append(("parse", get_clock_time() - parse_start_time)) - return parser.__class__.parse(parser, code, mode=mode, *args, **kwargs) + return code def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): """Version of try_subproc_toks that handles the fact that Coconut @@ -136,17 +132,23 @@ def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): finally: ctxtransformer.mode = mode - def new_ctxvisit(self, ctxtransformer, node, inp, *args, **kwargs): + def new_parse(self, parser, code, mode="exec", *args, **kwargs): + """Coconut-aware version of xonsh's _parse.""" + if self.loaded and mode not in disabled_xonsh_modes: + code = self.compile_code(code) + return parser.__class__.parse(parser, code, mode=mode, *args, **kwargs) + + def new_ctxvisit(self, ctxtransformer, node, inp, ctx, mode="exec", *args, **kwargs): """Version of ctxvisit that ensures looking up original lines in inp using Coconut line numbers will work properly.""" - if self.loaded: + if self.loaded and mode not in disabled_xonsh_modes: from xonsh.tools import get_logical_line # hide imports to avoid circular dependencies from coconut.terminal import logger from coconut.compiler.util import extract_line_num_from_comment - compiled = self.memoized_parse_xonsh(inp) + compiled = self.compile_code(inp) original_lines = tuple(inp.splitlines()) used_lines = set() @@ -166,7 +168,8 @@ def new_ctxvisit(self, ctxtransformer, node, inp, *args, **kwargs): new_inp_lines.append(line) last_ln = ln inp = "\n".join(new_inp_lines) + "\n" - return ctxtransformer.__class__.ctxvisit(ctxtransformer, node, inp, *args, **kwargs) + + return ctxtransformer.__class__.ctxvisit(ctxtransformer, node, inp, ctx, mode, *args, **kwargs) def __call__(self, xsh, **kwargs): # hide imports to avoid circular dependencies diff --git a/coconut/root.py b/coconut/root.py index bfd6058d5..6f19898a7 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 3 +DEVELOP = 4 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 3430261316540d7994f673b7fdc171c8364a5da0 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 25 May 2023 01:37:28 -0700 Subject: [PATCH 35/57] Further fix xonsh --- coconut/compiler/templates/header.py_template | 2 +- coconut/integrations.py | 48 ++++++++++--------- coconut/root.py | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 5fa1e9760..9f036c72a 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1217,7 +1217,7 @@ class groupsof(_coconut_has_iter): def __copy__(self): return self.__class__(self.group_size, self.get_new_iter()) class recursive_iterator(_coconut_baseclass): - """Decorator that optimizes a recursive function that returns an iterator (e.g. a recursive generator).""" + """Decorator that memoizes a recursive function that returns an iterator (e.g. a recursive generator).""" __slots__ = ("func", "reit_store", "backup_reit_store") def __init__(self, func): self.func = func diff --git a/coconut/integrations.py b/coconut/integrations.py index 09678cc82..883652fb5 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -109,17 +109,20 @@ def compile_code(self, code): parse_start_time = get_clock_time() quiet, logger.quiet = logger.quiet, True + success = False try: # .strip() outside the memoization code = self.memoized_parse_xonsh(code.strip()) except CoconutException as err: err_str = format_error(err).splitlines()[0] code += " #" + err_str + else: + success = True finally: logger.quiet = quiet self.timing_info.append(("parse", get_clock_time() - parse_start_time)) - return code + return code, success def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): """Version of try_subproc_toks that handles the fact that Coconut @@ -135,7 +138,7 @@ def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): def new_parse(self, parser, code, mode="exec", *args, **kwargs): """Coconut-aware version of xonsh's _parse.""" if self.loaded and mode not in disabled_xonsh_modes: - code = self.compile_code(code) + code, _ = self.compile_code(code) return parser.__class__.parse(parser, code, mode=mode, *args, **kwargs) def new_ctxvisit(self, ctxtransformer, node, inp, ctx, mode="exec", *args, **kwargs): @@ -148,26 +151,27 @@ def new_ctxvisit(self, ctxtransformer, node, inp, ctx, mode="exec", *args, **kwa from coconut.terminal import logger from coconut.compiler.util import extract_line_num_from_comment - compiled = self.compile_code(inp) - - original_lines = tuple(inp.splitlines()) - used_lines = set() - new_inp_lines = [] - last_ln = 1 - for compiled_line in compiled.splitlines(): - ln = extract_line_num_from_comment(compiled_line, default=last_ln + 1) - try: - line, _, _ = get_logical_line(original_lines, ln - 1) - except IndexError: - logger.log_exc() - line = original_lines[-1] - if line in used_lines: - line = "" - else: - used_lines.add(line) - new_inp_lines.append(line) - last_ln = ln - inp = "\n".join(new_inp_lines) + "\n" + compiled, success = self.compile_code(inp) + + if success: + original_lines = tuple(inp.splitlines()) + used_lines = set() + new_inp_lines = [] + last_ln = 1 + for compiled_line in compiled.splitlines(): + ln = extract_line_num_from_comment(compiled_line, default=last_ln + 1) + try: + line, _, _ = get_logical_line(original_lines, ln - 1) + except IndexError: + logger.log_exc() + line = original_lines[-1] + if line in used_lines: + line = "" + else: + used_lines.add(line) + new_inp_lines.append(line) + last_ln = ln + inp = "\n".join(new_inp_lines) + "\n" return ctxtransformer.__class__.ctxvisit(ctxtransformer, node, inp, ctx, mode, *args, **kwargs) diff --git a/coconut/root.py b/coconut/root.py index 6f19898a7..273c905ff 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 4 +DEVELOP = 5 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 6c6ae2e92199a42ff7cd646f013a2036b8ee01ee Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 25 May 2023 18:05:51 -0700 Subject: [PATCH 36/57] Improve xonsh testing --- coconut/compiler/grammar.py | 1 + coconut/constants.py | 15 ++++++++++----- coconut/integrations.py | 10 ++++++---- coconut/root.py | 2 +- coconut/tests/main_test.py | 12 ++++++++---- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index e77f942ca..dec9124b3 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -2348,6 +2348,7 @@ class Grammar(object): unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) unsafe_xonsh_command = originalTextFor( (Optional(at) + dollar | bang) + + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) + (parens | brackets | braces | unsafe_name), ) xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( diff --git a/coconut/constants.py b/coconut/constants.py index 3bcc1c6b2..2e5ce5706 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -863,7 +863,9 @@ def get_bool_env_var(env_var, default=False): "watchdog", ), "xonsh": ( - "xonsh", + ("xonsh", "py<36"), + ("xonsh", "py==37"), + ("xonsh", "py38"), ), "backports": ( ("trollius", "py2;cpy"), @@ -922,14 +924,16 @@ def get_bool_env_var(env_var, default=False): ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 18), ("pygments", "mark39"): (2, 15), + ("xonsh", "py38"): (0, 14), # pinned reqs: (must be added to pinned_reqs below) # don't upgrade until myst-parser supports the new version "sphinx": (6,), - # don't upgrade this; it breaks on Python 3.7 + # don't upgrade these; they breaks on Python 3.7 ("ipython", "py==37"): (7, 34), - # don't upgrade these; it breaks on Python 3.6 + ("xonsh", "py==37"): (0, 12), + # don't upgrade these; they breaks on Python 3.6 ("pandas", "py36"): (1,), ("jupyter-client", "py36"): (7, 1, 2), ("typing_extensions", "py==36"): (4, 1), @@ -940,7 +944,7 @@ def get_bool_env_var(env_var, default=False): ("jupyter-client", "py==35"): (6, 1, 12), ("jupytext", "py3"): (1, 8), ("jupyterlab", "py35"): (2, 2), - "xonsh": (0, 9), + ("xonsh", "py<36"): (0, 9), ("typing_extensions", "py==35"): (3, 10), # don't upgrade this to allow all versions ("prompt_toolkit", "mark3"): (1,), @@ -969,6 +973,7 @@ def get_bool_env_var(env_var, default=False): pinned_reqs = ( "sphinx", ("ipython", "py==37"), + ("xonsh", "py==37"), ("pandas", "py36"), ("jupyter-client", "py36"), ("typing_extensions", "py==36"), @@ -979,7 +984,7 @@ def get_bool_env_var(env_var, default=False): ("jupyter-client", "py==35"), ("jupytext", "py3"), ("jupyterlab", "py35"), - "xonsh", + ("xonsh", "py<36"), ("typing_extensions", "py==35"), ("prompt_toolkit", "mark3"), "pytest", diff --git a/coconut/integrations.py b/coconut/integrations.py index 883652fb5..087ab116e 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -112,17 +112,17 @@ def compile_code(self, code): success = False try: # .strip() outside the memoization - code = self.memoized_parse_xonsh(code.strip()) + compiled = self.memoized_parse_xonsh(code.strip()) except CoconutException as err: err_str = format_error(err).splitlines()[0] - code += " #" + err_str + compiled = code + " #" + err_str else: success = True finally: logger.quiet = quiet self.timing_info.append(("parse", get_clock_time() - parse_start_time)) - return code, success + return compiled, success def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): """Version of try_subproc_toks that handles the fact that Coconut @@ -171,7 +171,9 @@ def new_ctxvisit(self, ctxtransformer, node, inp, ctx, mode="exec", *args, **kwa used_lines.add(line) new_inp_lines.append(line) last_ln = ln - inp = "\n".join(new_inp_lines) + "\n" + inp = "\n".join(new_inp_lines) + + inp += "\n" return ctxtransformer.__class__.ctxvisit(ctxtransformer, node, inp, ctx, mode, *args, **kwargs) diff --git a/coconut/root.py b/coconut/root.py index 273c905ff..31efbdd33 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 5 +DEVELOP = 6 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index b60393986..cd5668980 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -46,9 +46,9 @@ WINDOWS, PYPY, IPY, + XONSH, MYPY, PY35, - PY36, PY38, PY310, icoconut_default_kernel_names, @@ -708,9 +708,7 @@ def test_import_runnable(self): for _ in range(2): # make sure we can import it twice call_python([runnable_py, "--arg"], assert_output=True, convert_to_import=True) - # not py36 is only because newer Python versions require newer xonsh - # versions that aren't always installed by pip install coconut[tests] - if not WINDOWS and PY35 and not PY36: + if not WINDOWS and XONSH: def test_xontrib(self): p = spawn_cmd("xonsh") p.expect("$") @@ -718,6 +716,12 @@ def test_xontrib(self): p.expect("$") p.sendline("!(ls -la) |> bool") p.expect("True") + p.sendline('$ENV_VAR = "ABC"') + p.expect("$") + p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') + p.expect("ABC\nABC") + p.sendline("echo 123;; 123") + p.expect("123;; 123") p.sendline("xontrib unload coconut") p.expect("$") p.sendeof() From f9ebee29edfdde2ee25ed021ad8f33c72f8416ab Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 25 May 2023 22:03:22 -0700 Subject: [PATCH 37/57] Further fix xonsh --- coconut/constants.py | 8 +++++--- coconut/integrations.py | 4 ++-- coconut/root.py | 2 +- coconut/tests/main_test.py | 7 ++++++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 2e5ce5706..760dfb4e9 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -889,7 +889,8 @@ def get_bool_env_var(env_var, default=False): "pydata-sphinx-theme", ), "tests": ( - "pytest", + ("pytest", "py<36"), + ("pytest", "py36"), "pexpect", ("numpy", "py34"), ("numpy", "py2;cpy"), @@ -925,6 +926,7 @@ def get_bool_env_var(env_var, default=False): ("jedi", "py39"): (0, 18), ("pygments", "mark39"): (2, 15), ("xonsh", "py38"): (0, 14), + ("pytest", "py36"): (7,), # pinned reqs: (must be added to pinned_reqs below) @@ -949,7 +951,7 @@ def get_bool_env_var(env_var, default=False): # don't upgrade this to allow all versions ("prompt_toolkit", "mark3"): (1,), # don't upgrade this; it breaks on Python 2.6 - "pytest": (3,), + ("pytest", "py<36"): (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), # don't upgrade this; it breaks on Python 3.4 @@ -987,7 +989,7 @@ def get_bool_env_var(env_var, default=False): ("xonsh", "py<36"), ("typing_extensions", "py==35"), ("prompt_toolkit", "mark3"), - "pytest", + ("pytest", "py<36"), "vprof", ("pygments", "mark<39"), ("pywinpty", "py2;windows"), diff --git a/coconut/integrations.py b/coconut/integrations.py index 087ab116e..f453cdbd8 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -212,8 +212,8 @@ def __call__(self, xsh, **kwargs): def unload(self, xsh): if not self.loaded: # hide imports to avoid circular dependencies - from coconut.exceptions import CoconutException - raise CoconutException("attempting to unload Coconut xontrib but it was never loaded") + from coconut.terminal import logger + logger.warn("attempting to unload Coconut xontrib but it was never loaded") self.loaded = False diff --git a/coconut/root.py b/coconut/root.py index 31efbdd33..6e2dbd917 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 6 +DEVELOP = 7 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index cd5668980..0603eeb8d 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -719,11 +719,16 @@ def test_xontrib(self): p.sendline('$ENV_VAR = "ABC"') p.expect("$") p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') - p.expect("ABC\nABC") + p.expect("ABC") + p.expect("ABC") p.sendline("echo 123;; 123") p.expect("123;; 123") + p.sendline('execx("10 |> print")') + p.expect("subprocess mode") p.sendline("xontrib unload coconut") p.expect("$") + p.sendline("1 |> print") + p.expect("subprocess mode") p.sendeof() if p.isalive(): p.terminate() From a18958bd1dce2076d1ce79e07afff3ee6c923c89 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Thu, 25 May 2023 23:20:50 -0700 Subject: [PATCH 38/57] Fix xonsh tests --- DOCS.md | 4 +++- FAQ.md | 2 +- coconut/compiler/templates/header.py_template | 9 ++++++++- coconut/constants.py | 7 ++++--- coconut/tests/main_test.py | 6 ++++-- coconut/tests/src/cocotest/agnostic/primary.coco | 1 + coconut/tests/src/cocotest/agnostic/suite.coco | 1 + coconut/tests/src/cocotest/agnostic/util.coco | 4 ++++ 8 files changed, 26 insertions(+), 8 deletions(-) diff --git a/DOCS.md b/DOCS.md index 361259d92..738fb2b8e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -3207,7 +3207,9 @@ _Can't be done without a series of method definitions for each data type. See th In Haskell, `fmap(func, obj)` takes a data type `obj` and returns a new data type with `func` mapped over the contents. Coconut's `fmap` function does the exact same thing for Coconut's [data types](#data). -`fmap` can also be used on built-ins such as `str`, `list`, `set`, and `dict` as a variant of `map` that returns back an object of the same type. The behavior of `fmap` for a given object can be overridden by defining an `__fmap__(self, func)` magic method that will be called whenever `fmap` is invoked on that object. Note that `__fmap__` implementations should always satisfy the [Functor Laws](https://wiki.haskell.org/Functor). +`fmap` can also be used on the built-in objects `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, and `dict` as a variant of `map` that returns back an object of the same type. + +The behavior of `fmap` for a given object can be overridden by defining an `__fmap__(self, func)` magic method that will be called whenever `fmap` is invoked on that object. Note that `__fmap__` implementations should always satisfy the [Functor Laws](https://wiki.haskell.org/Functor). For `dict`, or any other `collections.abc.Mapping`, `fmap` will map over the mapping's `.items()` instead of the default iteration through its `.keys()`, with the new mapping reconstructed from the mapped over items. _DEPRECATED: `fmap$(starmap_over_mappings=True)` will `starmap` over the `.items()` instead of `map` over them._ diff --git a/FAQ.md b/FAQ.md index 755cdbbb2..201885b2e 100644 --- a/FAQ.md +++ b/FAQ.md @@ -94,4 +94,4 @@ If you don't get the reference, the image above is from [Monty Python and the Ho ### Who developed Coconut? -[Evan Hubinger](https://github.com/evhub) is a [full-time AGI safety researcher](https://www.alignmentforum.org/users/evhub) at the [Machine Intelligence Research Institute](https://intelligence.org/). He can be reached by asking a question on [Coconut's Gitter chat room](https://gitter.im/evhub/coconut), through email at , or on [LinkedIn](https://www.linkedin.com/in/ehubinger). +[Evan Hubinger](https://github.com/evhub) is an [AI safety research scientist](https://www.alignmentforum.org/users/evhub) at [Anthropic](https://www.anthropic.com/). He can be reached by asking a question on [Coconut's Gitter chat room](https://gitter.im/evhub/coconut), through email at , or on [LinkedIn](https://www.linkedin.com/in/ehubinger). diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 9f036c72a..e9b16680f 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -1485,7 +1485,14 @@ def makedata(data_type, *args, **kwargs): {class_amap} def fmap(func, obj, **kwargs): """fmap(func, obj) creates a copy of obj with func applied to its contents. - Supports asynchronous iterables, mappings (maps over .items()), and numpy arrays (uses np.vectorize). + + Supports: + * Coconut data types + * `str`, `dict`, `list`, `tuple`, `set`, `frozenset` + * `dict` (maps over .items()) + * asynchronous iterables + * numpy arrays (uses np.vectorize) + * pandas objects (uses .apply) Override by defining obj.__fmap__(func). """ diff --git a/coconut/constants.py b/coconut/constants.py index 760dfb4e9..0fdc6a74f 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -90,6 +90,7 @@ def get_bool_env_var(env_var, default=False): XONSH = ( PY35 and not (PYPY and PY39) + and sys.version_info[:2] != (3, 7) ) py_version_str = sys.version.split()[0] @@ -864,7 +865,7 @@ def get_bool_env_var(env_var, default=False): ), "xonsh": ( ("xonsh", "py<36"), - ("xonsh", "py==37"), + ("xonsh", "py>=36;py<38"), ("xonsh", "py38"), ), "backports": ( @@ -934,8 +935,8 @@ def get_bool_env_var(env_var, default=False): "sphinx": (6,), # don't upgrade these; they breaks on Python 3.7 ("ipython", "py==37"): (7, 34), - ("xonsh", "py==37"): (0, 12), # don't upgrade these; they breaks on Python 3.6 + ("xonsh", "py>=36;py<38"): (0, 11), ("pandas", "py36"): (1,), ("jupyter-client", "py36"): (7, 1, 2), ("typing_extensions", "py==36"): (4, 1), @@ -975,7 +976,7 @@ def get_bool_env_var(env_var, default=False): pinned_reqs = ( "sphinx", ("ipython", "py==37"), - ("xonsh", "py==37"), + ("xonsh", "py>=36;py<38"), ("pandas", "py36"), ("jupyter-client", "py36"), ("typing_extensions", "py==36"), diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 0603eeb8d..765b32c84 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -49,6 +49,7 @@ XONSH, MYPY, PY35, + PY36, PY38, PY310, icoconut_default_kernel_names, @@ -721,8 +722,9 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") - p.sendline("echo 123;; 123") - p.expect("123;; 123") + if PY36: + p.sendline("echo 123;; 123") + p.expect("123;; 123") p.sendline('execx("10 |> print")') p.expect("subprocess mode") p.sendline("xontrib unload coconut") diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 453106920..1b77e269b 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1601,4 +1601,5 @@ def primary_test() -> bool: assert (...=really_long_var).really_long_var == 10 n = [0] assert n[0] == 0 + assert_raises(-> m{{1:2,2:3}}, TypeError) return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index e796a0114..2ccc7269a 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -219,6 +219,7 @@ def suite_test() -> bool: assert inh_a.inh_true4() is True assert inh_a.inh_true5() is True assert inh_A.inh_cls_true() is True + assert inh_inh_A().true() is False assert pt.__doc__ out0 = grid() |> grid_trim$(xmax=5, ymax=5) assert out0 == [ diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index 59b3ec93c..ee171e873 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -864,6 +864,10 @@ class clsC: class clsD: d = 4 +class inh_inh_A(inh_A): + @override + def true(self) = False + class MyExc(Exception): def __init__(self, m): super().__init__(m) From f1518cd05547a19bd100362ede28251fbae3cea6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 26 May 2023 00:29:07 -0700 Subject: [PATCH 39/57] Further fix xonsh tests --- coconut/constants.py | 2 +- coconut/tests/main_test.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 0fdc6a74f..dabdd2b5e 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -90,7 +90,7 @@ def get_bool_env_var(env_var, default=False): XONSH = ( PY35 and not (PYPY and PY39) - and sys.version_info[:2] != (3, 7) + and (PY38 or not PY36) ) py_version_str = sys.version.split()[0] diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 765b32c84..0c3e41cfe 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -729,8 +729,9 @@ def test_xontrib(self): p.expect("subprocess mode") p.sendline("xontrib unload coconut") p.expect("$") - p.sendline("1 |> print") - p.expect("subprocess mode") + if PY36: + p.sendline("1 |> print") + p.expect("subprocess mode") p.sendeof() if p.isalive(): p.terminate() From 01a0a8494fef73ef92a9fc28ff84839d666a1b51 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 26 May 2023 01:51:30 -0700 Subject: [PATCH 40/57] Fix pypy38 --- coconut/tests/main_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 0c3e41cfe..0ea36456d 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -51,6 +51,7 @@ PY35, PY36, PY38, + PY39, PY310, icoconut_default_kernel_names, icoconut_custom_kernel_name, @@ -722,7 +723,7 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") - if PY36: + if PY36 and (not PYPY or PY39): p.sendline("echo 123;; 123") p.expect("123;; 123") p.sendline('execx("10 |> print")') From ac63676f85c68f8964a76de707dd109f7d082380 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 26 May 2023 16:15:35 -0700 Subject: [PATCH 41/57] Further fix pypy38 --- coconut/tests/main_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 0ea36456d..990cc4ba5 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -723,14 +723,15 @@ def test_xontrib(self): p.sendline('echo f"{$ENV_VAR}"; echo f"{$ENV_VAR}"') p.expect("ABC") p.expect("ABC") - if PY36 and (not PYPY or PY39): - p.sendline("echo 123;; 123") - p.expect("123;; 123") - p.sendline('execx("10 |> print")') - p.expect("subprocess mode") + if not PYPY or PY39: + if PY36: + p.sendline("echo 123;; 123") + p.expect("123;; 123") + p.sendline('execx("10 |> print")') + p.expect("subprocess mode") p.sendline("xontrib unload coconut") p.expect("$") - if PY36: + if (not PYPY or PY39) and PY36: p.sendline("1 |> print") p.expect("subprocess mode") p.sendeof() From de574e7dc3fe0e3c76112d497669a61c31702f44 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 26 May 2023 21:24:45 -0700 Subject: [PATCH 42/57] Use typing_extensions whenever possible Resolves #752. --- DOCS.md | 25 +++---- coconut/compiler/compiler.py | 33 ++++++++-- coconut/compiler/grammar.py | 8 ++- coconut/compiler/header.py | 65 ++++++++++--------- coconut/compiler/templates/header.py_template | 17 ++++- coconut/constants.py | 6 ++ coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/primary.coco | 1 + .../tests/src/cocotest/agnostic/specific.coco | 1 + .../tests/src/cocotest/agnostic/suite.coco | 1 + coconut/tests/src/cocotest/agnostic/util.coco | 5 +- coconut/tests/src/extras.coco | 3 +- 12 files changed, 111 insertions(+), 56 deletions(-) diff --git a/DOCS.md b/DOCS.md index 738fb2b8e..6a79171b4 100644 --- a/DOCS.md +++ b/DOCS.md @@ -323,16 +323,17 @@ If the `--strict` (`-s` for short) flag is enabled, Coconut will perform additio The style issues which will cause `--strict` to throw an error are: -- mixing of tabs and spaces (without `--strict` will show a warning), -- use of `from __future__` imports (Coconut does these automatically) (without `--strict` will show a warning), -- inheriting from `object` in classes (Coconut does this automatically) (without `--strict` will show a warning), -- semicolons at end of lines (without `--strict` will show a warning), -- use of `u` to denote Unicode strings (all Coconut strings are Unicode strings) (without `--strict` will show a warning), -- missing new line at end of file, -- trailing whitespace at end of lines, -- use of the Python-style `lambda` statement (use [Coconut's lambda syntax](#lambdas) instead), -- use of backslash continuation (use [parenthetical continuation](#enhanced-parenthetical-continuation) instead), -- Python-3.10/PEP-634-style dotted names in pattern-matching (Coconut style is to preface these with `==`), and +- mixing of tabs and spaces (without `--strict` will show a warning). +- use of `from __future__` imports (Coconut does these automatically) (without `--strict` will show a warning). +- inheriting from `object` in classes (Coconut does this automatically) (without `--strict` will show a warning). +- semicolons at end of lines (without `--strict` will show a warning). +- use of `u` to denote Unicode strings (all Coconut strings are Unicode strings) (without `--strict` will show a warning). +- commas after [statement lambdas](#statement-lambdas) (not recommended as it can be unclear whether the comma is inside or outside the lambda) (without `--strict` will show a warning). +- missing new line at end of file. +- trailing whitespace at end of lines. +- use of the Python-style `lambda` statement (use [Coconut's lambda syntax](#lambdas) instead). +- use of backslash continuation (use [parenthetical continuation](#enhanced-parenthetical-continuation) instead). +- Python-3.10/PEP-634-style dotted names in pattern-matching (Coconut style is to preface these with `==`). - use of `:` instead of `<:` to specify upper bounds in [Coconut's type parameter syntax](#type-parameter-syntax). ## Integrations @@ -1613,7 +1614,7 @@ If the last `statement` (not followed by a semicolon) in a statement lambda is a Statement lambdas also support implicit lambda syntax such that `def -> _` is equivalent to `def (_=None) -> _` as well as explicitly marking them as pattern-matching such that `match def (x) -> x` will be a pattern-matching function. -Note that statement lambdas have a lower precedence than normal lambdas and thus capture things like trailing commas. +Note that statement lambdas have a lower precedence than normal lambdas and thus capture things like trailing commas. To avoid confusion, statement lambdas should always be wrapped in their own set of parentheses. ##### Example @@ -1779,7 +1780,7 @@ mod(5, 3) Since Coconut syntax is a superset of Python 3 syntax, it supports [Python 3 function type annotation syntax](https://www.python.org/dev/peps/pep-0484/) and [Python 3.6 variable type annotation syntax](https://www.python.org/dev/peps/pep-0526/). By default, Coconut compiles all type annotations into Python-2-compatible type comments. If you want to keep the type annotations instead, simply pass a `--target` that supports them. -Since not all supported Python versions support the [`typing`](https://docs.python.org/3/library/typing.html) module, Coconut provides the [`TYPE_CHECKING`](#type_checking) built-in for hiding your `typing` imports and `TypeVar` definitions from being executed at runtime. Coconut will also automatically use [`typing_extensions`](https://pypi.org/project/typing-extensions/) over `typing` when importing objects not available in `typing` on the current Python version. +Since not all supported Python versions support the [`typing`](https://docs.python.org/3/library/typing.html) module, Coconut provides the [`TYPE_CHECKING`](#type_checking) built-in for hiding your `typing` imports and `TypeVar` definitions from being executed at runtime. Coconut will also automatically use [`typing_extensions`](https://pypi.org/project/typing-extensions/) over `typing` objects at runtime when importing them from `typing`, even when they aren't natively supported on the current Python version (this works even if you just do `import typing` and then `typing.`). Furthermore, when compiling type annotations to Python 3 versions without [PEP 563](https://www.python.org/dev/peps/pep-0563/) support, Coconut wraps annotation in strings to prevent them from being evaluated at runtime (note that `--no-wrap-types` disables all wrapping, including via PEP 563 support). diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index b79419faf..2cc135e15 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -87,6 +87,7 @@ all_builtins, in_place_op_funcs, match_first_arg_var, + import_existing, ) from coconut.util import ( pickleable_obj, @@ -195,8 +196,27 @@ def set_to_tuple(tokens): raise CoconutInternalException("invalid set maker item", tokens[0]) -def import_stmt(imp_from, imp, imp_as): +def import_stmt(imp_from, imp, imp_as, raw=False): """Generate an import statement.""" + if not raw: + module_path = (imp if imp_from is None else imp_from).split(".", 1) + existing_imp = import_existing.get(module_path[0]) + if existing_imp is not None: + return handle_indentation( + """ +if _coconut.typing.TYPE_CHECKING: + {raw_import} +else: + try: + {imp_name} = {imp_lookup} + except _coconut.AttributeError as _coconut_imp_err: + raise _coconut.ImportError(_coconut.str(_coconut_imp_err)) + """, + ).format( + raw_import=import_stmt(imp_from, imp, imp_as, raw=True), + imp_name=imp_as if imp_as is not None else imp, + imp_lookup=".".join([existing_imp] + module_path[1:] + ([imp] if imp_from is not None else [])), + ) return ( ("from " + imp_from + " " if imp_from is not None else "") + "import " + imp @@ -3072,9 +3092,7 @@ def single_import(self, path, imp_as, type_ignore=False): imp_from += imp.rsplit("." + imp_as, 1)[0] imp, imp_as = imp_as, None - if imp_from is None and imp == "sys": - out.append((imp_as if imp_as is not None else imp) + " = _coconut_sys") - elif imp_as is not None and "." in imp_as: + if imp_as is not None and "." in imp_as: import_as_var = self.get_temp_var("import") out.append(import_stmt(imp_from, imp, import_as_var)) fake_mods = imp_as.split(".") @@ -3375,7 +3393,12 @@ def set_letter_literal_handle(self, tokens): def stmt_lambdef_handle(self, original, loc, tokens): """Process multi-line lambdef statements.""" - got_kwds, params, stmts_toks = tokens + got_kwds, params, stmts_toks, followed_by = tokens + + if followed_by == ",": + self.strict_err_or_warn("found statement lambda followed by comma; this isn't recommended as it can be unclear whether the comma is inside or outside the lambda (just wrap the lambda in parentheses)", original, loc) + else: + internal_assert(followed_by == "", "invalid stmt_lambdef followed_by", followed_by) is_async = False add_kwds = [] diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index dec9124b3..c2434dcb7 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1592,7 +1592,13 @@ class Grammar(object): + arrow.suppress() + stmt_lambdef_body ) - stmt_lambdef_ref = general_stmt_lambdef | match_stmt_lambdef + stmt_lambdef_ref = ( + general_stmt_lambdef + | match_stmt_lambdef + ) + ( + fixto(FollowedBy(comma), ",") + | fixto(always_match, "") + ) lambdef <<= addspace(lambdef_base + test) | stmt_lambdef lambdef_no_cond = trace(addspace(lambdef_base + test_no_cond)) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 8ccc2d172..c366419b6 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -534,28 +534,36 @@ async def __anext__(self): underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in".format(**format_dict), import_typing=pycondition( (3, 5), - if_ge="import typing", + if_ge=''' +import typing as _typing +for _name in dir(_typing): + if not hasattr(typing, _name): + setattr(typing, _name, getattr(_typing, _name)) + ''', if_lt=''' -class typing_mock{object}: - """The typing module is not available at runtime in Python 3.4 or earlier; - try hiding your typedefs behind an 'if TYPE_CHECKING:' block.""" - TYPE_CHECKING = False +if not hasattr(typing, "TYPE_CHECKING"): + typing.TYPE_CHECKING = False +if not hasattr(typing, "Any"): Any = Ellipsis - def cast(self, t, x): +if not hasattr(typing, "cast"): + def cast(t, x): """typing.cast[T](t: Type[T], x: Any) -> T = x""" return x - def __getattr__(self, name): - raise _coconut.ImportError("the typing module is not available at runtime in Python 3.4 or earlier; try hiding your typedefs behind an 'if TYPE_CHECKING:' block") + typing.cast = cast + cast = staticmethod(cast) +if not hasattr(typing, "TypeVar"): def TypeVar(name, *args, **kwargs): """Runtime mock of typing.TypeVar for Python 3.4 and earlier.""" return name + typing.TypeVar = TypeVar + TypeVar = staticmethod(TypeVar) +if not hasattr(typing, "Generic"): class Generic_mock{object}: """Runtime mock of typing.Generic for Python 3.4 and earlier.""" __slots__ = () def __getitem__(self, vars): return _coconut.object - Generic = Generic_mock() -typing = typing_mock() + typing.Generic = Generic_mock() '''.format(**format_dict), indent=1, ), @@ -563,10 +571,11 @@ def __getitem__(self, vars): import_typing_36=pycondition( (3, 6), if_lt=''' -def NamedTuple(name, fields): - return _coconut.collections.namedtuple(name, [x for x, t in fields]) -typing.NamedTuple = NamedTuple -NamedTuple = staticmethod(NamedTuple) +if not hasattr(typing, "NamedTuple"): + def NamedTuple(name, fields): + return _coconut.collections.namedtuple(name, [x for x, t in fields]) + typing.NamedTuple = NamedTuple + NamedTuple = staticmethod(NamedTuple) ''', indent=1, newline=True, @@ -574,15 +583,12 @@ def NamedTuple(name, fields): import_typing_38=pycondition( (3, 8), if_lt=''' -try: - from typing_extensions import Protocol -except ImportError: +if not hasattr(typing, "Protocol"): class YouNeedToInstallTypingExtensions{object}: __slots__ = () def __init__(self): raise _coconut.TypeError('Protocols cannot be instantiated') - Protocol = YouNeedToInstallTypingExtensions -typing.Protocol = Protocol + typing.Protocol = YouNeedToInstallTypingExtensions '''.format(**format_dict), indent=1, newline=True, @@ -590,18 +596,15 @@ def __init__(self): import_typing_310=pycondition( (3, 10), if_lt=''' -try: - from typing_extensions import ParamSpec, TypeAlias, Concatenate -except ImportError: +if not hasattr(typing, "ParamSpec"): def ParamSpec(name, *args, **kwargs): """Runtime mock of typing.ParamSpec for Python 3.9 and earlier.""" return _coconut.typing.TypeVar(name) + typing.ParamSpec = ParamSpec +if not hasattr(typing, "TypeAlias") or not hasattr(typing, "Concatenate"): class you_need_to_install_typing_extensions{object}: __slots__ = () - TypeAlias = Concatenate = you_need_to_install_typing_extensions() -typing.ParamSpec = ParamSpec -typing.TypeAlias = TypeAlias -typing.Concatenate = Concatenate + typing.TypeAlias = typing.Concatenate = you_need_to_install_typing_extensions() '''.format(**format_dict), indent=1, newline=True, @@ -609,17 +612,15 @@ class you_need_to_install_typing_extensions{object}: import_typing_311=pycondition( (3, 11), if_lt=''' -try: - from typing_extensions import TypeVarTuple, Unpack -except ImportError: +if not hasattr(typing, "TypeVarTuple"): def TypeVarTuple(name, *args, **kwargs): """Runtime mock of typing.TypeVarTuple for Python 3.10 and earlier.""" return _coconut.typing.TypeVar(name) + typing.TypeVarTuple = TypeVarTuple +if not hasattr(typing, "Unpack"): class you_need_to_install_typing_extensions{object}: __slots__ = () - Unpack = you_need_to_install_typing_extensions() -typing.TypeVarTuple = TypeVarTuple -typing.Unpack = Unpack + typing.Unpack = you_need_to_install_typing_extensions() '''.format(**format_dict), indent=1, newline=True, diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index e9b16680f..85684eb18 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -20,8 +20,23 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} {import_pickle} {import_OrderedDict} {import_collections_abc} + typing = types.ModuleType("typing") + try: + import typing_extensions + except ImportError: + typing_extensions = None + else: + for _name in dir(typing_extensions): + if not _name.startswith("__"): + setattr(typing, _name, getattr(typing_extensions, _name)) + typing.__doc__ = "Coconut version of typing that makes use of typing.typing_extensions when possible.\n\n" + (getattr(typing, "__doc__") or "The typing module is not available at runtime in Python 3.4 or earlier; try hiding your typedefs behind an 'if TYPE_CHECKING:' block.") {import_typing} -{import_typing_36}{import_typing_38}{import_typing_310}{import_typing_311}{set_zip_longest} +{import_typing_36}{import_typing_38}{import_typing_310}{import_typing_311} + def _typing_getattr(name): + raise _coconut.AttributeError("typing.%s is not available on the current Python version and couldn't be looked up in typing_extensions; try hiding your typedefs behind an 'if TYPE_CHECKING:' block" % (name,)) + typing.__getattr__ = _typing_getattr + _typing_getattr = staticmethod(_typing_getattr) +{set_zip_longest} try: import numpy except ImportError: diff --git a/coconut/constants.py b/coconut/constants.py index dabdd2b5e..491a9da3a 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -430,6 +430,8 @@ def get_bool_env_var(env_var, default=False): # third-party backports "asyncio": ("trollius", (3, 4)), "enum": ("aenum", (3, 4)), + + # typing_extensions "typing.AsyncContextManager": ("typing_extensions./AsyncContextManager", (3, 6)), "typing.AsyncGenerator": ("typing_extensions./AsyncGenerator", (3, 6)), "typing.AsyncIterable": ("typing_extensions./AsyncIterable", (3, 6)), @@ -482,6 +484,10 @@ def get_bool_env_var(env_var, default=False): "typing.Unpack": ("typing_extensions./Unpack", (3, 11)), } +import_existing = { + "typing": "_coconut.typing", +} + self_match_types = ( "bool", "bytearray", diff --git a/coconut/root.py b/coconut/root.py index 6e2dbd917..e59330e6e 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 7 +DEVELOP = 8 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/primary.coco b/coconut/tests/src/cocotest/agnostic/primary.coco index 1b77e269b..7b7d3ef5b 100644 --- a/coconut/tests/src/cocotest/agnostic/primary.coco +++ b/coconut/tests/src/cocotest/agnostic/primary.coco @@ -1602,4 +1602,5 @@ def primary_test() -> bool: n = [0] assert n[0] == 0 assert_raises(-> m{{1:2,2:3}}, TypeError) + assert_raises((def -> from typing import blah), ImportError) # NOQA return True diff --git a/coconut/tests/src/cocotest/agnostic/specific.coco b/coconut/tests/src/cocotest/agnostic/specific.coco index 9c936dddd..2cd9d3858 100644 --- a/coconut/tests/src/cocotest/agnostic/specific.coco +++ b/coconut/tests/src/cocotest/agnostic/specific.coco @@ -180,6 +180,7 @@ def py37_spec_test() -> bool: assert l == list(range(10)) class HasVarGen[*Ts] # type: ignore assert HasVarGen `issubclass` object + assert typing.Protocol.__module__ == "typing_extensions" return True diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index 2ccc7269a..cb4f2b6c1 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1051,6 +1051,7 @@ forward 2""") == 900 really_long_var = 10 assert ret_args_kwargs(...=really_long_var) == ((), {"really_long_var": 10}) == ret_args_kwargs$(...=really_long_var)() assert ret_args_kwargs(123, ...=really_long_var, abc="abc") == ((123,), {"really_long_var": 10, "abc": "abc"}) == ret_args_kwargs$(123, ...=really_long_var, abc="abc")() + assert "Coconut version of typing" in typing.__doc__ # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index ee171e873..eee8c2de5 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -22,7 +22,7 @@ class AccessCounter(): self.counts[attr] += 1 return super(AccessCounter, self).__getattribute__(attr) -def assert_raises(c, exc=Exception): +def assert_raises(c, exc): """Test whether callable c raises an exception of type exc.""" try: c() @@ -231,9 +231,8 @@ addpattern def x! if x = False # type: ignore addpattern def x! = True # type: ignore # Type aliases: +import typing if sys.version_info >= (3, 5) or TYPE_CHECKING: - import typing - type list_or_tuple = list | tuple type func_to_int = -> int diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 923c74f90..5efc90641 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -100,7 +100,7 @@ def test_setup_none() -> bool: assert version("tag") assert version("-v") assert_raises(-> version("other"), CoconutException) - assert_raises(def -> raise CoconutException("derp").syntax_err(), SyntaxError) + assert_raises((def -> raise CoconutException("derp").syntax_err()), SyntaxError) assert coconut_eval("x -> x + 1")(2) == 3 assert coconut_eval("addpattern") @@ -316,6 +316,7 @@ else: match x: pass"""), CoconutStyleError, err_has="case x:") assert_raises(-> parse("obj."), CoconutStyleError, err_has="getattr") + assert_raises(-> parse("def x -> pass, 1"), CoconutStyleError, err_has="statement lambda") setup(strict=True, target="sys") assert_raises(-> parse("await f x"), CoconutParseError, err_has='invalid use of the keyword "await"') From 644d5e891d115b372462ec7ef079ba2471f124c8 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 26 May 2023 21:48:51 -0700 Subject: [PATCH 43/57] Fix py2 --- coconut/compiler/compiler.py | 4 ++-- coconut/compiler/templates/header.py_template | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 2cc135e15..51eaa6476 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -3102,10 +3102,10 @@ def single_import(self, path, imp_as, type_ignore=False): "try:", openindent + mod_name, closeindent + "except:", - openindent + mod_name + ' = _coconut.types.ModuleType("' + mod_name + '")', + openindent + mod_name + ' = _coconut.types.ModuleType(_coconut_py_str("' + mod_name + '"))', closeindent + "else:", openindent + "if not _coconut.isinstance(" + mod_name + ", _coconut.types.ModuleType):", - openindent + mod_name + ' = _coconut.types.ModuleType("' + mod_name + '")' + closeindent * 2, + openindent + mod_name + ' = _coconut.types.ModuleType(_coconut_py_str("' + mod_name + '"))' + closeindent * 2, )) out.append(".".join(fake_mods) + " = " + import_as_var) else: diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index 85684eb18..fafaf6d4a 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -20,7 +20,7 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} {import_pickle} {import_OrderedDict} {import_collections_abc} - typing = types.ModuleType("typing") + typing = types.ModuleType(_coconut_py_str("typing")) try: import typing_extensions except ImportError: From a0dcabba46403c532b95401b313dfb1ea71d50a5 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 26 May 2023 22:34:41 -0700 Subject: [PATCH 44/57] Fix typing universalization --- coconut/compiler/header.py | 2 +- coconut/root.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index c366419b6..7b6436314 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -544,7 +544,7 @@ async def __anext__(self): if not hasattr(typing, "TYPE_CHECKING"): typing.TYPE_CHECKING = False if not hasattr(typing, "Any"): - Any = Ellipsis + typing.Any = Ellipsis if not hasattr(typing, "cast"): def cast(t, x): """typing.cast[T](t: Type[T], x: Any) -> T = x""" diff --git a/coconut/root.py b/coconut/root.py index e59330e6e..9a29dc57a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 8 +DEVELOP = 9 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" From 312882e3baad2491966d1c573009671273af3476 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 16:30:56 -0700 Subject: [PATCH 45/57] Add async with for Resolves #753. --- .pre-commit-config.yaml | 4 - DOCS.md | 54 +++ coconut/compiler/compiler.py | 46 ++ coconut/compiler/grammar.py | 451 +++++++++--------- coconut/root.py | 2 +- .../src/cocotest/target_36/py36_test.coco | 47 +- 6 files changed, 367 insertions(+), 237 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5784b994..df224ace7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,3 @@ repos: - --aggressive - --experimental - --ignore=W503,E501,E722,E402 -- repo: https://github.com/asottile/add-trailing-comma - rev: v2.4.0 - hooks: - - id: add-trailing-comma diff --git a/DOCS.md b/DOCS.md index 6a79171b4..7b46a40d2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1547,6 +1547,60 @@ b = 2 c = a + b ``` +### `async with for` + +In modern Python `async` code, such as when using [`contextlib.aclosing`](https://docs.python.org/3/library/contextlib.html#contextlib.aclosing), it is often recommended to use a pattern like +```coconut_python +async with aclosing(my_generator()) as values: + async for value in values: + ... +``` +since it is substantially safer than the more syntactically straightforward +```coconut_python +async for value in my_generator(): + ... +``` + +This is especially true when using [`trio`](https://github.com/python-trio/trio), which [completely disallows iterating over `async` generators with `async for`](https://discuss.python.org/t/preventing-yield-inside-certain-context-managers/1091), instead requiring the above `async with ... async for` pattern using utilities such as [`trio_util.trio_async_generator`](https://trio-util.readthedocs.io/en/latest/#trio_util.trio_async_generator). + +Since this pattern can often be quite syntactically cumbersome, Coconut provides the shortcut syntax +``` +async with for aclosing(my_generator()) as values: + ... +``` +which compiles to exactly the pattern above. + +`async with for` also [supports pattern-matching, just like normal Coconut `for` loops](#match-for). + +##### Example + +**Coconut:** +```coconut +from trio_util import trio_async_generator + +@trio_async_generator +async def my_generator(): + # yield values, possibly from a nursery or cancel scope + # ... + +async with for value in my_generator(): + print(value) +``` + +**Python:** +```coconut_python +from trio_util import trio_async_generator + +@trio_async_generator +async def my_generator(): + # yield values, possibly from a nursery or cancel scope + # ... + +async with my_generator() as agen: + async for value in agen: + print(value) +``` + ### Handling Keyword/Variable Name Overlap In Coconut, the following keywords are also valid variable names: diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 51eaa6476..206fd679c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -746,6 +746,7 @@ def bind(cls): cls.new_testlist_star_expr <<= trace_attach(cls.new_testlist_star_expr_ref, cls.method("new_testlist_star_expr_handle")) cls.anon_namedtuple <<= trace_attach(cls.anon_namedtuple_ref, cls.method("anon_namedtuple_handle")) cls.base_match_for_stmt <<= trace_attach(cls.base_match_for_stmt_ref, cls.method("base_match_for_stmt_handle")) + cls.async_with_for_stmt <<= trace_attach(cls.async_with_for_stmt_ref, cls.method("async_with_for_stmt_handle")) cls.unsafe_typedef_tuple <<= trace_attach(cls.unsafe_typedef_tuple_ref, cls.method("unsafe_typedef_tuple_handle")) cls.funcname_typeparams <<= trace_attach(cls.funcname_typeparams_ref, cls.method("funcname_typeparams_handle")) cls.impl_call <<= trace_attach(cls.impl_call_ref, cls.method("impl_call_handle")) @@ -4002,6 +4003,51 @@ def base_match_for_stmt_handle(self, original, loc, tokens): body=body, ) + def async_with_for_stmt_handle(self, original, loc, tokens): + """Handle async with for loops.""" + if self.target_info < (3, 5): + raise self.make_err(CoconutTargetError, "async with for statements require Python 3.5+", original, loc, target="35") + + inner_toks, = tokens + + if "match" in inner_toks: + is_match = True + else: + internal_assert("normal" in inner_toks, "invalid async_with_for_stmt inner_toks", inner_toks) + is_match = False + + loop_vars, iter_item, body = inner_toks + temp_var = self.get_temp_var("async_with_for") + + if is_match: + loop = "async " + self.base_match_for_stmt_handle( + original, + loc, + [loop_vars, temp_var, body], + ) + else: + loop = handle_indentation( + """ +async for {loop_vars} in {temp_var}: +{body} + """, + ).format( + loop_vars=loop_vars, + temp_var=temp_var, + body=body, + ) + + return handle_indentation( + """ +async with {iter_item} as {temp_var}: + {loop} + """, + ).format( + iter_item=iter_item, + temp_var=temp_var, + loop=loop + ) + def string_atom_handle(self, tokens): """Handle concatenation of string literals.""" internal_assert(len(tokens) >= 1, "invalid string literal tokens", tokens) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index c2434dcb7..a02978936 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -801,7 +801,7 @@ class Grammar(object): imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") basenum = combine( integer + dot + Optional(integer) - | Optional(integer) + dot + integer, + | Optional(integer) + dot + integer ) | integer sci_e = combine(caseless_literal("e") + Optional(plus | neg_minus)) numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) @@ -965,13 +965,11 @@ class Grammar(object): ) + rbrace.suppress() dict_literal_ref = ( lbrace.suppress() - + Optional( - tokenlist( - Group(test + colon + test) - | dubstar_expr, - comma, - ), - ) + + Optional(tokenlist( + Group(test + colon + test) + | dubstar_expr, + comma, + )) + rbrace.suppress() ) test_expr = yield_expr | testlist_star_expr @@ -1054,7 +1052,7 @@ class Grammar(object): op_item = trace( typedef_op_item | partial_op_item - | base_op_item, + | base_op_item ) partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() @@ -1093,10 +1091,10 @@ class Grammar(object): (star | dubstar) + tfpdef | star_sep_arg | slash_sep_arg - | tfpdef_default, - ), - ), - ), + | tfpdef_default + ) + ) + ) ) parameters = condense(lparen + args_list + rparen) set_args_list = trace( @@ -1108,10 +1106,10 @@ class Grammar(object): (star | dubstar) + setname + setarg_comma | star_sep_setarg | slash_sep_setarg - | setname + Optional(default) + setarg_comma, - ), - ), - ), + | setname + Optional(default) + setarg_comma + ) + ) + ) ) match_args_list = trace( Group( @@ -1121,12 +1119,12 @@ class Grammar(object): (star | dubstar) + match | star # not star_sep because pattern-matching can handle star separators on any Python version | slash # not slash_sep as above - | match + Optional(equals.suppress() + test), + | match + Optional(equals.suppress() + test) ), comma, - ), - ), - ), + ) + ) + ) ) call_item = ( @@ -1149,10 +1147,10 @@ class Grammar(object): Group( questionmark | unsafe_name + condense(equals + questionmark) - | call_item, + | call_item ), comma, - ), + ) ) methodcaller_args = ( itemlist(condense(call_item), comma) @@ -1165,7 +1163,7 @@ class Grammar(object): sliceop = condense(unsafe_colon + slicetest) subscript = condense( slicetest + sliceop + Optional(sliceop) - | Optional(subscript_star) + test, + | Optional(subscript_star) + test ) subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test @@ -1183,7 +1181,7 @@ class Grammar(object): anon_namedtuple_ref = tokenlist( Group( unsafe_name + maybe_typedef + equals.suppress() + test - | ellipsis_tokens + maybe_typedef + equals.suppress() + refname, + | ellipsis_tokens + maybe_typedef + equals.suppress() + refname ), comma, ) @@ -1205,7 +1203,7 @@ class Grammar(object): lparen.suppress() + typedef_tuple + rparen.suppress() - ), + ) ) list_expr = Forward() @@ -1215,7 +1213,7 @@ class Grammar(object): multisemicolon | attach(comprehension_expr, add_bracks_handle) | namedexpr_test + ~comma - | list_expr, + | list_expr ) + rbrack.suppress(), array_literal_handle, ) @@ -1244,7 +1242,7 @@ class Grammar(object): (new_namedexpr_test + FollowedBy(rbrace))("test") | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") - | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr"), + | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") ) set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() @@ -1263,7 +1261,7 @@ class Grammar(object): | set_letter_literal | lazy_list | typedef_ellipsis - | ellipsis, + | ellipsis ) atom = ( # known_atom must come before name to properly parse string prefixes @@ -1307,7 +1305,7 @@ class Grammar(object): trailer = simple_trailer | complex_trailer attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( - lparen + Optional(methodcaller_args) + rparen.suppress(), + lparen + Optional(methodcaller_args) + rparen.suppress() ) attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) itemgetter_atom_tokens = dot.suppress() + OneOrMore(condense(Optional(dollar) + lbrack) + subscriptgrouplist + rbrack.suppress()) @@ -1344,7 +1342,7 @@ class Grammar(object): base_assign_item = condense( simple_assign | lparen + assignlist + rparen - | lbrack + assignlist + rbrack, + | lbrack + assignlist + rbrack ) star_assign_item_ref = condense(star + base_assign_item) assign_item = star_assign_item | base_assign_item @@ -1386,7 +1384,7 @@ class Grammar(object): disallow_keywords(reserved_vars) + ~any_string + atom_item - + Optional(power_in_impl_call), + + Optional(power_in_impl_call) ) impl_call = Forward() impl_call_ref = ( @@ -1397,7 +1395,7 @@ class Grammar(object): ZeroOrMore(unary) + ( impl_call | await_item + Optional(power) - ), + ) ) mulop = mul_star | div_slash | div_dubslash | percent | matrix_at @@ -1440,7 +1438,7 @@ class Grammar(object): infix_item = attach( Group(Optional(compose_expr)) + OneOrMore( - infix_op + Group(Optional(lambdef | compose_expr)), + infix_op + Group(Optional(lambdef | compose_expr)) ), infix_handle, ) @@ -1516,7 +1514,7 @@ class Grammar(object): partial_atom_tokens("partial"), partial_op_atom_tokens("op partial"), comp_pipe_expr("expr"), - ), + ) ) normal_pipe_expr = Forward() normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item @@ -1570,24 +1568,20 @@ class Grammar(object): | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, ) general_stmt_lambdef = ( - Group( - any_len_perm( - keyword("async"), - keyword("copyclosure"), - ), - ) + keyword("def").suppress() + Group(any_len_perm( + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + stmt_lambdef_params + arrow.suppress() + stmt_lambdef_body ) match_stmt_lambdef = ( - Group( - any_len_perm( - keyword("match").suppress(), - keyword("async"), - keyword("copyclosure"), - ), - ) + keyword("def").suppress() + Group(any_len_perm( + keyword("match").suppress(), + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + stmt_lambdef_match_params + arrow.suppress() + stmt_lambdef_body @@ -1605,15 +1599,13 @@ class Grammar(object): typedef_callable_arg = Group( test("arg") - | (dubstar.suppress() + refname)("paramspec"), - ) - typedef_callable_params = Optional( - Group( - labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") - | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() - | labeled_group(negable_atom_item, "arg"), - ), + | (dubstar.suppress() + refname)("paramspec") ) + typedef_callable_params = Optional(Group( + labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") + | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() + | labeled_group(negable_atom_item, "arg") + )) unsafe_typedef_callable = attach( Optional(keyword("async"), default="") + typedef_callable_params @@ -1669,7 +1661,7 @@ class Grammar(object): setname + colon_eq + ( test + ~colon_eq | attach(namedexpr, add_parens_handle) - ), + ) ) namedexpr_test <<= ( test + ~colon_eq @@ -1750,26 +1742,26 @@ class Grammar(object): imp_as = keyword("as").suppress() - imp_name import_item = Group( unsafe_dotted_imp_name + imp_as - | dotted_imp_name, + | dotted_imp_name ) from_import_item = Group( unsafe_imp_name + imp_as - | imp_name, + | imp_name ) import_names = Group( maybeparens(lparen, tokenlist(import_item, comma), rparen) - | star, + | star ) from_import_names = Group( maybeparens(lparen, tokenlist(from_import_item, comma), rparen) - | star, + | star ) basic_import = keyword("import").suppress() - import_names import_from_name = condense( ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot) - | star, + | star ) from_import = ( keyword("from").suppress() @@ -1815,7 +1807,7 @@ class Grammar(object): | string_atom | complex_number | Optional(neg_minus) + number - | match_dotted_name_const, + | match_dotted_name_const ) empty_const = fixto( lparen + rparen @@ -1868,34 +1860,32 @@ class Grammar(object): | lparen.suppress() + matchlist_star + rparen.suppress() )("star") - base_match = trace( - Group( - (negable_atom_item + arrow.suppress() + match)("view") - | match_string - | match_const("const") - | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") - | (keyword("in").suppress() + negable_atom_item)("in") - | iter_match - | match_lazy("lazy") - | sequence_match - | star_match - | (lparen.suppress() + match + rparen.suppress())("paren") - | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") - | ( - Group(Optional(set_letter)) - + lbrace.suppress() - + ( - Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) - | Group(always_match) + set_star + Optional(comma.suppress()) - | Group(Optional(tokenlist(match_const, comma))) - ) + rbrace.suppress() - )("set") - | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") - | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") - | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") - | Optional(keyword("as").suppress()) + setname("var"), - ), - ) + base_match = trace(Group( + (negable_atom_item + arrow.suppress() + match)("view") + | match_string + | match_const("const") + | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") + | (keyword("in").suppress() + negable_atom_item)("in") + | iter_match + | match_lazy("lazy") + | sequence_match + | star_match + | (lparen.suppress() + match + rparen.suppress())("paren") + | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") + | ( + Group(Optional(set_letter)) + + lbrace.suppress() + + ( + Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) + | Group(always_match) + set_star + Optional(comma.suppress()) + | Group(Optional(tokenlist(match_const, comma))) + ) + rbrace.suppress() + )("set") + | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") + | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") + | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") + | Optional(keyword("as").suppress()) + setname("var") + )) matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match @@ -1943,29 +1933,25 @@ class Grammar(object): destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) # both syntaxes here must be kept the same except for the keywords - case_match_co_syntax = trace( - Group( - (keyword("match") | keyword("case")).suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite, - ), - ) + case_match_co_syntax = trace(Group( + (keyword("match") | keyword("case")).suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + )) cases_stmt_co_syntax = ( (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) + dedent.suppress() + Optional(keyword("else").suppress() + suite) ) - case_match_py_syntax = trace( - Group( - keyword("case").suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite, - ), - ) + case_match_py_syntax = trace(Group( + keyword("case").suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + )) cases_stmt_py_syntax = ( keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) @@ -1979,26 +1965,34 @@ class Grammar(object): - ( lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item | testlist - ), + ) ) if_stmt = condense( addspace(keyword("if") + condense(namedexpr_test + suite)) - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) - - Optional(else_stmt), + - Optional(else_stmt) ) while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) + suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) + base_match_for_stmt = Forward() - base_match_for_stmt_ref = keyword("for").suppress() + many_match + keyword("in").suppress() - new_testlist_star_expr - colon.suppress() - condense(nocolon_suite - Optional(else_stmt)) + base_match_for_stmt_ref = ( + keyword("for").suppress() + + many_match + + keyword("in").suppress() + - new_testlist_star_expr + - suite_with_else_tokens + ) match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt except_item = ( testlist_has_comma("list") | test("test") ) - Optional( - keyword("as").suppress() - setname, + keyword("as").suppress() - setname ) except_clause = attach(keyword("except") + except_item, except_handle) except_star_clause = Forward() @@ -2011,7 +2005,7 @@ class Grammar(object): | keyword("except") - suite | OneOrMore(except_star_clause - suite) ) - Optional(else_stmt) - Optional(keyword("finally") - suite) - ), + ) ) with_item = addspace(test + Optional(keyword("as") + base_assign_item)) @@ -2025,14 +2019,12 @@ class Grammar(object): op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() - op_funcdef = trace( - attach( - Group(Optional(op_funcdef_arg)) - + op_funcdef_name - + Group(Optional(op_funcdef_arg)), - op_funcdef_handle, - ), - ) + op_funcdef = trace(attach( + Group(Optional(op_funcdef_arg)) + + op_funcdef_name + + Group(Optional(op_funcdef_arg)), + op_funcdef_handle, + )) return_typedef = Forward() return_typedef_ref = arrow.suppress() + typedef_test @@ -2042,18 +2034,16 @@ class Grammar(object): name_match_funcdef = Forward() op_match_funcdef = Forward() - op_match_funcdef_arg = Group( - Optional( - Group( - ( - lparen.suppress() - + match - + Optional(equals.suppress() + test) - + rparen.suppress() - ) | interior_name_match, - ), - ), - ) + op_match_funcdef_arg = Group(Optional( + Group( + ( + lparen.suppress() + + match + + Optional(equals.suppress() + test) + + rparen.suppress() + ) | interior_name_match + ) + )) name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard base_match_funcdef = trace(op_match_funcdef | name_match_funcdef) @@ -2067,21 +2057,17 @@ class Grammar(object): - dedent.suppress() ) ) - def_match_funcdef = trace( - attach( - base_match_funcdef - + end_func_colon - - func_suite, - join_match_funcdef, - ), - ) - match_def_modifiers = trace( - any_len_perm( - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - ), - ) + def_match_funcdef = trace(attach( + base_match_funcdef + + end_func_colon + - func_suite, + join_match_funcdef, + )) + match_def_modifiers = trace(any_len_perm( + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + )) match_funcdef = addspace(match_def_modifiers + def_match_funcdef) where_stmt = attach( @@ -2111,56 +2097,71 @@ class Grammar(object): | condense(newline - indent - math_funcdef_body - dedent) ) end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") - math_funcdef = trace( - attach( - condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, - math_funcdef_handle, - ), - ) - math_match_funcdef = trace( - addspace( - match_def_modifiers - + attach( - base_match_funcdef - + end_func_equals - + ( - attach(implicit_return_stmt, make_suite_handle) - | ( - newline.suppress() - indent.suppress() - + Optional(docstring) - + attach(math_funcdef_body, make_suite_handle) - + dedent.suppress() - ) - ), - join_match_funcdef, + math_funcdef = trace(attach( + condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, + math_funcdef_handle, + )) + math_match_funcdef = trace(addspace( + match_def_modifiers + + attach( + base_match_funcdef + + end_func_equals + + ( + attach(implicit_return_stmt, make_suite_handle) + | ( + newline.suppress() - indent.suppress() + + Optional(docstring) + + attach(math_funcdef_body, make_suite_handle) + + dedent.suppress() + ) ), - ), - ) + join_match_funcdef, + ) + )) async_stmt = Forward() + async_with_for_stmt = Forward() + async_with_for_stmt_ref = ( + labeled_group( + (keyword("async") + keyword("with") + keyword("for")).suppress() + + assignlist + keyword("in").suppress() + - test + - suite_with_else_tokens, + "normal", + ) + | labeled_group( + (any_len_perm( + keyword("match"), + required=(keyword("async"),) + ) + keyword("with") + keyword("for")).suppress() + + many_match + keyword("in").suppress() + - test + - suite_with_else_tokens, + "match", + ) + ) async_stmt_ref = addspace( keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for - | keyword("match").suppress() + keyword("async") + base_match_for_stmt, # handles match async for + | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for + | async_with_for_stmt ) async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) - async_match_funcdef = trace( - addspace( - any_len_perm( - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - required=(keyword("async").suppress(),), - ) + (def_match_funcdef | math_match_funcdef), - ), - ) + async_match_funcdef = trace(addspace( + any_len_perm( + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + (def_match_funcdef | math_match_funcdef), + )) async_keyword_normal_funcdef = Group( any_len_perm_at_least_one( keyword("yield"), keyword("copyclosure"), required=(keyword("async").suppress(),), - ), + ) ) + (funcdef | math_funcdef) async_keyword_match_funcdef = Group( any_len_perm_at_least_one( @@ -2170,7 +2171,7 @@ class Grammar(object): # addpattern is detected later keyword("addpattern"), required=(keyword("async").suppress(),), - ), + ) ) + (def_match_funcdef | math_match_funcdef) async_keyword_funcdef = Forward() async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef @@ -2185,7 +2186,7 @@ class Grammar(object): any_len_perm_at_least_one( keyword("yield"), keyword("copyclosure"), - ), + ) ) + (funcdef | math_funcdef) keyword_match_funcdef = Group( any_len_perm_at_least_one( @@ -2194,7 +2195,7 @@ class Grammar(object): keyword("match").suppress(), # addpattern is detected later keyword("addpattern"), - ), + ) ) + (def_match_funcdef | math_match_funcdef) keyword_funcdef = Forward() keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef @@ -2208,27 +2209,23 @@ class Grammar(object): ) datadef = Forward() - data_args = Group( - Optional( - lparen.suppress() + ZeroOrMore( - Group( - # everything here must end with arg_comma - (unsafe_name + arg_comma.suppress())("name") - | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") - | (star.suppress() + unsafe_name + arg_comma.suppress())("star") - | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") - | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type"), - ), - ) + rparen.suppress(), - ), - ) + data_args = Group(Optional( + lparen.suppress() + ZeroOrMore(Group( + # everything here must end with arg_comma + (unsafe_name + arg_comma.suppress())("name") + | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") + | (star.suppress() + unsafe_name + arg_comma.suppress())("star") + | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") + | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") + )) + rparen.suppress() + )) data_inherit = Optional(keyword("from").suppress() + testlist) data_suite = Group( colon.suppress() - ( (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") | simple_stmt("simple") - ) | newline("empty"), + ) | newline("empty") ) datadef_ref = ( Optional(decorators, default="") @@ -2242,7 +2239,7 @@ class Grammar(object): match_datadef = Forward() match_data_args = lparen.suppress() + Group( - match_args_list + match_guard, + match_args_list + match_guard ) + rparen.suppress() # we don't support type_params here since we don't support types match_datadef_ref = ( @@ -2261,8 +2258,8 @@ class Grammar(object): at.suppress() - Group( simple_decorator - | complex_decorator, - ), + | complex_decorator + ) ) decoratable_normal_funcdef_stmt = Forward() @@ -2282,7 +2279,7 @@ class Grammar(object): if_stmt | try_stmt | match_stmt - | passthrough_stmt, + | passthrough_stmt ) compound_stmt = trace( decoratable_class_stmt @@ -2293,7 +2290,7 @@ class Grammar(object): | async_stmt | match_for_stmt | simple_compound_stmt - | where_stmt, + | where_stmt ) endline_semicolon = Forward() endline_semicolon_ref = semicolon.suppress() + newline @@ -2304,7 +2301,7 @@ class Grammar(object): | pass_stmt | del_stmt | global_stmt - | nonlocal_stmt, + | nonlocal_stmt ) special_stmt = ( keyword_stmt @@ -2321,7 +2318,7 @@ class Grammar(object): simple_stmt <<= condense( simple_stmt_item + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) - + (newline | endline_semicolon), + + (newline | endline_semicolon) ) anything_stmt = Forward() stmt <<= final( @@ -2330,7 +2327,7 @@ class Grammar(object): # must be after destructuring due to ambiguity | cases_stmt # at the very end as a fallback case for the anything parser - | anything_stmt, + | anything_stmt ) base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) simple_suite = attach(stmt, make_suite_handle) @@ -2355,7 +2352,7 @@ class Grammar(object): unsafe_xonsh_command = originalTextFor( (Optional(at) + dollar | bang) + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) - + (parens | brackets | braces | unsafe_name), + + (parens | brackets | braces | unsafe_name) ) xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( single_parser, @@ -2417,7 +2414,7 @@ def get_tre_return_grammar(self, func_name): dot + unsafe_name | brackets # don't match the last set of parentheses - | parens + ~end_marker + ~rparen, + | parens + ~end_marker + ~rparen ), ) + original_function_call_tokens, @@ -2436,7 +2433,7 @@ def get_tre_return_grammar(self, func_name): | brackets | braces | lambdas - | ~colon + any_char, + | ~colon + any_char ) rest_of_tfpdef = originalTextFor( ZeroOrMore( @@ -2445,27 +2442,25 @@ def get_tre_return_grammar(self, func_name): | brackets | braces | lambdas - | ~comma + ~rparen + ~equals + any_char, - ), + | ~comma + ~rparen + ~equals + any_char + ) ) tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) type_comment = Optional( comment_tokens - | passthrough_item, + | passthrough_item ).suppress() parameters_tokens = Group( - Optional( - tokenlist( - Group( - dubstar - tfpdef_tokens - | star - Optional(tfpdef_tokens) - | slash - | tfpdef_default_tokens, - ) + type_comment, - comma + type_comment, - ), - ), + Optional(tokenlist( + Group( + dubstar - tfpdef_tokens + | star - Optional(tfpdef_tokens) + | slash + | tfpdef_default_tokens + ) + type_comment, + comma + type_comment, + )) ) split_func = ( diff --git a/coconut/root.py b/coconut/root.py index 9a29dc57a..7bb4d2c4a 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 9 +DEVELOP = 10 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index 43e420fa0..7bcf81a0d 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -1,4 +1,18 @@ -import asyncio, typing +import asyncio, typing, sys + +if sys.version_info >= (3, 10): + from contextlib import aclosing +elif sys.version_info >= (3, 7): + from contextlib import asynccontextmanager + @asynccontextmanager + async def aclosing(thing): + try: + yield thing + finally: + await thing.aclose() +else: + aclosing = None + def py36_test() -> bool: """Performs Python-3.6-specific tests.""" @@ -12,24 +26,49 @@ def py36_test() -> bool: for i in range(n): yield :await ayield(i) async def afor_test(): - # syntax 1 + # match syntax 1 got = [] async for int(i) in arange(5): got.append(i) assert got == range(5) |> list - # syntax 2 + # match syntax 2 got = [] async match for int(i) in arange(5): got.append(i) assert got == range(5) |> list - # syntax 3 + # match syntax 3 got = [] match async for int(i) in arange(5): got.append(i) assert got == range(5) |> list + if aclosing is not None: + # non-match + got = [] + async with for i in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + # match syntax 1 + got = [] + async with for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + # match syntax 2 + got = [] + async match with for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + # match syntax 3 + got = [] + match async with for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + return True loop.run_until_complete(afor_test()) From 44a0ae177aa0a8f3693a35854dbeb7ee37d182c1 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 18:58:14 -0700 Subject: [PATCH 46/57] Backport async generators Resolves #754. --- DOCS.md | 9 +- _coconut/__init__.pyi | 16 +- coconut/compiler/compiler.py | 28 ++- coconut/compiler/grammar.py | 6 +- coconut/compiler/templates/header.py_template | 6 + coconut/constants.py | 208 +++++++++--------- coconut/requirements.py | 208 +++++++++--------- coconut/root.py | 2 +- coconut/tests/main_test.py | 96 ++++---- .../src/cocotest/target_35/py35_test.coco | 109 +++++++++ .../src/cocotest/target_36/py36_test.coco | 113 ---------- coconut/tests/src/extras.coco | 6 +- 12 files changed, 425 insertions(+), 382 deletions(-) diff --git a/DOCS.md b/DOCS.md index 7b46a40d2..e94109ce5 100644 --- a/DOCS.md +++ b/DOCS.md @@ -90,10 +90,11 @@ The full list of optional dependencies is: - `watch`: enables use of the `--watch` flag. - `mypy`: enables use of the `--mypy` flag. - `backports`: installs libraries that backport newer Python features to older versions, which Coconut will automatically use instead of the standard library if the standard library is not available. Specifically: - - Installs [`typing`](https://pypi.org/project/typing/) and [`typing_extensions`](https://pypi.org/project/typing-extensions/) to backport [`typing`](https://docs.python.org/3/library/typing.html). - - Installs [`aenum`](https://pypi.org/project/aenum) to backport [`enum`](https://docs.python.org/3/library/enum.html). - - Installs [`trollius`](https://pypi.python.org/pypi/trollius) to backport [`asyncio`](https://docs.python.org/3/library/asyncio.html). - Installs [`dataclasses`](https://pypi.org/project/dataclasses/) to backport [`dataclasses`](https://docs.python.org/3/library/dataclasses.html). + - Installs [`typing`](https://pypi.org/project/typing/) to backport [`typing`](https://docs.python.org/3/library/typing.html) ([`typing_extensions`](https://pypi.org/project/typing-extensions/) is always installed for backporting individual `typing` objects). + - Installs [`aenum`](https://pypi.org/project/aenum) to backport [`enum`](https://docs.python.org/3/library/enum.html). + - Installs [`async_generator`](https://github.com/python-trio/async_generator) to backport [`async` generators](https://peps.python.org/pep-0525/) and [`asynccontextmanager`](https://docs.python.org/3/library/contextlib.html#contextlib.asynccontextmanager). + - Installs [`trollius`](https://pypi.python.org/pypi/trollius) to backport [`async`/`await`](https://docs.python.org/3/library/asyncio-task.html) and [`asyncio`](https://docs.python.org/3/library/asyncio.html). - `xonsh`: enables use of Coconut's [`xonsh` support](#xonsh-support). - `kernel`: lightweight subset of `jupyter` that only includes the dependencies that are strictly necessary for Coconut's [Jupyter kernel](#kernel). - `tests`: everything necessary to test the Coconut language itself. @@ -281,7 +282,7 @@ Finally, while Coconut will try to compile Python-3-specific syntax to its unive - the `nonlocal` keyword, - keyword-only function parameters (use [pattern-matching function definition](#pattern-matching-functions) for universal code), -- `async` and `await` statements (requires `--target 3.5`), +- `async` and `await` statements (requires a specific target; Coconut will attempt different backports based on the targeted version), - `:=` assignment expressions (requires `--target 3.8`), - positional-only function parameters (use [pattern-matching function definition](#pattern-matching-functions) for universal code) (requires `--target 3.8`), - `a[x, *y]` variadic generic syntax (use [type parameter syntax](#type-parameter-syntax) for universal code) (requires `--target 3.11`), and diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi index ed242669c..c4bf9406b 100644 --- a/_coconut/__init__.pyi +++ b/_coconut/__init__.pyi @@ -36,20 +36,23 @@ if sys.version_info >= (3,): else: import copy_reg as _copyreg -if sys.version_info >= (3, 4): - import asyncio as _asyncio +if sys.version_info >= (3,): + from itertools import zip_longest as _zip_longest else: - import trollius as _asyncio # type: ignore + from itertools import izip_longest as _zip_longest if sys.version_info < (3, 3): _abc = _collections else: from collections import abc as _abc -if sys.version_info >= (3,): - from itertools import zip_longest as _zip_longest +if sys.version_info >= (3, 4): + import asyncio as _asyncio else: - from itertools import izip_longest as _zip_longest + import trollius as _asyncio # type: ignore + +if sys.version_info >= (3, 5): + import async_generator as _async_generator try: import numpy as _numpy # type: ignore @@ -117,6 +120,7 @@ multiprocessing_dummy = _multiprocessing_dummy copyreg = _copyreg asyncio = _asyncio +async_generator = _async_generator pickle = _pickle if sys.version_info >= (2, 7): OrderedDict = collections.OrderedDict diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 206fd679c..2426e416c 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -1815,6 +1815,8 @@ def transform_returns(self, original, loc, raw_lines, tre_return_grammar=None, i and (not is_gen or self.target_info >= (3, 3)) # don't transform async returns if they're supported and (not is_async or self.target_info >= (3, 5)) + # don't transform async generators if they're supported + and (not (is_gen and is_async) or self.target_info >= (3, 6)) ): func_code = "".join(raw_lines) return func_code, tco, tre @@ -1874,6 +1876,20 @@ def transform_returns(self, original, loc, raw_lines, tre_return_grammar=None, i ) line = indent + "raise " + ret_err + "(" + to_return + ")" + comment + dedent + # handle async generator yields + if is_async and is_gen and self.target_info < (3, 6): + if self.yield_regex.match(base): + to_yield = base[len("yield"):].strip() + line = indent + "await _coconut.async_generator.yield_(" + to_yield + ")" + comment + dedent + elif self.yield_regex.search(base): + raise self.make_err( + CoconutTargetError, + "found Python 3.6 async generator yield in non-statement position (Coconut only backports async generator yields to 3.5 if they are at the start of the line)", + original, + loc, + target="36", + ) + # TRE tre_base = None if attempt_tre: @@ -2025,15 +2041,17 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, original, loc, target="sys", ) - elif is_gen and self.target_info < (3, 6): + elif self.target_info >= (3, 5): + if is_gen and self.target_info < (3, 6): + decorators += "@_coconut.async_generator.async_generator\n" + def_stmt = "async " + def_stmt + elif is_gen: raise self.make_err( CoconutTargetError, - "found Python 3.6 async generator", + "found Python 3.6 async generator (Coconut can only backport async generators as far back as 3.5)", original, loc, - target="36", + target="35", ) - elif self.target_info >= (3, 5): - def_stmt = "async " + def_stmt else: decorators += "@_coconut.asyncio.coroutine\n" diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index a02978936..1f772e451 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -2372,11 +2372,11 @@ class Grammar(object): whitespace_regex = compile_regex(r"\s") - def_regex = compile_regex(r"((async|addpattern|copyclosure)\s+)*def\b") + def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") - tco_disable_regex = compile_regex(r"try\b|(async\s+)?(with\b|for\b)|while\b") - return_regex = compile_regex(r"return\b") + tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") + return_regex = compile_regex(r"\breturn\b") noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") diff --git a/coconut/compiler/templates/header.py_template b/coconut/compiler/templates/header.py_template index fafaf6d4a..33f3b8503 100644 --- a/coconut/compiler/templates/header.py_template +++ b/coconut/compiler/templates/header.py_template @@ -17,6 +17,12 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE} from multiprocessing import dummy as multiprocessing_dummy {maybe_bind_lru_cache}{import_copyreg} {import_asyncio} + try: + import async_generator + except ImportError: + class you_need_to_install_async_generator{object}: + __slots__ = () + async_generator = you_need_to_install_async_generator() {import_pickle} {import_OrderedDict} {import_collections_abc} diff --git a/coconut/constants.py b/coconut/constants.py index 491a9da3a..dcba7ac79 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -424,64 +424,66 @@ def get_bool_env_var(env_var, default=False): "itertools.zip_longest": ("itertools./izip_longest", (3,)), "math.gcd": ("fractions./gcd", (3, 5)), "time.process_time": ("time./clock", (3, 3)), - # _dummy_thread was removed in Python 3.9, so this no longer works + # # _dummy_thread was removed in Python 3.9, so this no longer works # "_dummy_thread": ("dummy_thread", (3,)), # third-party backports "asyncio": ("trollius", (3, 4)), "enum": ("aenum", (3, 4)), - - # typing_extensions - "typing.AsyncContextManager": ("typing_extensions./AsyncContextManager", (3, 6)), - "typing.AsyncGenerator": ("typing_extensions./AsyncGenerator", (3, 6)), - "typing.AsyncIterable": ("typing_extensions./AsyncIterable", (3, 6)), - "typing.AsyncIterator": ("typing_extensions./AsyncIterator", (3, 6)), - "typing.Awaitable": ("typing_extensions./Awaitable", (3, 6)), - "typing.ChainMap": ("typing_extensions./ChainMap", (3, 6)), - "typing.ClassVar": ("typing_extensions./ClassVar", (3, 6)), - "typing.ContextManager": ("typing_extensions./ContextManager", (3, 6)), - "typing.Coroutine": ("typing_extensions./Coroutine", (3, 6)), - "typing.Counter": ("typing_extensions./Counter", (3, 6)), - "typing.DefaultDict": ("typing_extensions./DefaultDict", (3, 6)), - "typing.Deque": ("typing_extensions./Deque", (3, 6)), - "typing.NamedTuple": ("typing_extensions./NamedTuple", (3, 6)), - "typing.NewType": ("typing_extensions./NewType", (3, 6)), - "typing.NoReturn": ("typing_extensions./NoReturn", (3, 6)), - "typing.overload": ("typing_extensions./overload", (3, 6)), - "typing.Text": ("typing_extensions./Text", (3, 6)), - "typing.Type": ("typing_extensions./Type", (3, 6)), - "typing.TYPE_CHECKING": ("typing_extensions./TYPE_CHECKING", (3, 6)), - "typing.get_type_hints": ("typing_extensions./get_type_hints", (3, 6)), - "typing.OrderedDict": ("typing_extensions./OrderedDict", (3, 7)), - "typing.final": ("typing_extensions./final", (3, 8)), - "typing.Final": ("typing_extensions./Final", (3, 8)), - "typing.Literal": ("typing_extensions./Literal", (3, 8)), - "typing.Protocol": ("typing_extensions./Protocol", (3, 8)), - "typing.runtime_checkable": ("typing_extensions./runtime_checkable", (3, 8)), - "typing.TypedDict": ("typing_extensions./TypedDict", (3, 8)), - "typing.get_origin": ("typing_extensions./get_origin", (3, 8)), - "typing.get_args": ("typing_extensions./get_args", (3, 8)), - "typing.Annotated": ("typing_extensions./Annotated", (3, 9)), - "typing.Concatenate": ("typing_extensions./Concatenate", (3, 10)), - "typing.ParamSpec": ("typing_extensions./ParamSpec", (3, 10)), - "typing.ParamSpecArgs": ("typing_extensions./ParamSpecArgs", (3, 10)), - "typing.ParamSpecKwargs": ("typing_extensions./ParamSpecKwargs", (3, 10)), - "typing.TypeAlias": ("typing_extensions./TypeAlias", (3, 10)), - "typing.TypeGuard": ("typing_extensions./TypeGuard", (3, 10)), - "typing.is_typeddict": ("typing_extensions./is_typeddict", (3, 10)), - "typing.assert_never": ("typing_extensions./assert_never", (3, 11)), - "typing.assert_type": ("typing_extensions./assert_type", (3, 11)), - "typing.clear_overloads": ("typing_extensions./clear_overloads", (3, 11)), - "typing.dataclass_transform": ("typing_extensions./dataclass_transform", (3, 11)), - "typing.get_overloads": ("typing_extensions./get_overloads", (3, 11)), - "typing.LiteralString": ("typing_extensions./LiteralString", (3, 11)), - "typing.Never": ("typing_extensions./Never", (3, 11)), - "typing.NotRequired": ("typing_extensions./NotRequired", (3, 11)), - "typing.reveal_type": ("typing_extensions./reveal_type", (3, 11)), - "typing.Required": ("typing_extensions./Required", (3, 11)), - "typing.Self": ("typing_extensions./Self", (3, 11)), - "typing.TypeVarTuple": ("typing_extensions./TypeVarTuple", (3, 11)), - "typing.Unpack": ("typing_extensions./Unpack", (3, 11)), + "contextlib.asynccontextmanager": ("async_generator./asynccontextmanager", (3, 7)), + + # # typing_extensions (not needed since _coconut.typing has them + # # and mypy is happy to accept that they always live in typing) + # "typing.AsyncContextManager": ("typing_extensions./AsyncContextManager", (3, 6)), + # "typing.AsyncGenerator": ("typing_extensions./AsyncGenerator", (3, 6)), + # "typing.AsyncIterable": ("typing_extensions./AsyncIterable", (3, 6)), + # "typing.AsyncIterator": ("typing_extensions./AsyncIterator", (3, 6)), + # "typing.Awaitable": ("typing_extensions./Awaitable", (3, 6)), + # "typing.ChainMap": ("typing_extensions./ChainMap", (3, 6)), + # "typing.ClassVar": ("typing_extensions./ClassVar", (3, 6)), + # "typing.ContextManager": ("typing_extensions./ContextManager", (3, 6)), + # "typing.Coroutine": ("typing_extensions./Coroutine", (3, 6)), + # "typing.Counter": ("typing_extensions./Counter", (3, 6)), + # "typing.DefaultDict": ("typing_extensions./DefaultDict", (3, 6)), + # "typing.Deque": ("typing_extensions./Deque", (3, 6)), + # "typing.NamedTuple": ("typing_extensions./NamedTuple", (3, 6)), + # "typing.NewType": ("typing_extensions./NewType", (3, 6)), + # "typing.NoReturn": ("typing_extensions./NoReturn", (3, 6)), + # "typing.overload": ("typing_extensions./overload", (3, 6)), + # "typing.Text": ("typing_extensions./Text", (3, 6)), + # "typing.Type": ("typing_extensions./Type", (3, 6)), + # "typing.TYPE_CHECKING": ("typing_extensions./TYPE_CHECKING", (3, 6)), + # "typing.get_type_hints": ("typing_extensions./get_type_hints", (3, 6)), + # "typing.OrderedDict": ("typing_extensions./OrderedDict", (3, 7)), + # "typing.final": ("typing_extensions./final", (3, 8)), + # "typing.Final": ("typing_extensions./Final", (3, 8)), + # "typing.Literal": ("typing_extensions./Literal", (3, 8)), + # "typing.Protocol": ("typing_extensions./Protocol", (3, 8)), + # "typing.runtime_checkable": ("typing_extensions./runtime_checkable", (3, 8)), + # "typing.TypedDict": ("typing_extensions./TypedDict", (3, 8)), + # "typing.get_origin": ("typing_extensions./get_origin", (3, 8)), + # "typing.get_args": ("typing_extensions./get_args", (3, 8)), + # "typing.Annotated": ("typing_extensions./Annotated", (3, 9)), + # "typing.Concatenate": ("typing_extensions./Concatenate", (3, 10)), + # "typing.ParamSpec": ("typing_extensions./ParamSpec", (3, 10)), + # "typing.ParamSpecArgs": ("typing_extensions./ParamSpecArgs", (3, 10)), + # "typing.ParamSpecKwargs": ("typing_extensions./ParamSpecKwargs", (3, 10)), + # "typing.TypeAlias": ("typing_extensions./TypeAlias", (3, 10)), + # "typing.TypeGuard": ("typing_extensions./TypeGuard", (3, 10)), + # "typing.is_typeddict": ("typing_extensions./is_typeddict", (3, 10)), + # "typing.assert_never": ("typing_extensions./assert_never", (3, 11)), + # "typing.assert_type": ("typing_extensions./assert_type", (3, 11)), + # "typing.clear_overloads": ("typing_extensions./clear_overloads", (3, 11)), + # "typing.dataclass_transform": ("typing_extensions./dataclass_transform", (3, 11)), + # "typing.get_overloads": ("typing_extensions./get_overloads", (3, 11)), + # "typing.LiteralString": ("typing_extensions./LiteralString", (3, 11)), + # "typing.Never": ("typing_extensions./Never", (3, 11)), + # "typing.NotRequired": ("typing_extensions./NotRequired", (3, 11)), + # "typing.reveal_type": ("typing_extensions./reveal_type", (3, 11)), + # "typing.Required": ("typing_extensions./Required", (3, 11)), + # "typing.Self": ("typing_extensions./Self", (3, 11)), + # "typing.TypeVarTuple": ("typing_extensions./TypeVarTuple", (3, 11)), + # "typing.Unpack": ("typing_extensions./Unpack", (3, 11)), } import_existing = { @@ -803,11 +805,20 @@ def get_bool_env_var(env_var, default=False): PURE_PYTHON = get_bool_env_var(pure_python_env_var) # the different categories here are defined in requirements.py, -# anything after a colon is ignored but allows different versions -# for different categories, and tuples denote the use of environment -# markers as specified in requirements.py +# tuples denote the use of environment markers all_reqs = { "main": ( + ("argparse", "py<27"), + ("psutil", "py>=27"), + ("futures", "py<3"), + ("backports.functools-lru-cache", "py<3"), + ("prompt_toolkit", "py<3"), + ("prompt_toolkit", "py>=3"), + ("pygments", "py<39"), + ("pygments", "py>=39"), + ("typing_extensions", "py==35"), + ("typing_extensions", "py==36"), + ("typing_extensions", "py37"), ), "cpython": ( "cPyparsing", @@ -815,32 +826,12 @@ def get_bool_env_var(env_var, default=False): "purepython": ( "pyparsing", ), - "non-py26": ( - "psutil", - ), - "py2": ( - "futures", - "backports.functools-lru-cache", - ("prompt_toolkit", "mark2"), - ), - "py3": ( - ("prompt_toolkit", "mark3"), - ), - "py26": ( - "argparse", - ), - "py<39": ( - ("pygments", "mark<39"), - ), - "py39": ( - ("pygments", "mark39"), - ), "kernel": ( - ("ipython", "py2"), + ("ipython", "py<3"), ("ipython", "py3;py<37"), ("ipython", "py==37"), ("ipython", "py38"), - ("ipykernel", "py2"), + ("ipykernel", "py<3"), ("ipykernel", "py3;py<38"), ("ipykernel", "py38"), ("jupyter-client", "py<35"), @@ -848,7 +839,7 @@ def get_bool_env_var(env_var, default=False): ("jupyter-client", "py36"), ("jedi", "py<39"), ("jedi", "py39"), - ("pywinpty", "py2;windows"), + ("pywinpty", "py<3;windows"), ), "jupyter": ( "jupyter", @@ -862,9 +853,7 @@ def get_bool_env_var(env_var, default=False): "mypy": ( "mypy[python2]", "types-backports", - ("typing_extensions", "py==35"), - ("typing_extensions", "py==36"), - ("typing_extensions", "py37"), + ("typing", "py<35"), ), "watch": ( "watchdog", @@ -875,13 +864,11 @@ def get_bool_env_var(env_var, default=False): ("xonsh", "py38"), ), "backports": ( - ("trollius", "py2;cpy"), + ("trollius", "py<3;cpy"), ("aenum", "py<34"), ("dataclasses", "py==36"), ("typing", "py<35"), - ("typing_extensions", "py==35"), - ("typing_extensions", "py==36"), - ("typing_extensions", "py37"), + ("async_generator", "py<37"), ), "dev": ( ("pre-commit", "py3"), @@ -890,8 +877,8 @@ def get_bool_env_var(env_var, default=False): ), "docs": ( "sphinx", - ("pygments", "mark<39"), - ("pygments", "mark39"), + ("pygments", "py<39"), + ("pygments", "py>=39"), "myst-parser", "pydata-sphinx-theme", ), @@ -900,7 +887,7 @@ def get_bool_env_var(env_var, default=False): ("pytest", "py36"), "pexpect", ("numpy", "py34"), - ("numpy", "py2;cpy"), + ("numpy", "py<3;cpy"), ("pandas", "py36"), ), } @@ -909,17 +896,17 @@ def get_bool_env_var(env_var, default=False): min_versions = { "cPyparsing": (2, 4, 7, 1, 2, 1), ("pre-commit", "py3"): (3,), - "psutil": (5,), + ("psutil", "py>=27"): (5,), "jupyter": (1, 0), "types-backports": (0, 1), - "futures": (3, 4), - "backports.functools-lru-cache": (1, 6), - "argparse": (1, 4), + ("futures", "py<3"): (3, 4), + ("backports.functools-lru-cache", "py<3"): (1, 6), + ("argparse", "py<27"): (1, 4), "pexpect": (4,), - ("trollius", "py2;cpy"): (2, 2), + ("trollius", "py<3;cpy"): (2, 2), "requests": (2, 31), ("numpy", "py34"): (1,), - ("numpy", "py2;cpy"): (1,), + ("numpy", "py<3;cpy"): (1,), ("dataclasses", "py==36"): (0, 8), ("aenum", "py<34"): (3,), "pydata-sphinx-theme": (0, 13), @@ -931,9 +918,10 @@ def get_bool_env_var(env_var, default=False): ("ipython", "py38"): (8,), ("ipykernel", "py38"): (6,), ("jedi", "py39"): (0, 18), - ("pygments", "mark39"): (2, 15), + ("pygments", "py>=39"): (2, 15), ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), + ("async_generator", "py<37"): (1, 10), # pinned reqs: (must be added to pinned_reqs below) @@ -956,20 +944,20 @@ def get_bool_env_var(env_var, default=False): ("xonsh", "py<36"): (0, 9), ("typing_extensions", "py==35"): (3, 10), # don't upgrade this to allow all versions - ("prompt_toolkit", "mark3"): (1,), + ("prompt_toolkit", "py>=3"): (1,), # don't upgrade this; it breaks on Python 2.6 ("pytest", "py<36"): (3,), # don't upgrade this; it breaks on unix "vprof": (0, 36), # don't upgrade this; it breaks on Python 3.4 - ("pygments", "mark<39"): (2, 3), + ("pygments", "py<39"): (2, 3), # don't upgrade these; they break on Python 2 ("jupyter-client", "py<35"): (5, 3), - ("pywinpty", "py2;windows"): (0, 5), + ("pywinpty", "py<3;windows"): (0, 5), ("jupyter-console", "py<35"): (5, 2), - ("ipython", "py2"): (5, 4), - ("ipykernel", "py2"): (4, 10), - ("prompt_toolkit", "mark2"): (1,), + ("ipython", "py<3"): (5, 4), + ("ipykernel", "py<3"): (4, 10), + ("prompt_toolkit", "py<3"): (1,), "watchdog": (0, 10), "papermill": (1, 2), # don't upgrade this; it breaks with old IPython versions @@ -995,15 +983,15 @@ def get_bool_env_var(env_var, default=False): ("jupyterlab", "py35"), ("xonsh", "py<36"), ("typing_extensions", "py==35"), - ("prompt_toolkit", "mark3"), + ("prompt_toolkit", "py>=3"), ("pytest", "py<36"), "vprof", - ("pygments", "mark<39"), - ("pywinpty", "py2;windows"), + ("pygments", "py<39"), + ("pywinpty", "py<3;windows"), ("jupyter-console", "py<35"), - ("ipython", "py2"), - ("ipykernel", "py2"), - ("prompt_toolkit", "mark2"), + ("ipython", "py<3"), + ("ipykernel", "py<3"), + ("prompt_toolkit", "py<3"), "watchdog", "papermill", ("jedi", "py<39"), @@ -1018,9 +1006,9 @@ def get_bool_env_var(env_var, default=False): ("jupyter-client", "py==35"): _, "pyparsing": _, "cPyparsing": (_, _, _), - ("prompt_toolkit", "mark2"): _, + ("prompt_toolkit", "py<3"): _, ("jedi", "py<39"): _, - ("pywinpty", "py2;windows"): _, + ("pywinpty", "py<3;windows"): _, ("ipython", "py3;py<37"): _, } diff --git a/coconut/requirements.py b/coconut/requirements.py index 04be698d7..93bf6b8e9 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -25,7 +25,6 @@ from coconut.constants import ( CPYTHON, PY34, - PY39, IPY, MYPY, XONSH, @@ -71,89 +70,122 @@ def get_base_req(req, include_extras=True): return req +def process_mark(mark): + """Get the check string and whether it currently applies for the given mark.""" + assert not mark.startswith("py2"), "confusing mark; should be changed: " + mark + if mark.startswith("py=="): + ver = mark[len("py=="):] + if len(ver) == 1: + ver_tuple = (int(ver),) + else: + ver_tuple = (int(ver[0]), int(ver[1:])) + next_ver_tuple = get_next_version(ver_tuple) + check_str = ( + "python_version>='" + ver_tuple_to_str(ver_tuple) + "'" + + " and python_version<'" + ver_tuple_to_str(next_ver_tuple) + "'" + ) + holds_now = ( + sys.version_info >= ver_tuple + and sys.version_info < next_ver_tuple + ) + elif mark in ("py3", "py>=3"): + check_str = "python_version>='3'" + holds_now = not PY2 + elif mark == "py<3": + check_str = "python_version<'3'" + holds_now = PY2 + elif mark.startswith("py<"): + full_ver = mark[len("py<"):] + main_ver, sub_ver = full_ver[0], full_ver[1:] + check_str = "python_version<'{main}.{sub}'".format(main=main_ver, sub=sub_ver) + holds_now = sys.version_info < (int(main_ver), int(sub_ver)) + elif mark.startswith("py") or mark.startswith("py>="): + full_ver = mark[len("py"):] + if full_ver.startswith(">="): + full_ver = full_ver[len(">="):] + main_ver, sub_ver = full_ver[0], full_ver[1:] + check_str = "python_version>='{main}.{sub}'".format(main=main_ver, sub=sub_ver) + holds_now = sys.version_info >= (int(main_ver), int(sub_ver)) + elif mark == "cpy": + check_str = "platform_python_implementation=='CPython'" + holds_now = CPYTHON + elif mark == "windows": + check_str = "os_name=='nt'" + holds_now = WINDOWS + elif mark.startswith("mark"): + check_str = None + holds_now = True + else: + raise ValueError("unknown env marker " + repr(mark)) + return check_str, holds_now + + +def get_req_str(req): + """Get the str that properly versions the given req.""" + req_str = get_base_req(req) + ">=" + ver_tuple_to_str(min_versions[req]) + if req in max_versions: + max_ver = max_versions[req] + if max_ver is None: + max_ver = get_next_version(min_versions[req]) + if None in max_ver: + assert all(v is None for v in max_ver), "invalid max version " + repr(max_ver) + max_ver = get_next_version(min_versions[req], len(max_ver) - 1) + req_str += ",<" + ver_tuple_to_str(max_ver) + return req_str + + +def get_env_markers(req): + """Get the environment markers for the given req.""" + if isinstance(req, tuple): + return req[1].split(";") + else: + return () + + def get_reqs(which): """Gets requirements from all_reqs with versions.""" reqs = [] for req in all_reqs[which]: + req_str = get_req_str(req) use_req = True - req_str = get_base_req(req) + ">=" + ver_tuple_to_str(min_versions[req]) - if req in max_versions: - max_ver = max_versions[req] - if max_ver is None: - max_ver = get_next_version(min_versions[req]) - if None in max_ver: - assert all(v is None for v in max_ver), "invalid max version " + repr(max_ver) - max_ver = get_next_version(min_versions[req], len(max_ver) - 1) - req_str += ",<" + ver_tuple_to_str(max_ver) - env_marker = req[1] if isinstance(req, tuple) else None - if env_marker: - markers = [] - for mark in env_marker.split(";"): - if mark.startswith("py=="): - ver = mark[len("py=="):] - if len(ver) == 1: - ver_tuple = (int(ver),) - else: - ver_tuple = (int(ver[0]), int(ver[1:])) - next_ver_tuple = get_next_version(ver_tuple) - if supports_env_markers: - markers.append("python_version>='" + ver_tuple_to_str(ver_tuple) + "'") - markers.append("python_version<'" + ver_tuple_to_str(next_ver_tuple) + "'") - elif sys.version_info < ver_tuple or sys.version_info >= next_ver_tuple: - use_req = False - break - elif mark == "py2": - if supports_env_markers: - markers.append("python_version<'3'") - elif not PY2: - use_req = False - break - elif mark == "py3": - if supports_env_markers: - markers.append("python_version>='3'") - elif PY2: - use_req = False - break - elif mark.startswith("py3") or mark.startswith("py>=3"): - mark = mark[len("py"):] - if mark.startswith(">="): - mark = mark[len(">="):] - ver = mark[len("3"):] - if supports_env_markers: - markers.append("python_version>='3.{ver}'".format(ver=ver)) - elif sys.version_info < (3, ver): - use_req = False - break - elif mark.startswith("py<3"): - ver = mark[len("py<3"):] - if supports_env_markers: - markers.append("python_version<'3.{ver}'".format(ver=ver)) - elif sys.version_info >= (3, ver): - use_req = False - break - elif mark == "cpy": - if supports_env_markers: - markers.append("platform_python_implementation=='CPython'") - elif not CPYTHON: - use_req = False - break - elif mark == "windows": - if supports_env_markers: - markers.append("os_name=='nt'") - elif not WINDOWS: - use_req = False - break - elif mark.startswith("mark"): - pass # ignore - else: - raise ValueError("unknown env marker " + repr(mark)) - if markers: - req_str += ";" + " and ".join(markers) + markers = [] + for mark in get_env_markers(req): + check_str, holds_now = process_mark(mark) + if supports_env_markers: + if check_str is not None: + markers.append(check_str) + else: + if not holds_now: + use_req = False + break + if markers: + req_str += ";" + " and ".join(markers) if use_req: reqs.append(req_str) return reqs +def get_main_reqs(main_reqs_name): + """Get the main requirements and extras.""" + requirements = [] + extras = {} + if using_modern_setuptools: + for req in all_reqs[main_reqs_name]: + req_str = get_req_str(req) + markers = [] + for mark in get_env_markers(req): + check_str, _ = process_mark(mark) + if check_str is not None: + markers.append(check_str) + if markers: + extras.setdefault(":" + " and ".join(markers), []).append(req_str) + else: + requirements.append(req_str) + else: + requirements += get_reqs(main_reqs_name) + return requirements, extras + + def uniqueify(reqs): """Make a list of requirements unique.""" return list(set(reqs)) @@ -181,7 +213,7 @@ def everything_in(req_dict): # SETUP: # ----------------------------------------------------------------------------------------------------------------------- -requirements = get_reqs("main") +requirements, reqs_extras = get_main_reqs("main") extras = { "kernel": get_reqs("kernel"), @@ -218,6 +250,9 @@ def everything_in(req_dict): if not PY34: extras["dev"] = unique_wrt(extras["dev"], extras["mypy"]) +# has to come after dev so they don't get included in it +extras.update(reqs_extras) + if PURE_PYTHON: # override necessary for readthedocs requirements += get_reqs("purepython") @@ -232,29 +267,6 @@ def everything_in(req_dict): else: requirements += get_reqs("purepython") -if using_modern_setuptools: - # modern method - extras[":python_version<'2.7'"] = get_reqs("py26") - extras[":python_version>='2.7'"] = get_reqs("non-py26") - extras[":python_version<'3'"] = get_reqs("py2") - extras[":python_version>='3'"] = get_reqs("py3") - extras[":python_version<'3.9'"] = get_reqs("py<39") - extras[":python_version>='3.9'"] = get_reqs("py39") -else: - # old method - if PY26: - requirements += get_reqs("py26") - else: - requirements += get_reqs("non-py26") - if PY2: - requirements += get_reqs("py2") - else: - requirements += get_reqs("py3") - if PY39: - requirements += get_reqs("py39") - else: - requirements += get_reqs("py<39") - # ----------------------------------------------------------------------------------------------------------------------- # MAIN: # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/root.py b/coconut/root.py index 7bb4d2c4a..c9937e948 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 10 +DEVELOP = 11 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 990cc4ba5..75796bd64 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -53,6 +53,8 @@ PY38, PY39, PY310, + supported_py2_vers, + supported_py3_vers, icoconut_default_kernel_names, icoconut_custom_kernel_name, mypy_err_infixes, @@ -143,6 +145,12 @@ + "', '".join((icoconut_custom_kernel_name,) + icoconut_default_kernel_names) + "'" ) +always_sys_versions = ( + supported_py2_vers[-1], + supported_py3_vers[-2], + supported_py3_vers[-1], +) + # ----------------------------------------------------------------------------------------------------------------------- # UTILITIES: @@ -485,7 +493,7 @@ def comp_agnostic(args=[], **kwargs): comp(path="cocotest", folder="agnostic", args=args, **kwargs) -def comp_2(args=[], **kwargs): +def comp_2(args=[], always_sys=False, **kwargs): """Compiles target_2.""" # remove --mypy checking for target_2 to avoid numpy errors try: @@ -494,27 +502,27 @@ def comp_2(args=[], **kwargs): pass else: args = args[:mypy_ind] - comp(path="cocotest", folder="target_2", args=["--target", "2"] + args, **kwargs) + comp(path="cocotest", folder="target_2", args=["--target", "2" if not always_sys else "sys"] + args, **kwargs) -def comp_3(args=[], **kwargs): +def comp_3(args=[], always_sys=False, **kwargs): """Compiles target_3.""" - comp(path="cocotest", folder="target_3", args=["--target", "3"] + args, **kwargs) + comp(path="cocotest", folder="target_3", args=["--target", "3" if not always_sys else "sys"] + args, **kwargs) -def comp_35(args=[], **kwargs): +def comp_35(args=[], always_sys=False, **kwargs): """Compiles target_35.""" - comp(path="cocotest", folder="target_35", args=["--target", "35"] + args, **kwargs) + comp(path="cocotest", folder="target_35", args=["--target", "35" if not always_sys else "sys"] + args, **kwargs) -def comp_36(args=[], **kwargs): +def comp_36(args=[], always_sys=False, **kwargs): """Compiles target_36.""" - comp(path="cocotest", folder="target_36", args=["--target", "36"] + args, **kwargs) + comp(path="cocotest", folder="target_36", args=["--target", "36" if not always_sys else "sys"] + args, **kwargs) -def comp_38(args=[], **kwargs): +def comp_38(args=[], always_sys=False, **kwargs): """Compiles target_38.""" - comp(path="cocotest", folder="target_38", args=["--target", "38"] + args, **kwargs) + comp(path="cocotest", folder="target_38", args=["--target", "38" if not always_sys else "sys"] + args, **kwargs) def comp_sys(args=[], **kwargs): @@ -538,7 +546,7 @@ def run_extras(**kwargs): call_python([os.path.join(dest, "extras.py")], assert_output=True, check_errors=False, stderr_first=True, **kwargs) -def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=False, **kwargs): +def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=False, always_sys=False, **kwargs): """Compiles and runs tests.""" if agnostic_target is None: agnostic_args = args @@ -548,16 +556,19 @@ def run(args=[], agnostic_target=None, use_run_arg=False, convert_to_import=Fals with using_dest(): with (using_dest(additional_dest) if "--and" in args else noop_ctx()): + spec_kwargs = kwargs.copy() + spec_kwargs["always_sys"] = always_sys if PY2: - comp_2(args, **kwargs) + comp_2(args, **spec_kwargs) else: - comp_3(args, **kwargs) + comp_3(args, **spec_kwargs) if sys.version_info >= (3, 5): - comp_35(args, **kwargs) + comp_35(args, **spec_kwargs) if sys.version_info >= (3, 6): - comp_36(args, **kwargs) + comp_36(args, **spec_kwargs) if sys.version_info >= (3, 8): - comp_38(args, **kwargs) + comp_38(args, **spec_kwargs) + comp_agnostic(agnostic_args, **kwargs) comp_sys(args, **kwargs) comp_non_strict(args, **kwargs) @@ -677,6 +688,31 @@ def test_code(self): def test_target_3_snip(self): call(["coconut", "-t3", "-c", target_3_snip], assert_output=True) + if MYPY: + def test_universal_mypy_snip(self): + call( + ["coconut", "-c", mypy_snip, "--mypy"], + assert_output=mypy_snip_err_3, + check_errors=False, + check_mypy=False, + ) + + def test_sys_mypy_snip(self): + call( + ["coconut", "--target", "sys", "-c", mypy_snip, "--mypy"], + assert_output=mypy_snip_err_3, + check_errors=False, + check_mypy=False, + ) + + def test_no_wrap_mypy_snip(self): + call( + ["coconut", "--target", "sys", "--no-wrap", "-c", mypy_snip, "--mypy"], + assert_output=mypy_snip_err_3, + check_errors=False, + check_mypy=False, + ) + def test_pipe(self): call('echo ' + escape(coconut_snip) + "| coconut -s", shell=True, assert_output=True) @@ -775,33 +811,13 @@ def test_normal(self): run() if MYPY: - def test_universal_mypy_snip(self): - call( - ["coconut", "-c", mypy_snip, "--mypy"], - assert_output=mypy_snip_err_3, - check_errors=False, - check_mypy=False, - ) - - def test_sys_mypy_snip(self): - call( - ["coconut", "--target", "sys", "-c", mypy_snip, "--mypy"], - assert_output=mypy_snip_err_3, - check_errors=False, - check_mypy=False, - ) - - def test_no_wrap_mypy_snip(self): - call( - ["coconut", "--target", "sys", "--no-wrap", "-c", mypy_snip, "--mypy"], - assert_output=mypy_snip_err_3, - check_errors=False, - check_mypy=False, - ) - def test_mypy_sys(self): run(["--mypy"] + mypy_args, agnostic_target="sys", expect_retcode=None, check_errors=False) # fails due to tutorial mypy errors + if sys.version_info[:2] in always_sys_versions: + def test_always_sys(self): + run(["--line-numbers"], agnostic_target="sys", always_sys=True) + # run fewer tests on Windows so appveyor doesn't time out if not WINDOWS: def test_line_numbers_keep_lines(self): diff --git a/coconut/tests/src/cocotest/target_35/py35_test.coco b/coconut/tests/src/cocotest/target_35/py35_test.coco index 892b98829..877e2f340 100644 --- a/coconut/tests/src/cocotest/target_35/py35_test.coco +++ b/coconut/tests/src/cocotest/target_35/py35_test.coco @@ -1,7 +1,116 @@ +import sys, asyncio, typing + +if sys.version_info >= (3, 10): + from contextlib import aclosing +else: + from contextlib import asynccontextmanager + @asynccontextmanager + async def aclosing(thing): + try: + yield thing + finally: + await thing.aclose() + + def py35_test() -> bool: """Performs Python-3.5-specific tests.""" assert .attr |> repr == "operator.attrgetter('attr')" assert .method(1) |> repr == "operator.methodcaller('method', 1)" assert pow$(1) |> repr == "functools.partial(, 1)" assert .[1] |> repr == "operator.itemgetter(1)" + + loop = asyncio.new_event_loop() + + async def ayield(x) = x + :async def arange(n): + for i in range(n): + yield :await ayield(i) + async def afor_test(): + # match syntax 1 + got = [] + async for int(i) in arange(5): + got.append(i) + assert got == range(5) |> list + + # match syntax 2 + got = [] + async match for int(i) in arange(5): + got.append(i) + assert got == range(5) |> list + + # match syntax 3 + got = [] + match async for int(i) in arange(5): + got.append(i) + assert got == range(5) |> list + + # non-match + got = [] + async with for i in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + # match syntax 1 + got = [] + async with for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + # match syntax 2 + got = [] + async match with for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + # match syntax 3 + got = [] + match async with for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + + return True + loop.run_until_complete(afor_test()) + + async yield def toa(it): + for x in it: + yield x + match yield async def arange_(int(n)): + for x in range(n): + yield x + async def aconsume(ait): + async for _ in ait: + pass + l: typing.List[int] = [] + async def aiter_test(): + range(10) |> toa |> fmap$(l.append) |> aconsume |> await + arange_(10) |> fmap$(l.append) |> aconsume |> await + loop.run_until_complete(aiter_test()) + assert l == list(range(10)) + list(range(10)) + + async def arec(x) = await arec(x-1) if x else x + async def outer_func(): + funcs = [] + for x in range(5): + funcs.append(async copyclosure def -> x) + return funcs + async def await_all(xs) = [await x for x in xs] + async def atest(): + assert ( + 10 + |> arec + |> await + |> (.+10) + |> arec + |> await + ) == 0 + assert ( + outer_func() + |> await + |> map$(call) + |> await_all + |> await + ) == range(5) |> list + loop.run_until_complete(atest()) + + loop.close() return True diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index 7bcf81a0d..f90b3254f 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -1,118 +1,5 @@ -import asyncio, typing, sys - -if sys.version_info >= (3, 10): - from contextlib import aclosing -elif sys.version_info >= (3, 7): - from contextlib import asynccontextmanager - @asynccontextmanager - async def aclosing(thing): - try: - yield thing - finally: - await thing.aclose() -else: - aclosing = None - - def py36_test() -> bool: """Performs Python-3.6-specific tests.""" assert f"{[] |> len}" == "0" assert (a=1, b=2) == _namedtuple_of(a=1, b=2) == (1, 2) # type: ignore - - loop = asyncio.new_event_loop() - - async def ayield(x) = x - :async def arange(n): - for i in range(n): - yield :await ayield(i) - async def afor_test(): - # match syntax 1 - got = [] - async for int(i) in arange(5): - got.append(i) - assert got == range(5) |> list - - # match syntax 2 - got = [] - async match for int(i) in arange(5): - got.append(i) - assert got == range(5) |> list - - # match syntax 3 - got = [] - match async for int(i) in arange(5): - got.append(i) - assert got == range(5) |> list - - if aclosing is not None: - # non-match - got = [] - async with for i in aclosing(arange(5)): - got.append(i) - assert got == range(5) |> list - - # match syntax 1 - got = [] - async with for int(i) in aclosing(arange(5)): - got.append(i) - assert got == range(5) |> list - - # match syntax 2 - got = [] - async match with for int(i) in aclosing(arange(5)): - got.append(i) - assert got == range(5) |> list - - # match syntax 3 - got = [] - match async with for int(i) in aclosing(arange(5)): - got.append(i) - assert got == range(5) |> list - - return True - loop.run_until_complete(afor_test()) - - async yield def toa(it): - for x in it: - yield x - match yield async def arange_(int(n)): - for x in range(n): - yield x - async def aconsume(ait): - async for _ in ait: - pass - l: typing.List[int] = [] - async def aiter_test(): - range(10) |> toa |> fmap$(l.append) |> aconsume |> await - arange_(10) |> fmap$(l.append) |> aconsume |> await - loop.run_until_complete(aiter_test()) - assert l == list(range(10)) + list(range(10)) - - async def arec(x) = await arec(x-1) if x else x - async def outer_func(): - funcs = [] - for x in range(5): - funcs.append(async copyclosure def -> x) - return funcs - async def await_all(xs) = [await x for x in xs] - async def atest(): - assert ( - 10 - |> arec - |> await - |> (.+10) - |> arec - |> await - ) == 0 - assert ( - outer_func() - |> await - |> map$(call) - |> await_all - |> await - ) == range(5) |> list - loop.run_until_complete(atest()) - - loop.close() - return True diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 5efc90641..196ad7434 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -348,12 +348,14 @@ async def async_map_test() = setup(target="3.2") assert parse(gen_func_def, mode="lenient") not in gen_func_def_outs - setup(target="3.5") + setup(target="3.4") assert_raises(-> parse("async def f(): yield 1"), CoconutTargetError) + setup(target="3.5") + assert parse("async def f(): yield 1") + setup(target="3.6") assert parse("def f(*, x=None) = x") - assert parse("async def f(): yield 1") setup(target="3.8") assert parse("(a := b)") From c0b3fa8a5776eb3cbcc71579fc94d5b2b5e8aac2 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 19:15:21 -0700 Subject: [PATCH 47/57] Fix reqs, tests --- coconut/constants.py | 4 ++-- coconut/tests/main_test.py | 6 ++++-- coconut/tests/src/cocotest/target_35/py35_test.coco | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index dcba7ac79..d6ed72cf7 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -868,7 +868,7 @@ def get_bool_env_var(env_var, default=False): ("aenum", "py<34"), ("dataclasses", "py==36"), ("typing", "py<35"), - ("async_generator", "py<37"), + ("async_generator", "py3;py<37"), ), "dev": ( ("pre-commit", "py3"), @@ -921,7 +921,7 @@ def get_bool_env_var(env_var, default=False): ("pygments", "py>=39"): (2, 15), ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), - ("async_generator", "py<37"): (1, 10), + ("async_generator", "py3;py<37"): (1, 10), # pinned reqs: (must be added to pinned_reqs below) diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index 75796bd64..a91843cfb 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -795,10 +795,12 @@ def test_kernel_installation(self): assert kernel in stdout if not WINDOWS and not PYPY: - def test_exit_jupyter(self): + def test_jupyter_console(self): p = spawn_cmd("coconut --jupyter console") p.expect("In", timeout=120) - p.sendline("exit()") + p.sendline("%load_ext coconut") + p.expect("In", timeout=120) + p.sendline("`exit`") p.expect("Shutting down kernel|shutting down") if p.isalive(): p.terminate() diff --git a/coconut/tests/src/cocotest/target_35/py35_test.coco b/coconut/tests/src/cocotest/target_35/py35_test.coco index 877e2f340..ac472336b 100644 --- a/coconut/tests/src/cocotest/target_35/py35_test.coco +++ b/coconut/tests/src/cocotest/target_35/py35_test.coco @@ -44,25 +44,25 @@ def py35_test() -> bool: got.append(i) assert got == range(5) |> list - # non-match + # with for non-match got = [] async with for i in aclosing(arange(5)): got.append(i) assert got == range(5) |> list - # match syntax 1 + # with for match syntax 1 got = [] async with for int(i) in aclosing(arange(5)): got.append(i) assert got == range(5) |> list - # match syntax 2 + # with for match syntax 2 got = [] async match with for int(i) in aclosing(arange(5)): got.append(i) assert got == range(5) |> list - # match syntax 3 + # with for match syntax 3 got = [] match async with for int(i) in aclosing(arange(5)): got.append(i) From 215b7ab01c8540f97ddd1c978bf6bb8811b612cb Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 19:31:58 -0700 Subject: [PATCH 48/57] Fix readthedocs --- .readthedocs.yaml | 35 +++++++++++++++++++++++++++++++++++ .readthedocs.yml | 10 ---------- 2 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 .readthedocs.yaml delete mode 100644 .readthedocs.yml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..56e6e605a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +formats: all + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +# - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - method: pip + path: . + extra_requirements: + - docs + system_packages: true diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index fffb2f7ce..000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,10 +0,0 @@ -python: - version: 3 - pip_install: true - extra_requirements: - - docs - -formats: - - htmlzip - - pdf - - epub From 8a5df760aa68819d6f20bd8495fcaba08e789db3 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 20:10:43 -0700 Subject: [PATCH 49/57] Further fix reqs, tests --- CONTRIBUTING.md | 7 ++++-- coconut/constants.py | 4 ++-- coconut/root.py | 2 +- .../src/cocotest/target_35/py35_test.coco | 13 ----------- .../src/cocotest/target_36/py36_test.coco | 23 +++++++++++++++++++ coconut/tests/src/extras.coco | 1 + 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7adce34b5..3ff5b942f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -161,8 +161,11 @@ After you've tested your changes locally, you'll want to add more permanent test 6. Check [Codebeat](https://codebeat.co/a/evhub/projects) and [LGTM](https://lgtm.com/dashboard) for `coconut` and `compiled-cocotest` 7. Make sure [`coconut-develop`](https://pypi.python.org/pypi/coconut-develop) package looks good 8. Run `make docs` and ensure local documentation looks good - 9. Make sure [develop documentation](http://coconut.readthedocs.io/en/develop/) looks good - 10. Make sure [Github Actions](https://github.com/evhub/coconut/actions) and [AppVeyor](https://ci.appveyor.com/project/evhub/coconut) are passing + 9. Make sure all of the following are passing: + 1. [Github Actions](https://github.com/evhub/coconut/actions) + 2. [AppVeyor](https://ci.appveyor.com/project/evhub/coconut) + 3. [readthedocs](https://readthedocs.org/projects/coconut/builds/) + 10. Make sure [develop documentation](http://coconut.readthedocs.io/en/develop/) looks good 11. Turn off `develop` in `root.py` 12. Set `root.py` to new version number 13. If major release, set `root.py` to new version name diff --git a/coconut/constants.py b/coconut/constants.py index d6ed72cf7..14b053b12 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -868,7 +868,7 @@ def get_bool_env_var(env_var, default=False): ("aenum", "py<34"), ("dataclasses", "py==36"), ("typing", "py<35"), - ("async_generator", "py3;py<37"), + ("async_generator", "py3"), ), "dev": ( ("pre-commit", "py3"), @@ -921,7 +921,7 @@ def get_bool_env_var(env_var, default=False): ("pygments", "py>=39"): (2, 15), ("xonsh", "py38"): (0, 14), ("pytest", "py36"): (7,), - ("async_generator", "py3;py<37"): (1, 10), + ("async_generator", "py3"): (1, 10), # pinned reqs: (must be added to pinned_reqs below) diff --git a/coconut/root.py b/coconut/root.py index c9937e948..422630f74 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 11 +DEVELOP = 12 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/target_35/py35_test.coco b/coconut/tests/src/cocotest/target_35/py35_test.coco index ac472336b..07c63dd26 100644 --- a/coconut/tests/src/cocotest/target_35/py35_test.coco +++ b/coconut/tests/src/cocotest/target_35/py35_test.coco @@ -88,12 +88,6 @@ def py35_test() -> bool: assert l == list(range(10)) + list(range(10)) async def arec(x) = await arec(x-1) if x else x - async def outer_func(): - funcs = [] - for x in range(5): - funcs.append(async copyclosure def -> x) - return funcs - async def await_all(xs) = [await x for x in xs] async def atest(): assert ( 10 @@ -103,13 +97,6 @@ def py35_test() -> bool: |> arec |> await ) == 0 - assert ( - outer_func() - |> await - |> map$(call) - |> await_all - |> await - ) == range(5) |> list loop.run_until_complete(atest()) loop.close() diff --git a/coconut/tests/src/cocotest/target_36/py36_test.coco b/coconut/tests/src/cocotest/target_36/py36_test.coco index f90b3254f..c7645db71 100644 --- a/coconut/tests/src/cocotest/target_36/py36_test.coco +++ b/coconut/tests/src/cocotest/target_36/py36_test.coco @@ -1,5 +1,28 @@ +import asyncio + + def py36_test() -> bool: """Performs Python-3.6-specific tests.""" assert f"{[] |> len}" == "0" assert (a=1, b=2) == _namedtuple_of(a=1, b=2) == (1, 2) # type: ignore + + loop = asyncio.new_event_loop() + + async def outer_func(): + funcs = [] + for x in range(5): + funcs.append(async copyclosure def -> x) + return funcs + async def await_all(xs) = [await x for x in xs] + async def atest(): + assert ( + outer_func() + |> await + |> map$(call) + |> await_all + |> await + ) == range(5) |> list + loop.run_until_complete(atest()) + + loop.close() return True diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index 196ad7434..d0c41ce23 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -356,6 +356,7 @@ async def async_map_test() = setup(target="3.6") assert parse("def f(*, x=None) = x") + assert "@" not in parse("async def f(x): yield x") setup(target="3.8") assert parse("(a := b)") From 75cffe29fb0f7ec584cb8b73ed3af9a8714716d6 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 21:02:59 -0700 Subject: [PATCH 50/57] Fix constants test --- coconut/tests/constants_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/tests/constants_test.py b/coconut/tests/constants_test.py index bb2d561c5..7c5186781 100644 --- a/coconut/tests/constants_test.py +++ b/coconut/tests/constants_test.py @@ -98,8 +98,8 @@ def test_imports(self): or PYPY and new_imp.startswith("tkinter") # don't test trollius on PyPy or PYPY and old_imp == "trollius" - # don't test typing_extensions on Python 2 - or PY2 and old_imp.startswith("typing_extensions") + # don't test typing_extensions, async_generator on Python 2 + or PY2 and old_imp.startswith(("typing_extensions", "async_generator")) ): pass elif sys.version_info >= ver_cutoff: From aaf692549db3e5ecc818d4d12209107d3263af5e Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 23:21:33 -0700 Subject: [PATCH 51/57] Fix types --- _coconut/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi index c4bf9406b..9788e9083 100644 --- a/_coconut/__init__.pyi +++ b/_coconut/__init__.pyi @@ -52,7 +52,7 @@ else: import trollius as _asyncio # type: ignore if sys.version_info >= (3, 5): - import async_generator as _async_generator + import async_generator as _async_generator # type: ignore try: import numpy as _numpy # type: ignore From 9d7d11ca7629679ccb6fc774a9aa454742fbab96 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sat, 27 May 2023 23:44:32 -0700 Subject: [PATCH 52/57] Improve async with for syntax --- coconut/compiler/grammar.py | 4 ++-- coconut/tests/src/cocotest/target_35/py35_test.coco | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 1f772e451..1a383422d 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -2132,8 +2132,8 @@ class Grammar(object): | labeled_group( (any_len_perm( keyword("match"), - required=(keyword("async"),) - ) + keyword("with") + keyword("for")).suppress() + required=(keyword("async"), keyword("with")), + ) + keyword("for")).suppress() + many_match + keyword("in").suppress() - test - suite_with_else_tokens, diff --git a/coconut/tests/src/cocotest/target_35/py35_test.coco b/coconut/tests/src/cocotest/target_35/py35_test.coco index 07c63dd26..ce71cd426 100644 --- a/coconut/tests/src/cocotest/target_35/py35_test.coco +++ b/coconut/tests/src/cocotest/target_35/py35_test.coco @@ -68,6 +68,12 @@ def py35_test() -> bool: got.append(i) assert got == range(5) |> list + # with for match syntax 4 + got = [] + async with match for int(i) in aclosing(arange(5)): + got.append(i) + assert got == range(5) |> list + return True loop.run_until_complete(afor_test()) From 5dce8273990707132e4afb6748ca3eb1f8877ad9 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 28 May 2023 14:55:23 -0700 Subject: [PATCH 53/57] Improve async generator support --- coconut/command/util.py | 11 ++--- coconut/compiler/compiler.py | 20 ++++++--- coconut/compiler/grammar.py | 6 ++- coconut/compiler/header.py | 5 ++- coconut/compiler/util.py | 35 ++++++++++++++++ coconut/constants.py | 7 +++- coconut/requirements.py | 9 ++-- coconut/root.py | 2 +- .../src/cocotest/target_35/py35_test.coco | 14 +------ coconut/tests/src/extras.coco | 2 + coconut/util.py | 42 +++---------------- 11 files changed, 85 insertions(+), 68 deletions(-) diff --git a/coconut/command/util.py b/coconut/command/util.py index 7f18d0d36..11ebce971 100644 --- a/coconut/command/util.py +++ b/coconut/command/util.py @@ -47,6 +47,7 @@ get_encoding, get_clock_time, memoize, + assert_remove_prefix, ) from coconut.constants import ( WINDOWS, @@ -173,7 +174,7 @@ def showpath(path): else: path = os.path.relpath(path) if path.startswith(os.curdir + os.sep): - path = path[len(os.curdir + os.sep):] + path = assert_remove_prefix(path, os.curdir + os.sep) return path @@ -423,13 +424,13 @@ def subpath(path, base_path): def invert_mypy_arg(arg): """Convert --arg into --no-arg or equivalent.""" if arg.startswith("--no-"): - return "--" + arg[len("--no-"):] + return "--" + assert_remove_prefix(arg, "--no-") elif arg.startswith("--allow-"): - return "--disallow-" + arg[len("--allow-"):] + return "--disallow-" + assert_remove_prefix(arg, "--allow-") elif arg.startswith("--disallow-"): - return "--allow-" + arg[len("--disallow-"):] + return "--allow-" + assert_remove_prefix(arg, "--disallow-") elif arg.startswith("--"): - return "--no-" + arg[len("--"):] + return "--no-" + assert_remove_prefix(arg, "--") else: return None diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 2426e416c..faa646acc 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -98,6 +98,7 @@ get_target_info, get_clock_time, get_name, + assert_remove_prefix, ) from coconut.exceptions import ( CoconutException, @@ -1855,9 +1856,18 @@ def transform_returns(self, original, loc, raw_lines, tre_return_grammar=None, i # attempt tco/tre/async universalization if disabled_until_level is None: + # disallow yield from in async generators + if is_async and is_gen and self.yield_from_regex.search(base): + raise self.make_err( + CoconutSyntaxError, + "yield from not allowed in async generators", + original, + loc, + ) + # handle generator/async returns if not normal_func and self.return_regex.match(base): - to_return = base[len("return"):].strip() + to_return = assert_remove_prefix(base, "return").strip() if to_return: to_return = "(" + to_return + ")" # only use trollius Return when trollius is imported @@ -1879,7 +1889,7 @@ def transform_returns(self, original, loc, raw_lines, tre_return_grammar=None, i # handle async generator yields if is_async and is_gen and self.target_info < (3, 6): if self.yield_regex.match(base): - to_yield = base[len("yield"):].strip() + to_yield = assert_remove_prefix(base, "yield").strip() line = indent + "await _coconut.async_generator.yield_(" + to_yield + ")" + comment + dedent elif self.yield_regex.search(base): raise self.make_err( @@ -1931,10 +1941,10 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, done = False while not done: if def_stmt.startswith("addpattern "): - def_stmt = def_stmt[len("addpattern "):] + def_stmt = assert_remove_prefix(def_stmt, "addpattern ") addpattern = True elif def_stmt.startswith("copyclosure "): - def_stmt = def_stmt[len("copyclosure "):] + def_stmt = assert_remove_prefix(def_stmt, "copyclosure ") copyclosure = True elif def_stmt.startswith("def"): done = True @@ -2274,7 +2284,7 @@ def deferred_code_proc(self, inputstring, add_code_at_start=False, ignore_names= # look for functions if line.startswith(funcwrapper): - func_id = int(line[len(funcwrapper):]) + func_id = int(assert_remove_prefix(line, funcwrapper)) original, loc, decorators, funcdef, is_async, in_method, is_stmt_lambda = self.get_ref("func", func_id) # process inner code diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 1a383422d..4cac82243 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -55,6 +55,7 @@ memoize, get_clock_time, keydefaultdict, + assert_remove_prefix, ) from coconut.exceptions import ( CoconutInternalException, @@ -597,10 +598,10 @@ def typedef_op_item_handle(loc, tokens): op_name, = tokens op_name = op_name.strip("_") if op_name.startswith("coconut"): - op_name = op_name[len("coconut"):] + op_name = assert_remove_prefix(op_name, "coconut") op_name = op_name.lstrip("._") if op_name.startswith("operator."): - op_name = op_name[len("operator."):] + op_name = assert_remove_prefix(op_name, "operator.") proto = op_func_protocols.get(op_name) if proto is None: @@ -2374,6 +2375,7 @@ class Grammar(object): def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") + yield_from_regex = compile_regex(r"\byield\s+from\b") tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") return_regex = compile_regex(r"\breturn\b") diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 7b6436314..61eb1897e 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -42,6 +42,7 @@ from coconut.util import ( univ_open, get_target_info, + assert_remove_prefix, ) from coconut.compiler.util import ( split_comment, @@ -60,7 +61,7 @@ def gethash(compiled): if len(lines) < 3 or not lines[2].startswith(hash_prefix): return None else: - return lines[2][len(hash_prefix):] + return assert_remove_prefix(lines[2], hash_prefix) def minify_header(compiled): @@ -748,7 +749,7 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap): header += "_coconut_header_info = " + header_info + "\n" if which.startswith("package"): - levels_up = int(which[len("package:"):]) + levels_up = int(assert_remove_prefix(which, "package:")) coconut_file_dir = "_coconut_os.path.dirname(_coconut_os.path.abspath(__file__))" for _ in range(levels_up): coconut_file_dir = "_coconut_os.path.dirname(" + coconut_file_dir + ")" diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index e6de4537f..dc6ebe84b 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -95,6 +95,7 @@ comment_chars, non_syntactic_newline, allow_explicit_keyword_vars, + reserved_prefix, ) from coconut.exceptions import ( CoconutException, @@ -1363,3 +1364,37 @@ def add_int_and_strs(int_part=0, str_parts=(), parens=False): if parens: out = "(" + out + ")" return out + + +# ----------------------------------------------------------------------------------------------------------------------- +# PYTEST: +# ----------------------------------------------------------------------------------------------------------------------- + + +class FixPytestNames(ast.NodeTransformer): + """Renames invalid names added by pytest assert rewriting.""" + + def fix_name(self, name): + """Make the given pytest name a valid but non-colliding identifier.""" + return name.replace("@", reserved_prefix + "_pytest_") + + def visit_Name(self, node): + """Special method to visit ast.Names.""" + node.id = self.fix_name(node.id) + return node + + def visit_alias(self, node): + """Special method to visit ast.aliases.""" + node.asname = self.fix_name(node.asname) + return node + + +def pytest_rewrite_asserts(code, module_name=reserved_prefix + "_pytest_module"): + """Uses pytest to rewrite the assert statements in the given code.""" + from _pytest.assertion.rewrite import rewrite_asserts # hidden since it's not always available + + module_name = module_name.encode("utf-8") + tree = ast.parse(code) + rewrite_asserts(tree, module_name) + fixed_tree = ast.fix_missing_locations(FixPytestNames().visit(tree)) + return ast.unparse(fixed_tree) diff --git a/coconut/constants.py b/coconut/constants.py index 14b053b12..adbfa2e9f 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -431,6 +431,11 @@ def get_bool_env_var(env_var, default=False): "asyncio": ("trollius", (3, 4)), "enum": ("aenum", (3, 4)), "contextlib.asynccontextmanager": ("async_generator./asynccontextmanager", (3, 7)), + "contextlib.aclosing": ("async_generator./aclosing", (3, 10)), + "inspect.isasyncgen": ("async_generator./isasyncgen", (3, 6)), + "inspect.isasyncgenfunction": ("async_generator./isasyncgenfunction", (3, 6)), + "sys.get_asyncgen_hooks": ("async_generator./get_asyncgen_hooks", (3, 6)), + "sys.set_asyncgen_hooks": ("async_generator./set_asyncgen_hooks", (3, 6)), # # typing_extensions (not needed since _coconut.typing has them # # and mypy is happy to accept that they always live in typing) @@ -536,7 +541,7 @@ def get_bool_env_var(env_var, default=False): '__file__', '__annotations__', '__debug__', - # don't include builtins that aren't always made available by Coconut: + # # don't include builtins that aren't always made available by Coconut: # 'BlockingIOError', 'ChildProcessError', 'ConnectionError', # 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError', # 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', diff --git a/coconut/requirements.py b/coconut/requirements.py index 93bf6b8e9..6ead04b53 100644 --- a/coconut/requirements.py +++ b/coconut/requirements.py @@ -41,6 +41,7 @@ ver_str_to_tuple, ver_tuple_to_str, get_next_version, + assert_remove_prefix, ) # ----------------------------------------------------------------------------------------------------------------------- @@ -74,7 +75,7 @@ def process_mark(mark): """Get the check string and whether it currently applies for the given mark.""" assert not mark.startswith("py2"), "confusing mark; should be changed: " + mark if mark.startswith("py=="): - ver = mark[len("py=="):] + ver = assert_remove_prefix(mark, "py==") if len(ver) == 1: ver_tuple = (int(ver),) else: @@ -95,14 +96,14 @@ def process_mark(mark): check_str = "python_version<'3'" holds_now = PY2 elif mark.startswith("py<"): - full_ver = mark[len("py<"):] + full_ver = assert_remove_prefix(mark, "py<") main_ver, sub_ver = full_ver[0], full_ver[1:] check_str = "python_version<'{main}.{sub}'".format(main=main_ver, sub=sub_ver) holds_now = sys.version_info < (int(main_ver), int(sub_ver)) elif mark.startswith("py") or mark.startswith("py>="): - full_ver = mark[len("py"):] + full_ver = assert_remove_prefix(mark, "py") if full_ver.startswith(">="): - full_ver = full_ver[len(">="):] + full_ver = assert_remove_prefix(full_ver, ">=") main_ver, sub_ver = full_ver[0], full_ver[1:] check_str = "python_version>='{main}.{sub}'".format(main=main_ver, sub=sub_ver) holds_now = sys.version_info >= (int(main_ver), int(sub_ver)) diff --git a/coconut/root.py b/coconut/root.py index 422630f74..41ef276e0 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 12 +DEVELOP = 13 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/target_35/py35_test.coco b/coconut/tests/src/cocotest/target_35/py35_test.coco index ce71cd426..baca2a698 100644 --- a/coconut/tests/src/cocotest/target_35/py35_test.coco +++ b/coconut/tests/src/cocotest/target_35/py35_test.coco @@ -1,15 +1,5 @@ -import sys, asyncio, typing - -if sys.version_info >= (3, 10): - from contextlib import aclosing -else: - from contextlib import asynccontextmanager - @asynccontextmanager - async def aclosing(thing): - try: - yield thing - finally: - await thing.aclose() +import asyncio, typing +from contextlib import aclosing def py35_test() -> bool: diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index d0c41ce23..bb333ac69 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -353,6 +353,8 @@ async def async_map_test() = setup(target="3.5") assert parse("async def f(): yield 1") + assert_raises(-> parse("""async def agen(): + yield from range(5)"""), CoconutSyntaxError, err_has="async generator") setup(target="3.6") assert parse("def f(*, x=None) = x") diff --git a/coconut/util.py b/coconut/util.py index 98489f5b4..1b1b21a62 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -25,7 +25,6 @@ import json import traceback import time -import ast from zlib import crc32 from warnings import warn from types import MethodType @@ -47,7 +46,6 @@ icoconut_custom_kernel_install_loc, icoconut_custom_kernel_file_loc, WINDOWS, - reserved_prefix, non_syntactic_newline, ) @@ -242,6 +240,12 @@ def __missing__(self, key): return self[key] +def assert_remove_prefix(inputstr, prefix): + """Remove prefix asserting that inputstr starts with it.""" + assert inputstr.startswith(prefix), inputstr + return inputstr[len(prefix):] + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # ----------------------------------------------------------------------------------------------------------------------- @@ -360,37 +364,3 @@ def make_custom_kernel(executable=None): raw_json = json.dumps(kernel_dict, indent=1) kernel_file.write(raw_json.encode(encoding=default_encoding)) return icoconut_custom_kernel_dir - - -# ----------------------------------------------------------------------------------------------------------------------- -# PYTEST: -# ----------------------------------------------------------------------------------------------------------------------- - - -class FixPytestNames(ast.NodeTransformer): - """Renames invalid names added by pytest assert rewriting.""" - - def fix_name(self, name): - """Make the given pytest name a valid but non-colliding identifier.""" - return name.replace("@", reserved_prefix + "_pytest_") - - def visit_Name(self, node): - """Special method to visit ast.Names.""" - node.id = self.fix_name(node.id) - return node - - def visit_alias(self, node): - """Special method to visit ast.aliases.""" - node.asname = self.fix_name(node.asname) - return node - - -def pytest_rewrite_asserts(code, module_name=reserved_prefix + "_pytest_module"): - """Uses pytest to rewrite the assert statements in the given code.""" - from _pytest.assertion.rewrite import rewrite_asserts # hidden since it's not always available - - module_name = module_name.encode("utf-8") - tree = ast.parse(code) - rewrite_asserts(tree, module_name) - fixed_tree = ast.fix_missing_locations(FixPytestNames().visit(tree)) - return ast.unparse(fixed_tree) From c236bd082aae458d4b43c86bdc86bd8699638000 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 28 May 2023 18:11:01 -0700 Subject: [PATCH 54/57] Add type param constraint support Refs #757. --- DOCS.md | 6 +- __coconut__/__init__.pyi | 60 ++++--- _coconut/__init__.pyi | 154 +++++++----------- coconut/compiler/compiler.py | 68 +++++--- coconut/compiler/grammar.py | 4 +- coconut/compiler/header.py | 2 +- coconut/compiler/matching.py | 6 +- coconut/compiler/util.py | 52 +++--- coconut/constants.py | 104 ++++++------ coconut/root.py | 2 +- .../tests/src/cocotest/agnostic/suite.coco | 1 + coconut/tests/src/cocotest/agnostic/util.coco | 12 +- coconut/tests/src/extras.coco | 6 + 13 files changed, 244 insertions(+), 233 deletions(-) diff --git a/DOCS.md b/DOCS.md index e94109ce5..5eabc019c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -427,7 +427,7 @@ To distribute your code with checkable type annotations, you'll need to include To explicitly annotate your code with types to be checked, Coconut supports: * [Python 3 function type annotations](https://www.python.org/dev/peps/pep-0484/), * [Python 3.6 variable type annotations](https://www.python.org/dev/peps/pep-0526/), -* [PEP 695 type parameter syntax](#type-parameter-syntax) for easily adding type parameters to classes, functions, [`data` types](#data), and type aliases, +* [Python 3.12 type parameter syntax](#type-parameter-syntax) for easily adding type parameters to classes, functions, [`data` types](#data), and type aliases, * Coconut's own [enhanced type annotation syntax](#enhanced-type-annotation), and * Coconut's [protocol intersection operator](#protocol-intersection). @@ -2579,7 +2579,7 @@ _Can't be done without a long series of checks in place of the destructuring ass ### Type Parameter Syntax -Coconut fully supports [PEP 695](https://peps.python.org/pep-0695/) type parameter syntax (with the caveat that all type variables are invariant rather than inferred). +Coconut fully supports [Python 3.12 PEP 695](https://peps.python.org/pep-0695/) type parameter syntax on all Python versions. That includes type parameters for classes, [`data` types](#data), and [all types of function definition](#function-definition). For different types of function definition, the type parameters always come in brackets right after the function name. Coconut's [enhanced type annotation syntax](#enhanced-type-annotation) is supported for all type parameter bounds. @@ -2587,6 +2587,8 @@ _Warning: until `mypy` adds support for `infer_variance=True` in `TypeVar`, `Typ Additionally, Coconut supports the alternative bounds syntax of `type NewType[T <: bound] = ...` rather than `type NewType[T: bound] = ...`, to make it more clear that it is an upper bound rather than a type. In `--strict` mode, `<:` is required over `:` for all type parameter bounds. _DEPRECATED: `<=` can also be used as an alternative to `<:`._ +Note that the `<:` syntax should only be used for [type bounds](https://peps.python.org/pep-0695/#upper-bound-specification), not [type constraints](https://peps.python.org/pep-0695/#constrained-type-specification)—for type constraints, Coconut style prefers the vanilla Python `:` syntax, which helps to disambiguate between the two cases, as they are functionally different but otherwise hard to tell apart at a glance. This is enforced in `--strict` mode. + _Note that, by default, all type declarations are wrapped in strings to enable forward references and improve runtime performance. If you don't want that—e.g. because you want to use type annotations at runtime—simply pass the `--no-wrap-types` flag._ ##### PEP 695 Docs diff --git a/__coconut__/__init__.pyi b/__coconut__/__init__.pyi index 75c660612..b85237ebc 100644 --- a/__coconut__/__init__.pyi +++ b/__coconut__/__init__.pyi @@ -8,13 +8,13 @@ License: Apache 2.0 Description: MyPy stub file for __coconut__.py. """ -import sys -import typing as _t - # ----------------------------------------------------------------------------------------------------------------------- # TYPE VARS: # ----------------------------------------------------------------------------------------------------------------------- +import sys +import typing as _t + _Callable = _t.Callable[..., _t.Any] _Iterable = _t.Iterable[_t.Any] _Tuple = _t.Tuple[_t.Any, ...] @@ -55,21 +55,14 @@ _P = _t.ParamSpec("_P") class _SupportsIndex(_t.Protocol): def __index__(self) -> int: ... - # ----------------------------------------------------------------------------------------------------------------------- # IMPORTS: # ----------------------------------------------------------------------------------------------------------------------- -if sys.version_info >= (3, 11): - from typing import dataclass_transform as _dataclass_transform +if sys.version_info >= (3,): + import builtins as _builtins else: - try: - from typing_extensions import dataclass_transform as _dataclass_transform - except ImportError: - dataclass_transform = ... - -import _coconut as __coconut # we mock _coconut as a package since mypy doesn't handle namespace classes very well -_coconut = __coconut + import __builtin__ as _builtins if sys.version_info >= (3, 2): from functools import lru_cache as _lru_cache @@ -81,13 +74,24 @@ if sys.version_info >= (3, 7): from dataclasses import dataclass as _dataclass else: @_dataclass_transform() - def _dataclass(cls: t_coype[_T], **kwargs: _t.Any) -> type[_T]: ... + def _dataclass(cls: type[_T], **kwargs: _t.Any) -> type[_T]: ... + +if sys.version_info >= (3, 11): + from typing import dataclass_transform as _dataclass_transform +else: + try: + from typing_extensions import dataclass_transform as _dataclass_transform + except ImportError: + dataclass_transform = ... try: from typing_extensions import deprecated as _deprecated # type: ignore except ImportError: def _deprecated(message: _t.Text) -> _t.Callable[[_T], _T]: ... # type: ignore +import _coconut as __coconut # we mock _coconut as a package since mypy doesn't handle namespace classes very well +_coconut = __coconut + # ----------------------------------------------------------------------------------------------------------------------- # STUB: @@ -153,18 +157,18 @@ py_repr = repr py_breakpoint = breakpoint # all py_ functions, but not py_ types, go here -chr = chr -hex = hex -input = input -map = map -oct = oct -open = open -print = print -range = range -zip = zip -filter = filter -reversed = reversed -enumerate = enumerate +chr = _builtins.chr +hex = _builtins.hex +input = _builtins.input +map = _builtins.map +oct = _builtins.oct +open = _builtins.open +print = _builtins.print +range = _builtins.range +zip = _builtins.zip +filter = _builtins.filter +reversed = _builtins.reversed +enumerate = _builtins.enumerate _coconut_py_str = py_str @@ -435,6 +439,9 @@ def recursive_iterator(func: _T_iter_func) -> _T_iter_func: return func +# if sys.version_info >= (3, 12): +# from typing import override +# else: try: from typing_extensions import override as _override # type: ignore override = _override @@ -442,6 +449,7 @@ except ImportError: def override(func: _Tfunc) -> _Tfunc: return func + def _coconut_call_set_names(cls: object) -> None: ... diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi index 9788e9083..38433b7ac 100644 --- a/_coconut/__init__.pyi +++ b/_coconut/__init__.pyi @@ -31,6 +31,11 @@ import multiprocessing as _multiprocessing import pickle as _pickle from multiprocessing import dummy as _multiprocessing_dummy +if sys.version_info >= (3,): + import builtins as _builtins +else: + import __builtin__ as _builtins + if sys.version_info >= (3,): import copyreg as _copyreg else: @@ -68,41 +73,6 @@ else: # ----------------------------------------------------------------------------------------------------------------------- typing = _t - -from typing_extensions import TypeVar -typing.TypeVar = TypeVar # type: ignore - -if sys.version_info < (3, 8): - try: - from typing_extensions import Protocol - except ImportError: - Protocol = ... # type: ignore - typing.Protocol = Protocol # type: ignore - -if sys.version_info < (3, 10): - try: - from typing_extensions import TypeAlias, ParamSpec, Concatenate - except ImportError: - TypeAlias = ... # type: ignore - ParamSpec = ... # type: ignore - Concatenate = ... # type: ignore - typing.TypeAlias = TypeAlias # type: ignore - typing.ParamSpec = ParamSpec # type: ignore - typing.Concatenate = Concatenate # type: ignore - -if sys.version_info < (3, 11): - try: - from typing_extensions import TypeVarTuple, Unpack - except ImportError: - TypeVarTuple = ... # type: ignore - Unpack = ... # type: ignore - typing.TypeVarTuple = TypeVarTuple # type: ignore - typing.Unpack = Unpack # type: ignore - -# ----------------------------------------------------------------------------------------------------------------------- -# STUB: -# ----------------------------------------------------------------------------------------------------------------------- - collections = _collections copy = _copy functools = _functools @@ -141,62 +111,62 @@ tee_type: _t.Any = ... reiterables: _t.Any = ... fmappables: _t.Any = ... -Ellipsis = Ellipsis -NotImplemented = NotImplemented -NotImplementedError = NotImplementedError -Exception = Exception -AttributeError = AttributeError -ImportError = ImportError -IndexError = IndexError -KeyError = KeyError -NameError = NameError -TypeError = TypeError -ValueError = ValueError -StopIteration = StopIteration -RuntimeError = RuntimeError -callable = callable -classmethod = classmethod -complex = complex -all = all -any = any -bool = bool -bytes = bytes -dict = dict -enumerate = enumerate -filter = filter -float = float -frozenset = frozenset -getattr = getattr -hasattr = hasattr -hash = hash -id = id -int = int -isinstance = isinstance -issubclass = issubclass -iter = iter +Ellipsis = _builtins.Ellipsis +NotImplemented = _builtins.NotImplemented +NotImplementedError = _builtins.NotImplementedError +Exception = _builtins.Exception +AttributeError = _builtins.AttributeError +ImportError = _builtins.ImportError +IndexError = _builtins.IndexError +KeyError = _builtins.KeyError +NameError = _builtins.NameError +TypeError = _builtins.TypeError +ValueError = _builtins.ValueError +StopIteration = _builtins.StopIteration +RuntimeError = _builtins.RuntimeError +callable = _builtins.callable +classmethod = _builtins.classmethod +complex = _builtins.complex +all = _builtins.all +any = _builtins.any +bool = _builtins.bool +bytes = _builtins.bytes +dict = _builtins.dict +enumerate = _builtins.enumerate +filter = _builtins.filter +float = _builtins.float +frozenset = _builtins.frozenset +getattr = _builtins.getattr +hasattr = _builtins.hasattr +hash = _builtins.hash +id = _builtins.id +int = _builtins.int +isinstance = _builtins.isinstance +issubclass = _builtins.issubclass +iter = _builtins.iter len: _t.Callable[..., int] = ... # pattern-matching needs an untyped _coconut.len to avoid type errors -list = list -locals = locals -globals = globals -map = map -min = min -max = max -next = next -object = object -print = print -property = property -range = range -reversed = reversed -set = set -setattr = setattr -slice = slice -str = str -sum = sum -super = super -tuple = tuple -type = type -zip = zip -vars = vars -repr = repr +list = _builtins.list +locals = _builtins.locals +globals = _builtins.globals +map = _builtins.map +min = _builtins.min +max = _builtins.max +next = _builtins.next +object = _builtins.object +print = _builtins.print +property = _builtins.property +range = _builtins.range +reversed = _builtins.reversed +set = _builtins.set +setattr = _builtins.setattr +slice = _builtins.slice +str = _builtins.str +sum = _builtins.sum +super = _builtins.super +tuple = _builtins.tuple +type = _builtins.type +zip = _builtins.zip +vars = _builtins.vars +repr = _builtins.repr if sys.version_info >= (3,): - bytearray = bytearray + bytearray = _builtins.bytearray diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index faa646acc..bc5a800b8 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -3609,32 +3609,22 @@ def funcname_typeparams_handle(self, tokens): def type_param_handle(self, original, loc, tokens): """Compile a type param into an assignment.""" - bounds = "" - kwargs = "" + args = "" + bound_op = None + bound_op_type = "" if "TypeVar" in tokens: TypeVarFunc = "TypeVar" + bound_op_type = "bound" if len(tokens) == 2: name_loc, name = tokens else: name_loc, name, bound_op, bound = tokens - if bound_op == "<=": - self.strict_err_or_warn( - "use of " + repr(bound_op) + " as a type parameter bound declaration operator is deprecated (Coconut style is to use '<:' operator)", - original, - loc, - ) - elif bound_op == ":": - self.strict_err( - "found use of " + repr(bound_op) + " as a type parameter bound declaration operator (Coconut style is to use '<:' operator)", - original, - loc, - ) - else: - self.internal_assert(bound_op == "<:", original, loc, "invalid type_param bound_op", bound_op) - bounds = ", bound=" + self.wrap_typedef(bound, for_py_typedef=False) - # uncomment this line whenever mypy adds support for infer_variance in TypeVar - # (and remove the warning about it in the DOCS) - # kwargs = ", infer_variance=True" + args = ", bound=" + self.wrap_typedef(bound, for_py_typedef=False) + elif "TypeVar constraint" in tokens: + TypeVarFunc = "TypeVar" + bound_op_type = "constraint" + name_loc, name, bound_op, constraints = tokens + args = ", " + ", ".join(self.wrap_typedef(c, for_py_typedef=False) for c in constraints) elif "TypeVarTuple" in tokens: TypeVarFunc = "TypeVarTuple" name_loc, name = tokens @@ -3644,6 +3634,27 @@ def type_param_handle(self, original, loc, tokens): else: raise CoconutInternalException("invalid type_param tokens", tokens) + kwargs = "" + if bound_op is not None: + self.internal_assert(bound_op_type in ("bound", "constraint"), original, loc, "invalid type_param bound_op", bound_op) + # # uncomment this line whenever mypy adds support for infer_variance in TypeVar + # # (and remove the warning about it in the DOCS) + # kwargs = ", infer_variance=True" + if bound_op == "<=": + self.strict_err_or_warn( + "use of " + repr(bound_op) + " as a type parameter " + bound_op_type + " declaration operator is deprecated (Coconut style is to use '<:' for bounds and ':' for constaints)", + original, + loc, + ) + else: + self.internal_assert(bound_op in (":", "<:"), original, loc, "invalid type_param bound_op", bound_op) + if bound_op_type == "bound" and bound_op != "<:" or bound_op_type == "constraint" and bound_op != ":": + self.strict_err( + "found use of " + repr(bound_op) + " as a type parameter " + bound_op_type + " declaration operator (Coconut style is to use '<:' for bounds and ':' for constaints)", + original, + loc, + ) + name_loc = int(name_loc) internal_assert(name_loc == loc if TypeVarFunc == "TypeVar" else name_loc >= loc, "invalid name location for " + TypeVarFunc, (name_loc, loc, tokens)) @@ -3661,10 +3672,10 @@ def type_param_handle(self, original, loc, tokens): typevar_info["typevar_locs"][name] = name_loc name = temp_name - return '{name} = _coconut.typing.{TypeVarFunc}("{name}"{bounds}{kwargs})\n'.format( + return '{name} = _coconut.typing.{TypeVarFunc}("{name}"{args}{kwargs})\n'.format( name=name, TypeVarFunc=TypeVarFunc, - bounds=bounds, + args=args, kwargs=kwargs, ) @@ -3707,11 +3718,14 @@ def type_alias_stmt_handle(self, tokens): paramdefs = () else: name, paramdefs, typedef = tokens - return "".join(paramdefs) + self.typed_assign_stmt_handle([ - name, - "_coconut.typing.TypeAlias", - self.wrap_typedef(typedef, for_py_typedef=False), - ]) + if self.target_info >= (3, 12): + return "type " + name + " = " + self.wrap_typedef(typedef, for_py_typedef=True) + else: + return "".join(paramdefs) + self.typed_assign_stmt_handle([ + name, + "_coconut.typing.TypeAlias", + self.wrap_typedef(typedef, for_py_typedef=False), + ]) def with_stmt_handle(self, tokens): """Process with statements.""" diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index 4cac82243..e5943da66 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -1356,8 +1356,10 @@ class Grammar(object): type_param = Forward() type_param_bound_op = lt_colon | colon | le type_var_name = stores_loc_item + setname + type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() type_param_ref = ( - (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") + (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") + | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") | (star.suppress() + type_var_name)("TypeVarTuple") | (dubstar.suppress() + type_var_name)("ParamSpec") ) diff --git a/coconut/compiler/header.py b/coconut/compiler/header.py index 61eb1897e..39ff27fca 100644 --- a/coconut/compiler/header.py +++ b/coconut/compiler/header.py @@ -793,7 +793,7 @@ def getheader(which, use_hash, target, no_tco, strict, no_wrap): if which == "sys": return header + '''from coconut.__coconut__ import * from coconut.__coconut__ import {underscore_imports} -'''.format(**format_dict) +'''.format(**format_dict) + section("Compiled Coconut") # __coconut__, code, file diff --git a/coconut/compiler/matching.py b/coconut/compiler/matching.py index 2bc2e5a8d..96765f91a 100644 --- a/coconut/compiler/matching.py +++ b/coconut/compiler/matching.py @@ -1064,7 +1064,7 @@ def match_class(self, tokens, item): match_args_var = other_cls_matcher.get_temp_var() other_cls_matcher.add_def( handle_indentation(""" -{match_args_var} = _coconut.getattr({cls_name}, '__match_args__', ()) +{match_args_var} = _coconut.getattr({cls_name}, '__match_args__', ()) {type_any} {type_ignore} if not _coconut.isinstance({match_args_var}, _coconut.tuple): raise _coconut.TypeError("{cls_name}.__match_args__ must be a tuple") if _coconut.len({match_args_var}) < {num_pos_matches}: @@ -1073,6 +1073,8 @@ def match_class(self, tokens, item): cls_name=cls_name, match_args_var=match_args_var, num_pos_matches=len(pos_matches), + type_any=self.comp.wrap_comment(" type: _coconut.typing.Any"), + type_ignore=self.comp.type_ignore_comment(), ), ) with other_cls_matcher.down_a_level(): @@ -1161,7 +1163,7 @@ def match_data_or_class(self, tokens, item): self.add_def( handle_indentation( """ -{is_data_result_var} = _coconut.getattr({cls_name}, "{is_data_var}", False) or _coconut.isinstance({cls_name}, _coconut.tuple) and _coconut.all(_coconut.getattr(_coconut_x, "{is_data_var}", False) for _coconut_x in {cls_name}){type_ignore} +{is_data_result_var} = _coconut.getattr({cls_name}, "{is_data_var}", False) or _coconut.isinstance({cls_name}, _coconut.tuple) and _coconut.all(_coconut.getattr(_coconut_x, "{is_data_var}", False) for _coconut_x in {cls_name}) {type_ignore} """, ).format( is_data_result_var=is_data_result_var, diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index dc6ebe84b..bebec4a09 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -703,24 +703,6 @@ def maybeparens(lparen, item, rparen, prefer_parens=False): return item | lparen.suppress() + item + rparen.suppress() -@memoize() -def tokenlist(item, sep, suppress=True, allow_trailing=True, at_least_two=False, require_sep=False): - """Create a list of tokens matching the item.""" - if suppress: - sep = sep.suppress() - if not require_sep: - out = item + (OneOrMore if at_least_two else ZeroOrMore)(sep + item) - if allow_trailing: - out += Optional(sep) - elif not allow_trailing: - out = item + OneOrMore(sep + item) - elif at_least_two: - out = item + OneOrMore(sep + item) + Optional(sep) - else: - out = OneOrMore(item + sep) + Optional(item) - return out - - def interleaved_tokenlist(required_item, other_item, sep, allow_trailing=False, at_least_two=False): """Create a grammar to match interleaved required_items and other_items, where required_item must show up at least once.""" @@ -751,6 +733,30 @@ def interleaved_tokenlist(required_item, other_item, sep, allow_trailing=False, return out +@memoize() +def tokenlist(item, sep, suppress=True, allow_trailing=True, at_least_two=False, require_sep=False, suppress_trailing=False): + """Create a list of tokens matching the item.""" + if suppress: + sep = sep.suppress() + if suppress_trailing: + trailing_sep = sep.suppress() + else: + trailing_sep = sep + if not require_sep: + out = item + (OneOrMore if at_least_two else ZeroOrMore)(sep + item) + if allow_trailing: + out += Optional(trailing_sep) + elif not allow_trailing: + out = item + OneOrMore(sep + item) + elif at_least_two: + out = item + OneOrMore(sep + item) + Optional(trailing_sep) + elif suppress_trailing: + out = item + OneOrMore(sep + item) + Optional(trailing_sep) | item + trailing_sep + else: + out = OneOrMore(item + sep) + Optional(item) + return out + + def add_list_spacing(tokens): """Parse action to add spacing after seps but not elsewhere.""" out = [] @@ -765,21 +771,19 @@ def add_list_spacing(tokens): add_list_spacing.ignore_one_token = True -def itemlist(item, sep, suppress_trailing=True): +def itemlist(item, sep, suppress_trailing=True, **kwargs): """Create a list of items separated by seps with comma-like spacing added. A trailing sep is allowed.""" return attach( - item - + ZeroOrMore(sep + item) - + Optional(sep.suppress() if suppress_trailing else sep), + tokenlist(item, sep, suppress=False, suppress_trailing=suppress_trailing, **kwargs), add_list_spacing, ) -def exprlist(expr, op): +def exprlist(expr, op, **kwargs): """Create a list of exprs separated by ops with plus-like spacing added. No trailing op is allowed.""" - return addspace(expr + ZeroOrMore(op + expr)) + return addspace(tokenlist(expr, op, suppress=False, allow_trailing=False, **kwargs)) def stores_loc_action(loc, tokens): diff --git a/coconut/constants.py b/coconut/constants.py index adbfa2e9f..2b397f4a7 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -437,58 +437,58 @@ def get_bool_env_var(env_var, default=False): "sys.get_asyncgen_hooks": ("async_generator./get_asyncgen_hooks", (3, 6)), "sys.set_asyncgen_hooks": ("async_generator./set_asyncgen_hooks", (3, 6)), - # # typing_extensions (not needed since _coconut.typing has them - # # and mypy is happy to accept that they always live in typing) - # "typing.AsyncContextManager": ("typing_extensions./AsyncContextManager", (3, 6)), - # "typing.AsyncGenerator": ("typing_extensions./AsyncGenerator", (3, 6)), - # "typing.AsyncIterable": ("typing_extensions./AsyncIterable", (3, 6)), - # "typing.AsyncIterator": ("typing_extensions./AsyncIterator", (3, 6)), - # "typing.Awaitable": ("typing_extensions./Awaitable", (3, 6)), - # "typing.ChainMap": ("typing_extensions./ChainMap", (3, 6)), - # "typing.ClassVar": ("typing_extensions./ClassVar", (3, 6)), - # "typing.ContextManager": ("typing_extensions./ContextManager", (3, 6)), - # "typing.Coroutine": ("typing_extensions./Coroutine", (3, 6)), - # "typing.Counter": ("typing_extensions./Counter", (3, 6)), - # "typing.DefaultDict": ("typing_extensions./DefaultDict", (3, 6)), - # "typing.Deque": ("typing_extensions./Deque", (3, 6)), - # "typing.NamedTuple": ("typing_extensions./NamedTuple", (3, 6)), - # "typing.NewType": ("typing_extensions./NewType", (3, 6)), - # "typing.NoReturn": ("typing_extensions./NoReturn", (3, 6)), - # "typing.overload": ("typing_extensions./overload", (3, 6)), - # "typing.Text": ("typing_extensions./Text", (3, 6)), - # "typing.Type": ("typing_extensions./Type", (3, 6)), - # "typing.TYPE_CHECKING": ("typing_extensions./TYPE_CHECKING", (3, 6)), - # "typing.get_type_hints": ("typing_extensions./get_type_hints", (3, 6)), - # "typing.OrderedDict": ("typing_extensions./OrderedDict", (3, 7)), - # "typing.final": ("typing_extensions./final", (3, 8)), - # "typing.Final": ("typing_extensions./Final", (3, 8)), - # "typing.Literal": ("typing_extensions./Literal", (3, 8)), - # "typing.Protocol": ("typing_extensions./Protocol", (3, 8)), - # "typing.runtime_checkable": ("typing_extensions./runtime_checkable", (3, 8)), - # "typing.TypedDict": ("typing_extensions./TypedDict", (3, 8)), - # "typing.get_origin": ("typing_extensions./get_origin", (3, 8)), - # "typing.get_args": ("typing_extensions./get_args", (3, 8)), - # "typing.Annotated": ("typing_extensions./Annotated", (3, 9)), - # "typing.Concatenate": ("typing_extensions./Concatenate", (3, 10)), - # "typing.ParamSpec": ("typing_extensions./ParamSpec", (3, 10)), - # "typing.ParamSpecArgs": ("typing_extensions./ParamSpecArgs", (3, 10)), - # "typing.ParamSpecKwargs": ("typing_extensions./ParamSpecKwargs", (3, 10)), - # "typing.TypeAlias": ("typing_extensions./TypeAlias", (3, 10)), - # "typing.TypeGuard": ("typing_extensions./TypeGuard", (3, 10)), - # "typing.is_typeddict": ("typing_extensions./is_typeddict", (3, 10)), - # "typing.assert_never": ("typing_extensions./assert_never", (3, 11)), - # "typing.assert_type": ("typing_extensions./assert_type", (3, 11)), - # "typing.clear_overloads": ("typing_extensions./clear_overloads", (3, 11)), - # "typing.dataclass_transform": ("typing_extensions./dataclass_transform", (3, 11)), - # "typing.get_overloads": ("typing_extensions./get_overloads", (3, 11)), - # "typing.LiteralString": ("typing_extensions./LiteralString", (3, 11)), - # "typing.Never": ("typing_extensions./Never", (3, 11)), - # "typing.NotRequired": ("typing_extensions./NotRequired", (3, 11)), - # "typing.reveal_type": ("typing_extensions./reveal_type", (3, 11)), - # "typing.Required": ("typing_extensions./Required", (3, 11)), - # "typing.Self": ("typing_extensions./Self", (3, 11)), - # "typing.TypeVarTuple": ("typing_extensions./TypeVarTuple", (3, 11)), - # "typing.Unpack": ("typing_extensions./Unpack", (3, 11)), + # typing_extensions (even though we have special support for getting + # these from typing, we need to do this for the sake of type checkers) + "typing.AsyncContextManager": ("typing_extensions./AsyncContextManager", (3, 6)), + "typing.AsyncGenerator": ("typing_extensions./AsyncGenerator", (3, 6)), + "typing.AsyncIterable": ("typing_extensions./AsyncIterable", (3, 6)), + "typing.AsyncIterator": ("typing_extensions./AsyncIterator", (3, 6)), + "typing.Awaitable": ("typing_extensions./Awaitable", (3, 6)), + "typing.ChainMap": ("typing_extensions./ChainMap", (3, 6)), + "typing.ClassVar": ("typing_extensions./ClassVar", (3, 6)), + "typing.ContextManager": ("typing_extensions./ContextManager", (3, 6)), + "typing.Coroutine": ("typing_extensions./Coroutine", (3, 6)), + "typing.Counter": ("typing_extensions./Counter", (3, 6)), + "typing.DefaultDict": ("typing_extensions./DefaultDict", (3, 6)), + "typing.Deque": ("typing_extensions./Deque", (3, 6)), + "typing.NamedTuple": ("typing_extensions./NamedTuple", (3, 6)), + "typing.NewType": ("typing_extensions./NewType", (3, 6)), + "typing.NoReturn": ("typing_extensions./NoReturn", (3, 6)), + "typing.overload": ("typing_extensions./overload", (3, 6)), + "typing.Text": ("typing_extensions./Text", (3, 6)), + "typing.Type": ("typing_extensions./Type", (3, 6)), + "typing.TYPE_CHECKING": ("typing_extensions./TYPE_CHECKING", (3, 6)), + "typing.get_type_hints": ("typing_extensions./get_type_hints", (3, 6)), + "typing.OrderedDict": ("typing_extensions./OrderedDict", (3, 7)), + "typing.final": ("typing_extensions./final", (3, 8)), + "typing.Final": ("typing_extensions./Final", (3, 8)), + "typing.Literal": ("typing_extensions./Literal", (3, 8)), + "typing.Protocol": ("typing_extensions./Protocol", (3, 8)), + "typing.runtime_checkable": ("typing_extensions./runtime_checkable", (3, 8)), + "typing.TypedDict": ("typing_extensions./TypedDict", (3, 8)), + "typing.get_origin": ("typing_extensions./get_origin", (3, 8)), + "typing.get_args": ("typing_extensions./get_args", (3, 8)), + "typing.Annotated": ("typing_extensions./Annotated", (3, 9)), + "typing.Concatenate": ("typing_extensions./Concatenate", (3, 10)), + "typing.ParamSpec": ("typing_extensions./ParamSpec", (3, 10)), + "typing.ParamSpecArgs": ("typing_extensions./ParamSpecArgs", (3, 10)), + "typing.ParamSpecKwargs": ("typing_extensions./ParamSpecKwargs", (3, 10)), + "typing.TypeAlias": ("typing_extensions./TypeAlias", (3, 10)), + "typing.TypeGuard": ("typing_extensions./TypeGuard", (3, 10)), + "typing.is_typeddict": ("typing_extensions./is_typeddict", (3, 10)), + "typing.assert_never": ("typing_extensions./assert_never", (3, 11)), + "typing.assert_type": ("typing_extensions./assert_type", (3, 11)), + "typing.clear_overloads": ("typing_extensions./clear_overloads", (3, 11)), + "typing.dataclass_transform": ("typing_extensions./dataclass_transform", (3, 11)), + "typing.get_overloads": ("typing_extensions./get_overloads", (3, 11)), + "typing.LiteralString": ("typing_extensions./LiteralString", (3, 11)), + "typing.Never": ("typing_extensions./Never", (3, 11)), + "typing.NotRequired": ("typing_extensions./NotRequired", (3, 11)), + "typing.reveal_type": ("typing_extensions./reveal_type", (3, 11)), + "typing.Required": ("typing_extensions./Required", (3, 11)), + "typing.Self": ("typing_extensions./Self", (3, 11)), + "typing.TypeVarTuple": ("typing_extensions./TypeVarTuple", (3, 11)), + "typing.Unpack": ("typing_extensions./Unpack", (3, 11)), } import_existing = { diff --git a/coconut/root.py b/coconut/root.py index 41ef276e0..46ef915e3 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 13 +DEVELOP = 14 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/src/cocotest/agnostic/suite.coco b/coconut/tests/src/cocotest/agnostic/suite.coco index cb4f2b6c1..666fb773f 100644 --- a/coconut/tests/src/cocotest/agnostic/suite.coco +++ b/coconut/tests/src/cocotest/agnostic/suite.coco @@ -1052,6 +1052,7 @@ forward 2""") == 900 assert ret_args_kwargs(...=really_long_var) == ((), {"really_long_var": 10}) == ret_args_kwargs$(...=really_long_var)() assert ret_args_kwargs(123, ...=really_long_var, abc="abc") == ((123,), {"really_long_var": 10, "abc": "abc"}) == ret_args_kwargs$(123, ...=really_long_var, abc="abc")() assert "Coconut version of typing" in typing.__doc__ + numlist: NumList = [1, 2.3, 5] # must come at end assert fibs_calls[0] == 1 diff --git a/coconut/tests/src/cocotest/agnostic/util.coco b/coconut/tests/src/cocotest/agnostic/util.coco index eee8c2de5..38cbadc26 100644 --- a/coconut/tests/src/cocotest/agnostic/util.coco +++ b/coconut/tests/src/cocotest/agnostic/util.coco @@ -243,6 +243,8 @@ if sys.version_info >= (3, 5) or TYPE_CHECKING: type TextMap[T <: typing.Text, U] = typing.Mapping[T, U] + type NumList[T : (int, float)] = typing.List[T] + class HasT: T = 1 @@ -297,7 +299,7 @@ def qsort4(l: int[]) -> int[]: return None # type: ignore def qsort5(l: int$[]) -> int$[]: """Iterator Match Quick Sort.""" - match (head,) :: tail in l: # type: ignore + match (head,) :: tail in l: tail, tail_ = tee(tail) return (qsort5((x for x in tail if x <= head)) :: (head,) # The pivot is a tuple @@ -306,8 +308,8 @@ def qsort5(l: int$[]) -> int$[]: else: return iter(()) def qsort6(l: int$[]) -> int$[]: - match [head] :: tail in l: # type: ignore - tail = reiterable(tail) # type: ignore + match [head] :: tail in l: + tail = reiterable(tail) yield from ( qsort6(x for x in tail if x <= head) :: (head,) @@ -619,11 +621,11 @@ def factorial5(value): return None raise TypeError() -match def fact(n) = fact(n, 1) # type: ignore +match def fact(n) = fact(n, 1) match addpattern def fact(0, acc) = acc # type: ignore addpattern match def fact(n, acc) = fact(n-1, acc*n) # type: ignore -addpattern def factorial(0, acc=1) = acc # type: ignore +addpattern def factorial(0, acc=1) = acc addpattern def factorial(int() as n, acc=1 if n > 0) = # type: ignore """this is a docstring""" factorial(n-1, acc*n) diff --git a/coconut/tests/src/extras.coco b/coconut/tests/src/extras.coco index bb333ac69..fb46c2e99 100644 --- a/coconut/tests/src/extras.coco +++ b/coconut/tests/src/extras.coco @@ -369,6 +369,12 @@ async def async_map_test() = setup(target="3.11") assert parse("a[x, *y]") + setup(target="3.12") + assert parse("type Num = int | float").strip().endswith(""" +# Compiled Coconut: ----------------------------------------------------------- + +type Num = int | float""".strip()) + setup(minify=True) assert parse("123 # derp", "lenient") == "123# derp" From 379d898b503b76bc0a1189a8108523692260d6fe Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Sun, 28 May 2023 18:33:35 -0700 Subject: [PATCH 55/57] Improve docs --- DOCS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCS.md b/DOCS.md index 5eabc019c..9b4d61a9a 100644 --- a/DOCS.md +++ b/DOCS.md @@ -424,14 +424,14 @@ To distribute your code with checkable type annotations, you'll need to include ##### Syntax -To explicitly annotate your code with types to be checked, Coconut supports: +To explicitly annotate your code with types to be checked, Coconut supports (on all Python versions): * [Python 3 function type annotations](https://www.python.org/dev/peps/pep-0484/), * [Python 3.6 variable type annotations](https://www.python.org/dev/peps/pep-0526/), * [Python 3.12 type parameter syntax](#type-parameter-syntax) for easily adding type parameters to classes, functions, [`data` types](#data), and type aliases, * Coconut's own [enhanced type annotation syntax](#enhanced-type-annotation), and * Coconut's [protocol intersection operator](#protocol-intersection). -By default, all type annotations are compiled to Python-2-compatible type comments, which means it all works on any Python version. +By default, all type annotations are compiled to Python-2-compatible type comments, which means they should all work on any Python version. Sometimes, MyPy will not know how to handle certain Coconut constructs, such as `addpattern`. For the `addpattern` case, it is recommended to pass `--allow-redefinition` to MyPy (i.e. run `coconut --mypy --allow-redefinition`), though in some cases `--allow-redefinition` may not be sufficient. In that case, either hide the offending code using [`TYPE_CHECKING`](#type_checking) or put a `# type: ignore` comment on the Coconut line which is generating the line MyPy is complaining about and the comment will be added to every generated line. From 5605b8064400b6297a0d76bc7fefac04981a7b20 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 29 May 2023 12:54:24 -0700 Subject: [PATCH 56/57] Improve xonsh, windows --- coconut/constants.py | 2 +- coconut/integrations.py | 6 +++--- coconut/root.py | 2 +- coconut/tests/main_test.py | 36 ++++++++++++++++++------------------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/coconut/constants.py b/coconut/constants.py index 2b397f4a7..c6bc04fdd 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -1183,7 +1183,7 @@ def get_bool_env_var(env_var, default=False): conda_build_env_var = "CONDA_BUILD" -disabled_xonsh_modes = ("exec", "eval") +enabled_xonsh_modes = ("single",) # ----------------------------------------------------------------------------------------------------------------------- # DOCUMENTATION CONSTANTS: diff --git a/coconut/integrations.py b/coconut/integrations.py index f453cdbd8..f13375c65 100644 --- a/coconut/integrations.py +++ b/coconut/integrations.py @@ -23,7 +23,7 @@ from coconut.constants import ( coconut_kernel_kwargs, - disabled_xonsh_modes, + enabled_xonsh_modes, ) from coconut.util import memoize_with_exceptions @@ -137,14 +137,14 @@ def new_try_subproc_toks(self, ctxtransformer, node, *args, **kwargs): def new_parse(self, parser, code, mode="exec", *args, **kwargs): """Coconut-aware version of xonsh's _parse.""" - if self.loaded and mode not in disabled_xonsh_modes: + if self.loaded and mode in enabled_xonsh_modes: code, _ = self.compile_code(code) return parser.__class__.parse(parser, code, mode=mode, *args, **kwargs) def new_ctxvisit(self, ctxtransformer, node, inp, ctx, mode="exec", *args, **kwargs): """Version of ctxvisit that ensures looking up original lines in inp using Coconut line numbers will work properly.""" - if self.loaded and mode not in disabled_xonsh_modes: + if self.loaded and mode in enabled_xonsh_modes: from xonsh.tools import get_logical_line # hide imports to avoid circular dependencies diff --git a/coconut/root.py b/coconut/root.py index 46ef915e3..30a9c5058 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -26,7 +26,7 @@ VERSION = "3.0.1" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 14 +DEVELOP = 15 ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1" diff --git a/coconut/tests/main_test.py b/coconut/tests/main_test.py index a91843cfb..b5183d6fb 100644 --- a/coconut/tests/main_test.py +++ b/coconut/tests/main_test.py @@ -877,24 +877,24 @@ def test_simple_minify(self): run_runnable(["-n", "--minify"]) -@add_test_func_names -class TestExternal(unittest.TestCase): - - if not PYPY or PY2: - def test_prelude(self): - with using_path(prelude): - comp_prelude() - if MYPY and PY38: - run_prelude() - - def test_bbopt(self): - with using_path(bbopt): - comp_bbopt() - if not PYPY and PY38 and not PY310: - install_bbopt() - - # more appveyor timeout prevention - if not WINDOWS: +# more appveyor timeout prevention +if not WINDOWS: + @add_test_func_names + class TestExternal(unittest.TestCase): + + if not PYPY or PY2: + def test_prelude(self): + with using_path(prelude): + comp_prelude() + if MYPY and PY38: + run_prelude() + + def test_bbopt(self): + with using_path(bbopt): + comp_bbopt() + if not PYPY and PY38 and not PY310: + install_bbopt() + def test_pyprover(self): with using_path(pyprover): comp_pyprover() From 5ca9eb5f5d6d5e92cac99242af3a900440d40396 Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Mon, 29 May 2023 15:49:51 -0700 Subject: [PATCH 57/57] Prepare for v3.0.2 release --- coconut/root.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coconut/root.py b/coconut/root.py index 30a9c5058..4b0454312 100644 --- a/coconut/root.py +++ b/coconut/root.py @@ -23,10 +23,10 @@ # VERSION: # ----------------------------------------------------------------------------------------------------------------------- -VERSION = "3.0.1" +VERSION = "3.0.2" VERSION_NAME = None # False for release, int >= 1 for develop -DEVELOP = 15 +DEVELOP = False ALPHA = False # for pre releases rather than post releases assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"