diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 585589d6ce3bf7..2fd933a22f3caa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -55,7 +55,7 @@ Python/traceback.c @iritkatriel /Lib/html/ @ezio-melotti /Lib/_markupbase.py @ezio-melotti /Lib/test/test_html*.py @ezio-melotti -/Tools/scripts/*html5* @ezio-melotti +/Tools/build/parse_html5_entities.py @ezio-melotti # Import (including importlib). # Ignoring importlib.h so as to not get flagged on diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 9f4754f912b09f..969515ed287b55 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -6,12 +6,12 @@ on: paths: - 'Lib/ensurepip/_bundled/**' - '.github/workflows/verify-ensurepip-wheels.yml' - - 'Tools/scripts/verify_ensurepip_wheels.py' + - 'Tools/build/verify_ensurepip_wheels.py' pull_request: paths: - 'Lib/ensurepip/_bundled/**' - '.github/workflows/verify-ensurepip-wheels.yml' - - 'Tools/scripts/verify_ensurepip_wheels.py' + - 'Tools/build/verify_ensurepip_wheels.py' permissions: contents: read @@ -29,4 +29,4 @@ jobs: with: python-version: '3' - name: Compare checksums of bundled pip and setuptools to ones published on PyPI - run: ./Tools/scripts/verify_ensurepip_wheels.py + run: ./Tools/build/verify_ensurepip_wheels.py diff --git a/.gitignore b/.gitignore index 924c136ba9aa05..6934faa91e9874 100644 --- a/.gitignore +++ b/.gitignore @@ -143,7 +143,7 @@ Tools/ssl/win32 Tools/freeze/test/outdir # The frozen modules are always generated by the build so we don't -# keep them in the repo. Also see Tools/scripts/freeze_modules.py. +# keep them in the repo. Also see Tools/build/freeze_modules.py. Python/frozen_modules/*.h # The manifest can be generated at any time with "make regen-frozen". Python/frozen_modules/MANIFEST diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 1cff4e7215cf49..cd1f2ef7076836 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -11,6 +11,26 @@ The macros in this section are used for managing reference counts of Python objects. +.. c:function:: Py_ssize_t Py_REFCNT(PyObject *o) + + Get the reference count of the Python object *o*. + + Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. + + .. versionchanged:: 3.11 + The parameter type is no longer :c:expr:`const PyObject*`. + + .. versionchanged:: 3.10 + :c:func:`Py_REFCNT()` is changed to the inline static function. + + +.. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) + + Set the object *o* reference counter to *refcnt*. + + .. versionadded:: 3.9 + + .. c:function:: void Py_INCREF(PyObject *o) Increment the reference count for object *o*. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 76803a093353b5..ee757e3a186e58 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -17,7 +17,8 @@ All Python objects ultimately share a small number of fields at the beginning of the object's representation in memory. These are represented by the :c:type:`PyObject` and :c:type:`PyVarObject` types, which are defined, in turn, by the expansions of some macros also used, whether directly or indirectly, in -the definition of all other Python objects. +the definition of all other Python objects. Additional macros can be found +under :ref:`reference counting `. .. c:type:: PyObject @@ -121,26 +122,6 @@ the definition of all other Python objects. .. versionadded:: 3.9 -.. c:function:: Py_ssize_t Py_REFCNT(PyObject *o) - - Get the reference count of the Python object *o*. - - Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. - - .. versionchanged:: 3.11 - The parameter type is no longer :c:expr:`const PyObject*`. - - .. versionchanged:: 3.10 - :c:func:`Py_REFCNT()` is changed to the inline static function. - - -.. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) - - Set the object *o* reference counter to *refcnt*. - - .. versionadded:: 3.9 - - .. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) Get the size of the Python object *o*. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 99e886c61b4c5f..eac34aaab3aa53 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -276,6 +276,211 @@ choose a different directory name for the log - just ensure that the directory e and that you have the permissions to create and update files in it. +.. _custom-level-handling: + +Custom handling of levels +------------------------- + +Sometimes, you might want to do something slightly different from the standard +handling of levels in handlers, where all levels above a threshold get +processed by a handler. To do this, you need to use filters. Let's look at a +scenario where you want to arrange things as follows: + +* Send messages of severity ``INFO`` and ``WARNING`` to ``sys.stdout`` +* Send messages of severity ``ERROR`` and above to ``sys.stderr`` +* Send messages of severity ``DEBUG`` and above to file ``app.log`` + +Suppose you configure logging with the following JSON: + +.. code-block:: json + + { + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(levelname)-8s - %(message)s" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout", + }, + "stderr": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "simple", + "stream": "ext://sys.stderr" + }, + "file": { + "class": "logging.FileHandler", + "formatter": "simple", + "filename": "app.log", + "mode": "w" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "stderr", + "stdout", + "file" + ] + } + } + +This configuration does *almost* what we want, except that ``sys.stdout`` would +show messages of severity ``ERROR`` and above as well as ``INFO`` and +``WARNING`` messages. To prevent this, we can set up a filter which excludes +those messages and add it to the relevant handler. This can be configured by +adding a ``filters`` section parallel to ``formatters`` and ``handlers``: + +.. code-block:: json + + "filters": { + "warnings_and_below": { + "()" : "__main__.filter_maker", + "level": "WARNING" + } + } + +and changing the section on the ``stdout`` handler to add it: + +.. code-block:: json + + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout", + "filters": ["warnings_and_below"] + } + +A filter is just a function, so we can define the ``filter_maker`` (a factory +function) as follows: + +.. code-block:: python + + def filter_maker(level): + level = getattr(logging, level) + + def filter(record): + return record.levelno <= level + + return filter + +This converts the string argument passed in to a numeric level, and returns a +function which only returns ``True`` if the level of the passed in record is +at or below the specified level. Note that in this example I have defined the +``filter_maker`` in a test script ``main.py`` that I run from the command line, +so its module will be ``__main__`` - hence the ``__main__.filter_maker`` in the +filter configuration. You will need to change that if you define it in a +different module. + +With the filter added, we can run ``main.py``, which in full is: + +.. code-block:: python + + import json + import logging + import logging.config + + CONFIG = ''' + { + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(levelname)-8s - %(message)s" + } + }, + "filters": { + "warnings_and_below": { + "()" : "__main__.filter_maker", + "level": "WARNING" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout", + "filters": ["warnings_and_below"] + }, + "stderr": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "simple", + "stream": "ext://sys.stderr" + }, + "file": { + "class": "logging.FileHandler", + "formatter": "simple", + "filename": "app.log", + "mode": "w" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "stderr", + "stdout", + "file" + ] + } + } + ''' + + def filter_maker(level): + level = getattr(logging, level) + + def filter(record): + return record.levelno <= level + + return filter + + logging.config.dictConfig(json.loads(CONFIG)) + logging.debug('A DEBUG message') + logging.info('An INFO message') + logging.warning('A WARNING message') + logging.error('An ERROR message') + logging.critical('A CRITICAL message') + +And after running it like this: + +.. code-block:: shell + + python main.py 2>stderr.log >stdout.log + +We can see the results are as expected: + +.. code-block:: shell + + $ more *.log + :::::::::::::: + app.log + :::::::::::::: + DEBUG - A DEBUG message + INFO - An INFO message + WARNING - A WARNING message + ERROR - An ERROR message + CRITICAL - A CRITICAL message + :::::::::::::: + stderr.log + :::::::::::::: + ERROR - An ERROR message + CRITICAL - A CRITICAL message + :::::::::::::: + stdout.log + :::::::::::::: + INFO - An INFO message + WARNING - A WARNING message + + Configuration server example ---------------------------- @@ -3503,7 +3708,7 @@ instance). Then, you'd get this kind of result: WARNING:demo:Bar >>> -Of course, these above examples show output according to the format used by +Of course, the examples above show output according to the format used by :func:`~logging.basicConfig`, but you can use a different formatter when you configure logging. @@ -3517,7 +3722,6 @@ need to do or deal with, it is worth mentioning some usage patterns which are *unhelpful*, and which should therefore be avoided in most cases. The following sections are in no particular order. - Opening the same log file multiple times ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3566,7 +3770,6 @@ that in other languages such as Java and C#, loggers are often static class attributes. However, this pattern doesn't make sense in Python, where the module (and not the class) is the unit of software decomposition. - Adding handlers other than :class:`NullHandler` to a logger in a library ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3575,7 +3778,6 @@ responsibility of the application developer, not the library developer. If you are maintaining a library, ensure that you don't add handlers to any of your loggers other than a :class:`~logging.NullHandler` instance. - Creating a lot of loggers ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 53cbe01e92144b..588e895b04bde2 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -186,8 +186,8 @@ The `Timsort `_ algorithm used in Python does multiple sorts efficiently because it can take advantage of any ordering already present in a dataset. -The Old Way Using Decorate-Sort-Undecorate -========================================== +Decorate-Sort-Undecorate +======================== This idiom is called Decorate-Sort-Undecorate after its three steps: @@ -226,90 +226,36 @@ after Randal L. Schwartz, who popularized it among Perl programmers. Now that Python sorting provides key-functions, this technique is not often needed. +Comparison Functions +==================== -The Old Way Using the *cmp* Parameter -===================================== - -Many constructs given in this HOWTO assume Python 2.4 or later. Before that, -there was no :func:`sorted` builtin and :meth:`list.sort` took no keyword -arguments. Instead, all of the Py2.x versions supported a *cmp* parameter to -handle user specified comparison functions. - -In Py3.0, the *cmp* parameter was removed entirely (as part of a larger effort to -simplify and unify the language, eliminating the conflict between rich -comparisons and the :meth:`__cmp__` magic method). - -In Py2.x, sort allowed an optional function which can be called for doing the -comparisons. That function should take two arguments to be compared and then -return a negative value for less-than, return zero if they are equal, or return -a positive value for greater-than. For example, we can do: - -.. doctest:: - - >>> def numeric_compare(x, y): - ... return x - y - >>> sorted([5, 2, 4, 1, 3], cmp=numeric_compare) # doctest: +SKIP - [1, 2, 3, 4, 5] - -Or you can reverse the order of comparison with: - -.. doctest:: - - >>> def reverse_numeric(x, y): - ... return y - x - >>> sorted([5, 2, 4, 1, 3], cmp=reverse_numeric) # doctest: +SKIP - [5, 4, 3, 2, 1] - -When porting code from Python 2.x to 3.x, the situation can arise when you have -the user supplying a comparison function and you need to convert that to a key -function. The following wrapper makes that easy to do: - -.. testcode:: - - def cmp_to_key(mycmp): - 'Convert a cmp= function into a key= function' - class K: - def __init__(self, obj, *args): - self.obj = obj - def __lt__(self, other): - return mycmp(self.obj, other.obj) < 0 - def __gt__(self, other): - return mycmp(self.obj, other.obj) > 0 - def __eq__(self, other): - return mycmp(self.obj, other.obj) == 0 - def __le__(self, other): - return mycmp(self.obj, other.obj) <= 0 - def __ge__(self, other): - return mycmp(self.obj, other.obj) >= 0 - def __ne__(self, other): - return mycmp(self.obj, other.obj) != 0 - return K - -.. doctest:: - :hide: +Unlike key functions that return an absolute value for sorting, a comparison +function computes the relative ordering for two inputs. - >>> sorted([5, 2, 4, 1, 3], key=cmp_to_key(reverse_numeric)) - [5, 4, 3, 2, 1] +For example, a `balance scale +`_ +compares two samples giving a relative ordering: lighter, equal, or heavier. +Likewise, a comparison function such as ``cmp(a, b)`` will return a negative +value for less-than, zero if the inputs are equal, or a positive value for +greater-than. -To convert to a key function, just wrap the old comparison function: - -.. testsetup:: - - from functools import cmp_to_key - -.. doctest:: +It is common to encounter comparison functions when translating algorithms from +other languages. Also, some libraries provide comparison functions as part of +their API. For example, :func:`locale.strcoll` is a comparison function. - >>> sorted([5, 2, 4, 1, 3], key=cmp_to_key(reverse_numeric)) - [5, 4, 3, 2, 1] +To accommodate those situations, Python provides +:class:`functools.cmp_to_key` to wrap the comparison function +to make it usable as a key function:: -In Python 3.2, the :func:`functools.cmp_to_key` function was added to the -:mod:`functools` module in the standard library. + sorted(words, key=cmp_to_key(strcoll) Odds and Ends ============= * For locale aware sorting, use :func:`locale.strxfrm` for a key function or - :func:`locale.strcoll` for a comparison function. + :func:`locale.strcoll` for a comparison function. This is necessary + because "alphabetical" sort orderings can vary across cultures even + if the underlying alphabet is the same. * The *reverse* parameter still maintains sort stability (so that records with equal keys retain the original order). Interestingly, that effect can be diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 2f306b7edb8fe2..4f12074dd70098 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1253,7 +1253,13 @@ Executing code in thread or process pools pool, cpu_bound) print('custom process pool', result) - asyncio.run(main()) + if __name__ == '__main__': + asyncio.run(main()) + + Note that the entry point guard (``if __name__ == '__main__'``) + is required for option 3 due to the peculiarities of :mod:`multiprocessing`, + which is used by :class:`~concurrent.futures.ProcessPoolExecutor`. + See :ref:`Safe importing of main module `. This method returns a :class:`asyncio.Future` object. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index bfc3e3090fdcdf..052378ef32743b 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -88,12 +88,16 @@ The abstract event loop policy base class is defined as follows: This function is Unix specific. + .. deprecated:: 3.12 + .. method:: set_child_watcher(watcher) Set the current child process watcher to *watcher*. This function is Unix specific. + .. deprecated:: 3.12 + .. _asyncio-policy-builtin: @@ -158,12 +162,16 @@ implementation used by the asyncio event loop: Return the current child watcher for the current policy. + .. deprecated:: 3.12 + .. function:: set_child_watcher(watcher) Set the current child watcher to *watcher* for the current policy. *watcher* must implement methods defined in the :class:`AbstractChildWatcher` base class. + .. deprecated:: 3.12 + .. note:: Third-party event loops implementations might not support custom child watchers. For such event loops, using @@ -245,6 +253,8 @@ implementation used by the asyncio event loop: .. versionadded:: 3.8 + .. deprecated:: 3.12 + .. class:: SafeChildWatcher This implementation uses active event loop from the main thread to handle @@ -257,6 +267,8 @@ implementation used by the asyncio event loop: This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* complexity but requires a running event loop in the main thread to work. + .. deprecated:: 3.12 + .. class:: FastChildWatcher This implementation reaps every terminated processes by calling @@ -269,6 +281,8 @@ implementation used by the asyncio event loop: This solution requires a running event loop in the main thread to work, as :class:`SafeChildWatcher`. + .. deprecated:: 3.12 + .. class:: PidfdChildWatcher This implementation polls process file descriptors (pidfds) to await child diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 93c9f4ad2bc23a..8108c98332e964 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -54,14 +54,14 @@ are always available. They are listed here in alphabetical order. .. |func-bytearray| replace:: ``bytearray()`` .. |func-bytes| replace:: ``bytes()`` -.. function:: abs(x) +.. function:: abs(x, /) Return the absolute value of a number. The argument may be an integer, a floating point number, or an object implementing :meth:`__abs__`. If the argument is a complex number, its magnitude is returned. -.. function:: aiter(async_iterable) +.. function:: aiter(async_iterable, /) Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. @@ -70,7 +70,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: all(iterable) +.. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable is empty). Equivalent to:: @@ -82,7 +82,8 @@ are always available. They are listed here in alphabetical order. return True -.. awaitablefunction:: anext(async_iterator[, default]) +.. awaitablefunction:: anext(async_iterator, /) + anext(async_iterator, default, /) When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. @@ -97,7 +98,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: any(iterable) +.. function:: any(iterable, /) Return ``True`` if any element of the *iterable* is true. If the iterable is empty, return ``False``. Equivalent to:: @@ -109,7 +110,7 @@ are always available. They are listed here in alphabetical order. return False -.. function:: ascii(object) +.. function:: ascii(object, /) As :func:`repr`, return a string containing a printable representation of an object, but escape the non-ASCII characters in the string returned by @@ -117,7 +118,7 @@ are always available. They are listed here in alphabetical order. similar to that returned by :func:`repr` in Python 2. -.. function:: bin(x) +.. function:: bin(x, /) Convert an integer number to a binary string prefixed with "0b". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it @@ -139,7 +140,7 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. -.. class:: bool([x]) +.. class:: bool(x=False, /) Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted using the standard :ref:`truth testing procedure `. If *x* is false @@ -172,7 +173,9 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.7 .. _func-bytearray: -.. class:: bytearray([source[, encoding[, errors]]]) +.. class:: bytearray(source=b'') + bytearray(source, encoding) + bytearray(source, encoding, errors) :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable @@ -202,7 +205,9 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: -.. class:: bytes([source[, encoding[, errors]]]) +.. class:: bytes(source=b'') + bytes(source, encoding) + bytes(source, encoding, errors) :noindex: Return a new "bytes" object which is an immutable sequence of integers in @@ -217,7 +222,7 @@ are always available. They are listed here in alphabetical order. See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. -.. function:: callable(object) +.. function:: callable(object, /) Return :const:`True` if the *object* argument appears callable, :const:`False` if not. If this returns ``True``, it is still possible that a @@ -230,7 +235,7 @@ are always available. They are listed here in alphabetical order. in Python 3.2. -.. function:: chr(i) +.. function:: chr(i, /) Return the string representing a character whose Unicode code point is the integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while @@ -358,7 +363,8 @@ are always available. They are listed here in alphabetical order. support for top-level ``await``, ``async for``, and ``async with``. -.. class:: complex([real[, imag]]) +.. class:: complex(real=0, imag=0) + complex(string, /) Return a complex number with the value *real* + *imag*\*1j or convert a string or number to a complex number. If the first parameter is a string, it will @@ -391,7 +397,7 @@ are always available. They are listed here in alphabetical order. :meth:`__float__` are not defined. -.. function:: delattr(object, name) +.. function:: delattr(object, name, /) This is a relative of :func:`setattr`. The arguments are an object and a string. The string must be the name of one of the object's attributes. The @@ -402,8 +408,8 @@ are always available. They are listed here in alphabetical order. .. _func-dict: .. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) + dict(mapping, /, **kwarg) + dict(iterable, /, **kwarg) :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. @@ -413,7 +419,8 @@ are always available. They are listed here in alphabetical order. :class:`tuple` classes, as well as the :mod:`collections` module. -.. function:: dir([object]) +.. function:: dir() + dir(object, /) Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. @@ -469,7 +476,7 @@ are always available. They are listed here in alphabetical order. class. -.. function:: divmod(a, b) +.. function:: divmod(a, b, /) Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With @@ -505,7 +512,7 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression[, globals[, locals]]) +.. function:: eval(expression, /, globals=None, locals=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -556,7 +563,7 @@ are always available. They are listed here in alphabetical order. .. index:: builtin: exec -.. function:: exec(object[, globals[, locals]], *, closure=None) +.. function:: exec(object, globals=None, locals=None, /, *, closure=None) This function supports dynamic execution of Python code. *object* must be either a string or a code object. If it is a string, the string is parsed as @@ -612,7 +619,7 @@ are always available. They are listed here in alphabetical order. Added the *closure* parameter. -.. function:: filter(function, iterable) +.. function:: filter(function, iterable, /) Construct an iterator from those elements of *iterable* for which *function* returns true. *iterable* may be either a sequence, a container which @@ -629,7 +636,7 @@ are always available. They are listed here in alphabetical order. elements of *iterable* for which *function* returns false. -.. class:: float([x]) +.. class:: float(x=0.0, /) .. index:: single: NaN @@ -697,7 +704,7 @@ are always available. They are listed here in alphabetical order. single: __format__ single: string; format() (built-in function) -.. function:: format(value[, format_spec]) +.. function:: format(value, format_spec="", /) Convert a *value* to a "formatted" representation, as controlled by *format_spec*. The interpretation of *format_spec* will depend on the type @@ -720,7 +727,7 @@ are always available. They are listed here in alphabetical order. .. _func-frozenset: -.. class:: frozenset([iterable]) +.. class:: frozenset(iterable=set(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from @@ -732,7 +739,8 @@ are always available. They are listed here in alphabetical order. module. -.. function:: getattr(object, name[, default]) +.. function:: getattr(object, name, /) + getattr(object, name, default, /) Return the value of the named attribute of *object*. *name* must be a string. If the string is the name of one of the object's attributes, the result is the @@ -756,7 +764,7 @@ are always available. They are listed here in alphabetical order. regardless of where the function is called. -.. function:: hasattr(object, name) +.. function:: hasattr(object, name, /) The arguments are an object and a string. The result is ``True`` if the string is the name of one of the object's attributes, ``False`` if not. (This @@ -764,7 +772,7 @@ are always available. They are listed here in alphabetical order. raises an :exc:`AttributeError` or not.) -.. function:: hash(object) +.. function:: hash(object, /) Return the hash value of the object (if it has one). Hash values are integers. They are used to quickly compare dictionary keys during a @@ -777,7 +785,8 @@ are always available. They are listed here in alphabetical order. truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. -.. function:: help([object]) +.. function:: help() + help(request) Invoke the built-in help system. (This function is intended for interactive use.) If no argument is given, the interactive help system starts on the @@ -798,7 +807,7 @@ are always available. They are listed here in alphabetical order. signatures for callables are now more comprehensive and consistent. -.. function:: hex(x) +.. function:: hex(x, /) Convert an integer number to a lowercase hexadecimal string prefixed with "0x". If *x* is not a Python :class:`int` object, it has to define an @@ -830,7 +839,7 @@ are always available. They are listed here in alphabetical order. :meth:`float.hex` method. -.. function:: id(object) +.. function:: id(object, /) Return the "identity" of an object. This is an integer which is guaranteed to be unique and constant for this object during its lifetime. @@ -842,7 +851,8 @@ are always available. They are listed here in alphabetical order. .. audit-event:: builtins.id id id -.. function:: input([prompt]) +.. function:: input() + input(prompt, /) If the *prompt* argument is present, it is written to standard output without a trailing newline. The function then reads a line from input, converts it @@ -868,8 +878,8 @@ are always available. They are listed here in alphabetical order. with the result after successfully reading input. -.. class:: int([x]) - int(x, base=10) +.. class:: int(x=0, /) + int(x, /, base=10) Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* defines :meth:`__int__`, @@ -920,7 +930,7 @@ are always available. They are listed here in alphabetical order. See the :ref:`integer string conversion length limitation ` documentation. -.. function:: isinstance(object, classinfo) +.. function:: isinstance(object, classinfo, /) Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect, or :term:`virtual `) of *classinfo*. A @@ -951,7 +961,8 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: iter(object[, sentinel]) +.. function:: iter(object, /) + iter(object, sentinel, /) Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a @@ -978,7 +989,7 @@ are always available. They are listed here in alphabetical order. process_block(block) -.. function:: len(s) +.. function:: len(s, /) Return the length (the number of items) of an object. The argument may be a sequence (such as a string, bytes, tuple, list, or range) or a collection @@ -991,7 +1002,8 @@ are always available. They are listed here in alphabetical order. .. _func-list: -.. class:: list([iterable]) +.. class:: list() + list(iterable, /) :noindex: Rather than being a function, :class:`list` is actually a mutable @@ -1009,18 +1021,19 @@ are always available. They are listed here in alphabetical order. The contents of this dictionary should not be modified; changes may not affect the values of local and free variables used by the interpreter. -.. function:: map(function, iterable, ...) +.. function:: map(function, iterable, /, *iterables) Return an iterator that applies *function* to every item of *iterable*, - yielding the results. If additional *iterable* arguments are passed, + yielding the results. If additional *iterables* arguments are passed, *function* must take that many arguments and is applied to the items from all iterables in parallel. With multiple iterables, the iterator stops when the shortest iterable is exhausted. For cases where the function inputs are already arranged into argument tuples, see :func:`itertools.starmap`\. -.. function:: max(iterable, *[, key, default]) - max(arg1, arg2, *args[, key]) +.. function:: max(iterable, /, *, key=None) + max(iterable, /, *, default, key=None) + max(arg1, arg2, /, *args, key=None) Return the largest item in an iterable or the largest of two or more arguments. @@ -1056,8 +1069,9 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, *[, key, default]) - min(arg1, arg2, *args[, key]) +.. function:: min(iterable, /, *, key=None) + min(iterable, /, *, default, key=None) + min(arg1, arg2, /, *args, key=None) Return the smallest item in an iterable or the smallest of two or more arguments. @@ -1085,7 +1099,8 @@ are always available. They are listed here in alphabetical order. The *key* can be ``None``. -.. function:: next(iterator[, default]) +.. function:: next(iterator, /) + next(iterator, default, /) Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned @@ -1104,7 +1119,7 @@ are always available. They are listed here in alphabetical order. assign arbitrary attributes to an instance of the :class:`object` class. -.. function:: oct(x) +.. function:: oct(x, /) Convert an integer number to an octal string prefixed with "0o". The result is a valid Python expression. If *x* is not a Python :class:`int` object, it @@ -1356,7 +1371,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. -.. function:: ord(c) +.. function:: ord(c, /) Given a string representing one Unicode character, return an integer representing the Unicode code point of that character. For example, @@ -1364,7 +1379,7 @@ are always available. They are listed here in alphabetical order. returns ``8364``. This is the inverse of :func:`chr`. -.. function:: pow(base, exp[, mod]) +.. function:: pow(base, exp, mod=None) Return *base* to the power *exp*; if *mod* is present, return *base* to the power *exp*, modulo *mod* (computed more efficiently than @@ -1507,15 +1522,15 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. class:: range(stop) - range(start, stop[, step]) +.. class:: range(stop, /) + range(start, stop, step=1, /) :noindex: Rather than being a function, :class:`range` is actually an immutable sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. -.. function:: repr(object) +.. function:: repr(object, /) Return a string containing a printable representation of an object. For many types, this function makes an attempt to return a string that would yield an @@ -1528,7 +1543,7 @@ are always available. They are listed here in alphabetical order. :exc:`RuntimeError`. -.. function:: reversed(seq) +.. function:: reversed(seq, /) Return a reverse :term:`iterator`. *seq* must be an object which has a :meth:`__reversed__` method or supports the sequence protocol (the @@ -1536,7 +1551,7 @@ are always available. They are listed here in alphabetical order. arguments starting at ``0``). -.. function:: round(number[, ndigits]) +.. function:: round(number, ndigits=None) Return *number* rounded to *ndigits* precision after the decimal point. If *ndigits* is omitted or is ``None``, it returns the @@ -1564,7 +1579,8 @@ are always available. They are listed here in alphabetical order. .. _func-set: -.. class:: set([iterable]) +.. class:: set() + set(iterable, /) :noindex: Return a new :class:`set` object, optionally with elements taken from @@ -1576,7 +1592,7 @@ are always available. They are listed here in alphabetical order. module. -.. function:: setattr(object, name, value) +.. function:: setattr(object, name, value, /) This is the counterpart of :func:`getattr`. The arguments are an object, a string, and an arbitrary value. The string may name an existing attribute or a @@ -1598,8 +1614,8 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: slice(stop) - slice(start, stop[, step]) +.. class:: slice(stop, /) + slice(start, stop, step=1, /) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to @@ -1716,21 +1732,22 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.8 The *start* parameter can be specified as a keyword argument. -.. class:: super([type[, object-or-type]]) +.. class:: super() + super(type, object_or_type=None, /) Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have been overridden in a class. - The *object-or-type* determines the :term:`method resolution order` + The *object_or_type* determines the :term:`method resolution order` to be searched. The search starts from the class right after the *type*. - For example, if :attr:`~class.__mro__` of *object-or-type* is + For example, if :attr:`~class.__mro__` of *object_or_type* is ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, then :func:`super` searches ``C -> A -> object``. - The :attr:`~class.__mro__` attribute of the *object-or-type* lists the method + The :attr:`~class.__mro__` attribute of the *object_or_type* lists the method resolution search order used by both :func:`getattr` and :func:`super`. The attribute is dynamic and can change whenever the inheritance hierarchy is updated. @@ -1786,15 +1803,16 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: -.. class:: tuple([iterable]) +.. class:: tuple() + tuple(iterable, /) :noindex: Rather than being a function, :class:`tuple` is actually an immutable sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. -.. class:: type(object) - type(name, bases, dict, **kwds) +.. class:: type(object, /) + type(name, bases, dict, /, **kwds) .. index:: object: type @@ -1834,7 +1852,8 @@ are always available. They are listed here in alphabetical order. Subclasses of :class:`type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. -.. function:: vars([object]) +.. function:: vars() + vars(object, /) Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`~object.__dict__` attribute. diff --git a/Doc/library/imghdr.rst b/Doc/library/imghdr.rst index 318fe650776d2a..630fd7019f94de 100644 --- a/Doc/library/imghdr.rst +++ b/Doc/library/imghdr.rst @@ -21,8 +21,8 @@ The :mod:`imghdr` module defines the following function: .. function:: what(file, h=None) - Tests the image data contained in the file named by *file*, and returns a - string describing the image type. If optional *h* is provided, the *file* + Test the image data contained in the file named *file* and return a + string describing the image type. If *h* is provided, the *file* argument is ignored and *h* is assumed to contain the byte stream to test. .. versionchanged:: 3.6 diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 6571114ef311f9..88e1e5aa6ef7f3 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -809,15 +809,26 @@ which incur interpreter overhead. for k in range(len(roots) + 1) ] + def iter_index(seq, value, start=0): + "Return indices where a value occurs in a sequence." + # iter_index('AABCADEAF', 'A') --> 0 1 4 7 + i = start - 1 + try: + while True: + yield (i := seq.index(value, i+1)) + except ValueError: + pass + def sieve(n): - "Primes less than n" - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 - data = bytearray([1]) * n - data[:2] = 0, 0 - limit = math.isqrt(n) + 1 - for p in compress(range(limit), data): - data[p+p : n : p] = bytearray(len(range(p+p, n, p))) - return compress(count(), data) + "Primes less than n" + # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + data = bytearray((0, 1)) * (n // 2) + data[:3] = 0, 0, 0 + limit = math.isqrt(n) + 1 + for p in compress(range(limit), data): + data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) + data[2] = 1 + return iter_index(data, 1) if n > 2 else iter([]) def flatten(list_of_lists): "Flatten one level of nesting" @@ -1170,6 +1181,15 @@ which incur interpreter overhead. >>> all(factored(x) == expanded(x) for x in range(-10, 11)) True + >>> list(iter_index('AABCADEAF', 'A')) + [0, 1, 4, 7] + >>> list(iter_index('AABCADEAF', 'B')) + [2] + >>> list(iter_index('AABCADEAF', 'X')) + [] + >>> list(iter_index('', 'X')) + [] + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index f3b26e726aafc7..d4429d3d0a4f73 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -650,6 +650,17 @@ supports sending logging messages to a remote or local Unix syslog. Closes the socket to the remote host. + .. method:: createSocket() + + Tries to create a socket and, if it's not a datagram socket, connect it + to the other end. This method is called during handler initialization, + but it's not regarded as an error if the other end isn't listening at + this point - the method will be called again when emitting an event, if + but it's not regarded as an error if the other end isn't listening yet + --- the method will be called again when emitting an event, + if there is no socket at that point. + + .. versionadded:: 3.11 .. method:: emit(record) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index dab115acdc207c..5516084780673f 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2954,6 +2954,8 @@ Global variables However, global variables which are just module level constants cause no problems. +.. _multiprocessing-safe-main-import: + Safe importing of main module Make sure that the main module can be safely imported by a new Python diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst index e1dbe4a1a34483..fa9323e18dc348 100644 --- a/Doc/library/sndhdr.rst +++ b/Doc/library/sndhdr.rst @@ -54,3 +54,51 @@ be the sample size in bits or ``'A'`` for A-LAW or ``'U'`` for u-LAW. .. versionchanged:: 3.5 Result changed from a tuple to a namedtuple. +The following sound header types are recognized, as listed below with the return value +from :func:`whathdr`: and :func:`what`: + ++------------+------------------------------------+ +| Value | Sound header format | ++============+====================================+ +| ``'aifc'`` | Compressed Audio Interchange Files | ++------------+------------------------------------+ +| ``'aiff'`` | Audio Interchange Files | ++------------+------------------------------------+ +| ``'au'`` | Au Files | ++------------+------------------------------------+ +| ``'hcom'`` | HCOM Files | ++------------+------------------------------------+ +| ``'sndt'`` | Sndtool Sound Files | ++------------+------------------------------------+ +| ``'voc'`` | Creative Labs Audio Files | ++------------+------------------------------------+ +| ``'wav'`` | Waveform Audio File Format Files | ++------------+------------------------------------+ +| ``'8svx'`` | 8-Bit Sampled Voice Files | ++------------+------------------------------------+ +| ``'sb'`` | Signed Byte Audio Data Files | ++------------+------------------------------------+ +| ``'ub'`` | UB Files | ++------------+------------------------------------+ +| ``'ul'`` | uLAW Audio Files | ++------------+------------------------------------+ + +.. data:: tests + + A list of functions performing the individual tests. Each function takes two + arguments: the byte-stream and an open file-like object. When :func:`what` is + called with a byte-stream, the file-like object will be ``None``. + + The test function should return a string describing the image type if the test + succeeded, or ``None`` if it failed. + +Example: + +.. code-block:: pycon + + >>> import sndhdr + >>> imghdr.what('bass.wav') + 'wav' + >>> imghdr.whathdr('bass.wav') + 'wav' + diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index 1a99f0518d1b47..2739d5bfc1dfa2 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -1,4 +1,4 @@ -.. Auto-generated by Tools/scripts/generate_token.py +.. Auto-generated by Tools/build/generate_token.py .. data:: ENDMARKER .. data:: NAME diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index fee55becc7f57d..dc5696aff69bd4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -248,7 +248,7 @@ respectively. .. versionchanged:: 3.10 ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. - See :pep:`612` for more information. + See :pep:`612` for more details. .. seealso:: The documentation for :class:`ParamSpec` and :class:`Concatenate` provides @@ -723,7 +723,7 @@ These can be used as types in annotations and do not support ``[]``. of the ``cls`` parameter. - Annotating an :meth:`~object.__enter__` method which returns self. - For more information, see :pep:`673`. + See :pep:`673` for more details. .. versionadded:: 3.11 @@ -854,7 +854,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionchanged:: 3.10 ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. - See :pep:`612` for more information. + See :pep:`612` for more details. .. seealso:: The documentation for :class:`ParamSpec` and :class:`Concatenate` provide @@ -1040,8 +1040,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn Special typing constructs that mark individual keys of a :class:`TypedDict` as either required or non-required respectively. - For more information, see :class:`TypedDict` and - :pep:`655` ("Marking individual TypedDict items as required or potentially missing"). + See :class:`TypedDict` and :pep:`655` for more details. .. versionadded:: 3.11 @@ -1192,8 +1191,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn is not a subtype of the former, since ``list`` is invariant. The responsibility of writing type-safe type guards is left to the user. - ``TypeGuard`` also works with type variables. For more information, see - :pep:`647` (User-Defined Type Guards). + ``TypeGuard`` also works with type variables. See :pep:`647` for more details. .. versionadded:: 3.10 @@ -1393,7 +1391,7 @@ These are not used in annotations. They are building blocks for creating generic to ``call_soon`` match the types of the (positional) arguments of ``callback``. - For more details on type variable tuples, see :pep:`646`. + See :pep:`646` for more details on type variable tuples. .. versionadded:: 3.11 @@ -1556,7 +1554,7 @@ These are not used in annotations. They are building blocks for creating generic func(C()) # Passes static type check - See :pep:`544` for details. Protocol classes decorated with + See :pep:`544` for more details. Protocol classes decorated with :func:`runtime_checkable` (described later) act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. @@ -2632,7 +2630,7 @@ Functions and decorators def process(response): - See :pep:`484` for details and comparison with other typing semantics. + See :pep:`484` for more details and comparison with other typing semantics. .. versionchanged:: 3.11 Overloaded functions can now be introspected at runtime using diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index d9183561820b2b..3f01180e13f776 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -67,7 +67,7 @@ The following constructs bind names: + :keyword:`for` loop header, + after :keyword:`!as` in a :keyword:`with` statement, :keyword:`except` - clause or in the as-pattern in structural pattern matching, + clause, :keyword:`except* ` clause, or in the as-pattern in structural pattern matching, + in a capture pattern in structural pattern matching * :keyword:`import` statements. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 56a35f4e4802ba..bb9538338e48f8 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -530,6 +530,17 @@ Other CPython Implementation Changes and with the new :option:`--help-all`. (Contributed by Éric Araujo in :issue:`46142`.) +* Converting between :class:`int` and :class:`str` in bases other than 2 + (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) + now raises a :exc:`ValueError` if the number of digits in string form is + above a limit to avoid potential denial of service attacks due to the + algorithmic complexity. This is a mitigation for `CVE-2020-10735 + `_. + This limit can be configured or disabled by environment variable, command + line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion + length limitation ` documentation. The default limit + is 4300 digits in string form. + .. _whatsnew311-new-modules: @@ -1667,6 +1678,10 @@ Removed Python's test suite." (Contributed by Victor Stinner in :issue:`46852`.) +* The ``--experimental-isolated-subinterpreters`` configure flag + (and corresponding ``EXPERIMENTAL_ISOLATED_SUBINTERPRETERS``) + have been removed. + * Pynche --- The Pythonically Natural Color and Hue Editor --- has been moved out of ``Tools/scripts`` and is `being developed independently `_ from the Python source tree. @@ -1884,6 +1899,17 @@ New Features Porting to Python 3.11 ---------------------- +* Some macros have been converted to static inline functions to avoid + `macro pitfalls `_. + The change should be mostly transparent to users, + as the replacement functions will cast their arguments to the expected types + to avoid compiler warnings due to static type checks. + However, when the limited C API is set to >=3.11, + these casts are not done, + and callers will need to cast arguments to their expected types. + See :pep:`670` for more details. + (Contributed by Victor Stinner and Erlend E. Aasland in :gh:`89653`.) + * :c:func:`PyErr_SetExcInfo()` no longer uses the ``type`` and ``traceback`` arguments, the interpreter now derives those values from the exception instance (the ``value`` argument). The function still steals references diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index b3d66d92332544..56abda199d947a 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -126,6 +126,12 @@ asyncio if supported and :class:`~asyncio.ThreadedChildWatcher` otherwise). (Contributed by Kumar Aditya in :gh:`94597`.) +* :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, + :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + pathlib ------- @@ -395,10 +401,6 @@ Removed (Contributed by Erlend E. Aasland in :gh:`92548`.) -* The ``--experimental-isolated-subinterpreters`` configure flag - (and corresponding ``EXPERIMENTAL_ISOLATED_SUBINTERPRETERS``) - have been removed. - * ``smtpd`` has been removed according to the schedule in :pep:`594`, having been deprecated in Python 3.4.7 and 3.5.4. Use aiosmtpd_ PyPI module or any other diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f646979910c887..811cfc147fcf6b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// The data structure & init here are inspired by Tools/scripts/deepfreeze.py. +// The data structure & init here are inspired by Tools/build/deepfreeze.py. // All field names generated by ASCII_STR() have a common prefix, // to help avoid collisions with keywords, etc. @@ -25,7 +25,7 @@ extern "C" { // XXX Order by frequency of use? -/* The following is auto-generated by Tools/scripts/generate_global_objects.py. */ +/* The following is auto-generated by Tools/build/generate_global_objects.py. */ struct _Py_global_strings { struct { STRUCT_FOR_STR(anon_dictcomp, "") diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 15925511cc1f41..c8ef5dd7352518 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -1,4 +1,4 @@ -// Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py +// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py #ifndef Py_INTERNAL_OPCODE_H #define Py_INTERNAL_OPCODE_H diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index bd1fedebd65cf5..8ce95884ccdd41 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -/* The following is auto-generated by Tools/scripts/generate_global_objects.py. */ +/* The following is auto-generated by Tools/build/generate_global_objects.py. */ #define _Py_global_objects_INIT { \ .singletons = { \ .small_ints = { \ diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index f9b8240e2168f2..95459ab9f7d004 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -1,4 +1,4 @@ -/* Auto-generated by Tools/scripts/generate_token.py */ +/* Auto-generated by Tools/build/generate_token.py */ /* Token types */ #ifndef Py_INTERNAL_TOKEN_H diff --git a/Include/opcode.h b/Include/opcode.h index 42825df6217b46..0871eb1bfe0a8b 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -1,4 +1,4 @@ -// Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py +// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py #ifndef Py_OPCODE_H #define Py_OPCODE_H diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 9f9b03e0cb21d3..2de7a1bfd68155 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -195,22 +195,25 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with events.get_child_watcher() as watcher: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = events.get_child_watcher() + + with watcher: if not watcher.is_active(): # Check early. # Raising exception before process creation # prevents subprocess execution if the watcher # is not ready to handle it. raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") + "subprocess support is not installed.") waiter = self.create_future() transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) + self._child_watcher_callback, transp) try: await waiter except (SystemExit, KeyboardInterrupt): @@ -1469,6 +1472,9 @@ def get_child_watcher(self): if self._watcher is None: self._init_watcher() + warnings._deprecated("get_child_watcher", + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", remove=(3, 14)) return self._watcher def set_child_watcher(self, watcher): @@ -1480,6 +1486,9 @@ def set_child_watcher(self, watcher): self._watcher.close() self._watcher = watcher + warnings._deprecated("set_child_watcher", + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", remove=(3, 14)) SelectorEventLoop = _UnixSelectorEventLoop diff --git a/Lib/gzip.py b/Lib/gzip.py index 8edcda4493c894..75c6ddc3f2cffb 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -21,6 +21,8 @@ _COMPRESS_LEVEL_TRADEOFF = 6 _COMPRESS_LEVEL_BEST = 9 +READ_BUFFER_SIZE = 128 * 1024 + def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, encoding=None, errors=None, newline=None): @@ -446,7 +448,7 @@ def _read_gzip_header(fp): class _GzipReader(_compression.DecompressReader): def __init__(self, fp): - super().__init__(_PaddedFile(fp), zlib.decompressobj, + super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor, wbits=-zlib.MAX_WBITS) # Set flag indicating start of a new member self._new_member = True @@ -494,12 +496,13 @@ def read(self, size=-1): self._new_member = False # Read a chunk of data from the file - buf = self._fp.read(io.DEFAULT_BUFFER_SIZE) + if self._decompressor.needs_input: + buf = self._fp.read(READ_BUFFER_SIZE) + uncompress = self._decompressor.decompress(buf, size) + else: + uncompress = self._decompressor.decompress(b"", size) - uncompress = self._decompressor.decompress(buf, size) - if self._decompressor.unconsumed_tail != b"": - self._fp.prepend(self._decompressor.unconsumed_tail) - elif self._decompressor.unused_data != b"": + if self._decompressor.unused_data != b"": # Prepend the already read bytes to the fileobj so they can # be seen by _read_eof() and _read_gzip_header() self._fp.prepend(self._decompressor.unused_data) @@ -510,14 +513,11 @@ def read(self, size=-1): raise EOFError("Compressed file ended before the " "end-of-stream marker was reached") - self._add_read_data( uncompress ) + self._crc = zlib.crc32(uncompress, self._crc) + self._stream_size += len(uncompress) self._pos += len(uncompress) return uncompress - def _add_read_data(self, data): - self._crc = zlib.crc32(data, self._crc) - self._stream_size = self._stream_size + len(data) - def _read_eof(self): # We've read to the end of the file # We check that the computed CRC and size of the @@ -647,7 +647,7 @@ def main(): f = builtins.open(arg, "rb") g = open(arg + ".gz", "wb") while True: - chunk = f.read(io.DEFAULT_BUFFER_SIZE) + chunk = f.read(READ_BUFFER_SIZE) if not chunk: break g.write(chunk) diff --git a/Lib/html/entities.py b/Lib/html/entities.py index cc59bc314499ad..eb6dc12190586d 100644 --- a/Lib/html/entities.py +++ b/Lib/html/entities.py @@ -261,7 +261,7 @@ # HTML5 named character references -# Generated by 'Tools/scripts/parse_html5_entities.py' +# Generated by Tools/build/parse_html5_entities.py # from https://html.spec.whatwg.org/entities.json and # https://html.spec.whatwg.org/multipage/named-characters.html. # Map HTML5 named character references to the equivalent Unicode character(s). diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 7fa7facf8cf780..e64e96f75e6cfc 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -4,6 +4,11 @@ Released on 2022-10-03 ========================= +gh-97527: Fix a bug in the previous bugfix that caused IDLE to not +start when run with 3.10.8, 3.12.0a1, and at least Microsoft Python +3.10.2288.0 installed without the Lib/test package. 3.11.0 was never +affected. + gh-65802: Document handling of extensions in Save As dialogs. gh-95191: Include prompts when saving Shell (interactive input/output). diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index ec4637c5ca617a..a38e43dcb9d1c4 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -39,7 +39,8 @@ def __eq__(self, other): data = (('1/0', ZeroDivisionError, "division by zero\n"), ('abc', NameError, "name 'abc' is not defined. " - "Did you mean: 'abs'?\n"), + "Did you mean: 'abs'? " + "Or did you forget to import 'abc'?\n"), ('int.reel', AttributeError, "type object 'int' has no attribute 'reel'. " "Did you mean: 'real'?\n"), diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index 12399f152aea2a..89b645702d334d 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -4,7 +4,6 @@ from os.path import expanduser import plistlib from sys import platform # Used in _init_tk_type, changed by test. -from test.support import requires, ResourceDenied import tkinter @@ -16,27 +15,38 @@ def _init_tk_type(): """ Initialize _tk_type for isXyzTk functions. + + This function is only called once, when _tk_type is still None. """ global _tk_type if platform == 'darwin': - try: - requires('gui') - except ResourceDenied: # Possible when testing. - _tk_type = "cocoa" # Newest and most common. - else: - root = tkinter.Tk() - ws = root.tk.call('tk', 'windowingsystem') - if 'x11' in ws: - _tk_type = "xquartz" - elif 'aqua' not in ws: - _tk_type = "other" - elif 'AppKit' in root.tk.call('winfo', 'server', '.'): + + # When running IDLE, GUI is present, test/* may not be. + # When running tests, test/* is present, GUI may not be. + # If not, guess most common. Does not matter for testing. + from idlelib.__init__ import testing + if testing: + from test.support import requires, ResourceDenied + try: + requires('gui') + except ResourceDenied: _tk_type = "cocoa" - else: - _tk_type = "carbon" - root.destroy() + return + + root = tkinter.Tk() + ws = root.tk.call('tk', 'windowingsystem') + if 'x11' in ws: + _tk_type = "xquartz" + elif 'aqua' not in ws: + _tk_type = "other" + elif 'AppKit' in root.tk.call('winfo', 'server', '.'): + _tk_type = "cocoa" + else: + _tk_type = "carbon" + root.destroy() else: _tk_type = "other" + return def isAquaTk(): """ diff --git a/Lib/imghdr.py b/Lib/imghdr.py index 6a372e66c7f261..33868883470764 100644 --- a/Lib/imghdr.py +++ b/Lib/imghdr.py @@ -14,6 +14,7 @@ #-------------------------# def what(file, h=None): + """Return the type of image contained in a file or byte stream.""" f = None try: if h is None: @@ -40,7 +41,7 @@ def what(file, h=None): tests = [] def test_jpeg(h, f): - """JPEG data with JFIF or Exif markers; and raw JPEG""" + """Test for JPEG data with JFIF or Exif markers; and raw JPEG.""" if h[6:10] in (b'JFIF', b'Exif'): return 'jpeg' elif h[:4] == b'\xff\xd8\xff\xdb': @@ -49,34 +50,35 @@ def test_jpeg(h, f): tests.append(test_jpeg) def test_png(h, f): + """Verify if the image is a PNG.""" if h.startswith(b'\211PNG\r\n\032\n'): return 'png' tests.append(test_png) def test_gif(h, f): - """GIF ('87 and '89 variants)""" + """Verify if the image is a GIF ('87 or '89 variants).""" if h[:6] in (b'GIF87a', b'GIF89a'): return 'gif' tests.append(test_gif) def test_tiff(h, f): - """TIFF (can be in Motorola or Intel byte order)""" + """Verify if the image is a TIFF (can be in Motorola or Intel byte order).""" if h[:2] in (b'MM', b'II'): return 'tiff' tests.append(test_tiff) def test_rgb(h, f): - """SGI image library""" + """test for the SGI image library.""" if h.startswith(b'\001\332'): return 'rgb' tests.append(test_rgb) def test_pbm(h, f): - """PBM (portable bitmap)""" + """Verify if the image is a PBM (portable bitmap).""" if len(h) >= 3 and \ h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': return 'pbm' @@ -84,7 +86,7 @@ def test_pbm(h, f): tests.append(test_pbm) def test_pgm(h, f): - """PGM (portable graymap)""" + """Verify if the image is a PGM (portable graymap).""" if len(h) >= 3 and \ h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': return 'pgm' @@ -92,7 +94,7 @@ def test_pgm(h, f): tests.append(test_pgm) def test_ppm(h, f): - """PPM (portable pixmap)""" + """Verify if the image is a PPM (portable pixmap).""" if len(h) >= 3 and \ h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': return 'ppm' @@ -100,32 +102,35 @@ def test_ppm(h, f): tests.append(test_ppm) def test_rast(h, f): - """Sun raster file""" + """test for the Sun raster file.""" if h.startswith(b'\x59\xA6\x6A\x95'): return 'rast' tests.append(test_rast) def test_xbm(h, f): - """X bitmap (X10 or X11)""" + """Verify if the image is a X bitmap (X10 or X11).""" if h.startswith(b'#define '): return 'xbm' tests.append(test_xbm) def test_bmp(h, f): + """Verify if the image is a BMP file.""" if h.startswith(b'BM'): return 'bmp' tests.append(test_bmp) def test_webp(h, f): + """Verify if the image is a WebP.""" if h.startswith(b'RIFF') and h[8:12] == b'WEBP': return 'webp' tests.append(test_webp) def test_exr(h, f): + """verify is the image ia a OpenEXR fileOpenEXR.""" if h.startswith(b'\x76\x2f\x31\x01'): return 'exr' diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index ca1fa8ab2fe6ff..8c0be7ee547e01 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -67,10 +67,14 @@ def from_package(package): @contextlib.contextmanager -def _tempfile(reader, suffix='', - # gh-93353: Keep a reference to call os.remove() in late Python - # finalization. - *, _os_remove=os.remove): +def _tempfile( + reader, + suffix='', + # gh-93353: Keep a reference to call os.remove() in late Python + # finalization. + *, + _os_remove=os.remove, +): # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' # blocks due to the need to close the temporary file to work on Windows # properly. @@ -89,13 +93,30 @@ def _tempfile(reader, suffix='', pass +def _temp_file(path): + return _tempfile(path.read_bytes, suffix=path.name) + + +def _is_present_dir(path: Traversable) -> bool: + """ + Some Traversables implement ``is_dir()`` to raise an + exception (i.e. ``FileNotFoundError``) when the + directory doesn't exist. This function wraps that call + to always return a boolean and only return True + if there's a dir and it exists. + """ + with contextlib.suppress(FileNotFoundError): + return path.is_dir() + return False + + @functools.singledispatch def as_file(path): """ Given a Traversable object, return that object as a path on the local file system in a context manager. """ - return _tempfile(path.read_bytes, suffix=path.name) + return _temp_dir(path) if _is_present_dir(path) else _temp_file(path) @as_file.register(pathlib.Path) @@ -105,3 +126,34 @@ def _(path): Degenerate behavior for pathlib.Path objects. """ yield path + + +@contextlib.contextmanager +def _temp_path(dir: tempfile.TemporaryDirectory): + """ + Wrap tempfile.TemporyDirectory to return a pathlib object. + """ + with dir as result: + yield pathlib.Path(result) + + +@contextlib.contextmanager +def _temp_dir(path): + """ + Given a traversable dir, recursively replicate the whole tree + to the file system in a context manager. + """ + assert path.is_dir() + with _temp_path(tempfile.TemporaryDirectory()) as temp_dir: + yield _write_contents(temp_dir, path) + + +def _write_contents(target, source): + child = target.joinpath(source.name) + if source.is_dir(): + child.mkdir() + for item in source.iterdir(): + _write_contents(child, item) + else: + child.open('wb').write(source.read_bytes()) + return child diff --git a/Lib/importlib/resources/abc.py b/Lib/importlib/resources/abc.py index 0b7bfdc415829e..67c78c07856705 100644 --- a/Lib/importlib/resources/abc.py +++ b/Lib/importlib/resources/abc.py @@ -1,6 +1,8 @@ import abc import io +import itertools import os +import pathlib from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional from typing import runtime_checkable, Protocol from typing import Union @@ -53,6 +55,10 @@ def contents(self) -> Iterable[str]: raise FileNotFoundError +class TraversalError(Exception): + pass + + @runtime_checkable class Traversable(Protocol): """ @@ -95,7 +101,6 @@ def is_file(self) -> bool: Return True if self is a file """ - @abc.abstractmethod def joinpath(self, *descendants: StrPath) -> "Traversable": """ Return Traversable resolved with any descendants applied. @@ -104,6 +109,22 @@ def joinpath(self, *descendants: StrPath) -> "Traversable": and each may contain multiple levels separated by ``posixpath.sep`` (``/``). """ + if not descendants: + return self + names = itertools.chain.from_iterable( + path.parts for path in map(pathlib.PurePosixPath, descendants) + ) + target = next(names) + matches = ( + traversable for traversable in self.iterdir() if traversable.name == target + ) + try: + match = next(matches) + except StopIteration: + raise TraversalError( + "Target not found during traversal.", target, list(names) + ) + return match.joinpath(*names) def __truediv__(self, child: StrPath) -> "Traversable": """ diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index b470a2062b2b3a..80cb320dd8bda0 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -82,15 +82,13 @@ def is_dir(self): def is_file(self): return False - def joinpath(self, child): - # first try to find child in current paths - for file in self.iterdir(): - if file.name == child: - return file - # if it does not exist, construct it with the first path - return self._paths[0] / child - - __truediv__ = joinpath + def joinpath(self, *descendants): + try: + return super().joinpath(*descendants) + except abc.TraversalError: + # One of the paths did not resolve (a directory does not exist). + # Just return something that will not exist. + return self._paths[0].joinpath(*descendants) def open(self, *args, **kwargs): raise FileNotFoundError(f'{self} is not a file') diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index d0fbf237762c2f..b85e4694acedff 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -99,20 +99,6 @@ def iterdir(self): def open(self, *args, **kwargs): raise IsADirectoryError() - @staticmethod - def _flatten(compound_names): - for name in compound_names: - yield from name.split('/') - - def joinpath(self, *descendants): - if not descendants: - return self - names = self._flatten(descendants) - target = next(names) - return next( - traversable for traversable in self.iterdir() if traversable.name == target - ).joinpath(*names) - class TraversableReader(TraversableResources, SimpleReader): """ diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index b4c5c1304b9752..9847104446eaf6 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -891,6 +891,13 @@ def _connect_unixsocket(self, address): raise def createSocket(self): + """ + Try to create a socket and, if it's not a datagram socket, connect it + to the other end. This method is called during handler initialization, + but it's not regarded as an error if the other end isn't listening yet + --- the method will be called again when emitting an event, + if there is no socket at that point. + """ address = self.address socktype = self.socktype @@ -898,7 +905,7 @@ def createSocket(self): self.unixsocket = True # Syslog server may be unavailable during handler initialisation. # C's openlog() function also ignores connection errors. - # Moreover, we ignore these errors while logging, so it not worse + # Moreover, we ignore these errors while logging, so it's not worse # to ignore it also here. try: self._connect_unixsocket(address) diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index 98a783448239a8..45def9ad16d32c 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py @@ -77,6 +77,7 @@ def whathdr(filename): tests = [] def test_aifc(h, f): + """AIFC and AIFF files""" with warnings.catch_warnings(): warnings.simplefilter('ignore', category=DeprecationWarning) import aifc @@ -100,6 +101,7 @@ def test_aifc(h, f): def test_au(h, f): + """AU and SND files""" if h.startswith(b'.snd'): func = get_long_be elif h[:4] in (b'\0ds.', b'dns.'): @@ -133,6 +135,7 @@ def test_au(h, f): def test_hcom(h, f): + """HCOM file""" if h[65:69] != b'FSSD' or h[128:132] != b'HCOM': return None divisor = get_long_be(h[144:148]) @@ -146,6 +149,7 @@ def test_hcom(h, f): def test_voc(h, f): + """VOC file""" if not h.startswith(b'Creative Voice File\032'): return None sbseek = get_short_le(h[20:22]) @@ -160,6 +164,7 @@ def test_voc(h, f): def test_wav(h, f): + """WAV file""" import wave # 'RIFF' 'WAVE' 'fmt ' if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': @@ -176,6 +181,7 @@ def test_wav(h, f): def test_8svx(h, f): + """8SVX file""" if not h.startswith(b'FORM') or h[8:12] != b'8SVX': return None # Should decode it to get #channels -- assume always 1 @@ -185,6 +191,7 @@ def test_8svx(h, f): def test_sndt(h, f): + """SNDT file""" if h.startswith(b'SOUND'): nsamples = get_long_le(h[8:12]) rate = get_short_le(h[20:22]) @@ -194,6 +201,7 @@ def test_sndt(h, f): def test_sndr(h, f): + """SNDR file""" if h.startswith(b'\0\0'): rate = get_short_le(h[2:4]) if 4000 <= rate <= 25000: diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index e998146c190df4..709cac7a27a449 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -505,6 +505,11 @@ def test_split(self): self.checkraises(ValueError, 'hello', 'split', '', 0) def test_rsplit(self): + # without arg + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit') + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit') + self.checkequal([], '', 'rsplit') + # by a char self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|') self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', 1) @@ -558,6 +563,9 @@ def test_rsplit(self): # with keyword args self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', sep='|') + self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', sep=None) + self.checkequal(['a b c', 'd'], + 'a b c d', 'rsplit', sep=None, maxsplit=1) self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', maxsplit=1) self.checkequal(['a|b|c', 'd'], diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 98b55dec37f478..cabe75f56d9fb0 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2058,11 +2058,13 @@ def setUp(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) + watcher.attach_loop(self.loop) + asyncio.set_child_watcher(watcher) def tearDown(self): - asyncio.set_child_watcher(None) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + asyncio.set_child_watcher(None) super().tearDown() @@ -2657,13 +2659,15 @@ def setUp(self): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) + watcher.attach_loop(self.loop) + asyncio.set_child_watcher(watcher) def tearDown(self): try: if sys.platform != 'win32': - asyncio.set_child_watcher(None) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + asyncio.set_child_watcher(None) super().tearDown() finally: diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index a10504b1c4130e..0198da21d77028 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -1,6 +1,7 @@ """Tests for sendfile functionality.""" import asyncio +import errno import os import socket import sys @@ -484,8 +485,17 @@ def sendfile_native(transp, file, offset, count): srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) with self.assertRaises(ConnectionError): - self.run_loop( - self.loop.sendfile(cli_proto.transport, self.file)) + try: + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + except OSError as e: + # macOS may raise OSError of EPROTOTYPE when writing to a + # socket that is in the process of closing down. + if e.errno == errno.EPROTOTYPE and sys.platform == "darwin": + raise ConnectionError + else: + raise + self.run_loop(srv_proto.done) self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 8fb9313e09dd0e..01d5407a497a04 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -797,7 +797,9 @@ def test_read_all_from_pipe_reader(self): watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) try: - asyncio.set_child_watcher(watcher) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + asyncio.set_child_watcher(watcher) create = asyncio.create_subprocess_exec( *args, pass_fds={wfd}, @@ -805,7 +807,9 @@ def test_read_all_from_pipe_reader(self): proc = self.loop.run_until_complete(create) self.loop.run_until_complete(proc.wait()) finally: - asyncio.set_child_watcher(None) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + asyncio.set_child_watcher(None) os.close(wfd) data = self.loop.run_until_complete(reader.read(-1)) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 64df1b17189b74..fe1d060c77d4c7 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -582,7 +582,9 @@ async def kill_running(): # manually to avoid a warning when the watcher is detached. if (sys.platform != 'win32' and isinstance(self, SubprocessFastWatcherTests)): - asyncio.get_child_watcher()._callbacks.clear() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + asyncio.get_child_watcher()._callbacks.clear() async def _test_popen_error(self, stdin): if sys.platform == 'win32': @@ -696,13 +698,17 @@ def setUp(self): watcher = self._get_watcher() watcher.attach_loop(self.loop) - policy.set_child_watcher(watcher) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + policy.set_child_watcher(watcher) def tearDown(self): super().tearDown() policy = asyncio.get_event_loop_policy() - watcher = policy.get_child_watcher() - policy.set_child_watcher(None) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = policy.get_child_watcher() + policy.set_child_watcher(None) watcher.attach_loop(None) watcher.close() @@ -746,10 +752,8 @@ def _get_watcher(self): class GenericWatcherTests(test_utils.TestCase): def test_create_subprocess_fails_with_inactive_watcher(self): - watcher = mock.create_autospec( - asyncio.AbstractChildWatcher, - **{"__enter__.return_value.is_active.return_value": False} - ) + watcher = mock.create_autospec(asyncio.AbstractChildWatcher) + watcher.is_active.return_value = False async def execute(): asyncio.set_child_watcher(watcher) @@ -761,12 +765,14 @@ async def execute(): watcher.add_child_handler.assert_not_called() with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - self.assertIsNone(runner.run(execute())) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.assertIsNone(runner.run(execute())) self.assertListEqual(watcher.mock_calls, [ mock.call.__enter__(), - mock.call.__enter__().is_active(), + mock.call.is_active(), mock.call.__exit__(RuntimeError, mock.ANY, mock.ANY), - ]) + ], watcher.mock_calls) @unittest.skipUnless( @@ -788,15 +794,16 @@ async def main(): with self.assertRaises(RuntimeError): asyncio.get_event_loop_policy().get_event_loop() return await asyncio.to_thread(asyncio.run, in_thread()) - - asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) + with self.assertWarns(DeprecationWarning): + asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) try: with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: returncode, stdout = runner.run(main()) self.assertEqual(returncode, 0) self.assertEqual(stdout, b'some data') finally: - asyncio.set_child_watcher(None) + with self.assertWarns(DeprecationWarning): + asyncio.set_child_watcher(None) else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 025da0f20ed47e..d806ed497aaab4 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1709,33 +1709,36 @@ def test_get_default_child_watcher(self): self.assertIsNone(policy._watcher) unix_events.can_use_pidfd = mock.Mock() unix_events.can_use_pidfd.return_value = False - watcher = policy.get_child_watcher() + with self.assertWarns(DeprecationWarning): + watcher = policy.get_child_watcher() self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) self.assertIs(policy._watcher, watcher) - - self.assertIs(watcher, policy.get_child_watcher()) + with self.assertWarns(DeprecationWarning): + self.assertIs(watcher, policy.get_child_watcher()) policy = self.create_policy() self.assertIsNone(policy._watcher) unix_events.can_use_pidfd = mock.Mock() unix_events.can_use_pidfd.return_value = True - watcher = policy.get_child_watcher() + with self.assertWarns(DeprecationWarning): + watcher = policy.get_child_watcher() self.assertIsInstance(watcher, asyncio.PidfdChildWatcher) self.assertIs(policy._watcher, watcher) - - self.assertIs(watcher, policy.get_child_watcher()) + with self.assertWarns(DeprecationWarning): + self.assertIs(watcher, policy.get_child_watcher()) def test_get_child_watcher_after_set(self): policy = self.create_policy() with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) watcher = asyncio.FastChildWatcher() + policy.set_child_watcher(watcher) - policy.set_child_watcher(watcher) self.assertIs(policy._watcher, watcher) - self.assertIs(watcher, policy.get_child_watcher()) + with self.assertWarns(DeprecationWarning): + self.assertIs(watcher, policy.get_child_watcher()) def test_get_child_watcher_thread(self): @@ -1769,7 +1772,7 @@ def test_child_watcher_replace_mainloop_existing(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) watcher = asyncio.SafeChildWatcher() - policy.set_child_watcher(watcher) + policy.set_child_watcher(watcher) watcher.attach_loop(loop) self.assertIs(watcher._loop, loop) diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 96be5a1c3bcf77..5b9c86eb9859a0 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -14,7 +14,7 @@ import threading import unittest import weakref - +import warnings from unittest import mock from http.server import HTTPServer @@ -544,7 +544,9 @@ def close_loop(loop): policy = support.maybe_get_event_loop_policy() if policy is not None: try: - watcher = policy.get_child_watcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = policy.get_child_watcher() except NotImplementedError: # watcher is not implemented by EventLoopPolicy, e.g. Windows pass diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index f3b3232c4866b3..64b92b0c52fa0c 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -880,6 +880,56 @@ def __init__(self): _testcapi.clear_managed_dict(c) self.assertEqual(c.__dict__, {}) + def test_eval_get_func_name(self): + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(_testcapi.eval_get_func_name(function_example), + "function_example") + self.assertEqual(_testcapi.eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(_testcapi.eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(_testcapi.eval_get_func_name(sum), "sum") # c function + self.assertEqual(_testcapi.eval_get_func_name(A), "type") + + def test_function_get_code(self): + import types + + def some(): + pass + + code = _testcapi.function_get_code(some) + self.assertIsInstance(code, types.CodeType) + self.assertEqual(code, some.__code__) + + with self.assertRaises(SystemError): + _testcapi.function_get_code(None) # not a function + + def test_function_get_globals(self): + def some(): + pass + + globals_ = _testcapi.function_get_globals(some) + self.assertIsInstance(globals_, dict) + self.assertEqual(globals_, some.__globals__) + + with self.assertRaises(SystemError): + _testcapi.function_get_globals(None) # not a function + + def test_function_get_module(self): + def some(): + pass + + module = _testcapi.function_get_module(some) + self.assertIsInstance(module, str) + self.assertEqual(module, some.__module__) + + with self.assertRaises(SystemError): + _testcapi.function_get_module(None) # not a function + class TestPendingCalls(unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 9d20c976b82505..4fd9e6bbe4281c 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -75,6 +75,11 @@ def test_join_path(self): str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), ) + self.assertEqual(path.joinpath(), path) + + def test_join_path_compound(self): + path = MultiplexedPath(self.folder) + assert not path.joinpath('imaginary/foo.py').exists() def test_repr(self): self.assertEqual( diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 1d6df0cc843173..f7e3abbdc805a7 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -111,6 +111,14 @@ def test_submodule_contents_by_name(self): {'__init__.py', 'binary.file'}, ) + def test_as_file_directory(self): + with resources.as_file(resources.files('ziptestdata')) as data: + assert data.name == 'ziptestdata' + assert data.is_dir() + assert data.joinpath('subdirectory').is_dir() + assert len(list(data.iterdir())) + assert not data.parent.exists() + class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): ZIP_MODULE = zipdata02 # type: ignore diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c07ac2a64c289f..0b6dcc5eaf03d2 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -298,7 +298,7 @@ def writes_bytecode_files(fxn): """Decorator to protect sys.dont_write_bytecode from mutation and to skip tests that require it to be set to False.""" if sys.dont_write_bytecode: - return lambda *args, **kwargs: None + return unittest.skip("relies on writing bytecode")(fxn) @functools.wraps(fxn) def wrapper(*args, **kwargs): original = sys.dont_write_bytecode diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 7f18edaaa8ca60..c083a04475e726 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -155,6 +155,26 @@ class C: ''')) self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + def test_moduledocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_moduledocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_moduledocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + def test_msgid(self): msgids = self.extract_docstrings_from_str( '''_("""doc""" r'str' u"ing")''') diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 4864b5c10b01bf..2d17e060065075 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3185,6 +3185,21 @@ def func(): actual = self.get_suggestion(func) self.assertNotIn("something", actual) + def test_name_error_for_stdlib_modules(self): + def func(): + stream = io.StringIO() + + actual = self.get_suggestion(func) + self.assertIn("forget to import 'io'", actual) + + def test_name_error_for_private_stdlib_modules(self): + def func(): + stream = _io.StringIO() + + actual = self.get_suggestion(func) + self.assertIn("forget to import '_io'", actual) + + class PurePythonSuggestionFormattingTests( PurePythonExceptionFormattingMixin, diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 15244cb949e3db..05e7e30d639a29 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -445,10 +445,10 @@ def test_split(self): def test_rsplit(self): string_tests.CommonTest.test_rsplit(self) # test mixed kinds - for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): + for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): left *= 9 right *= 9 - for delim in ('c', '\u0102', '\U00010302'): + for delim in ('c', 'ы', '\u0102', '\U00010302'): self.checkequal([left + right], left + right, 'rsplit', delim) self.checkequal([left, right], @@ -458,6 +458,10 @@ def test_rsplit(self): self.checkequal([left, right], left + delim * 2 + right, 'rsplit', delim *2) + # Check `None` as well: + self.checkequal([left + right], + left + right, 'rsplit', None) + def test_partition(self): string_tests.MixinStrUnicodeUserStringTest.test_partition(self) # test mixed kinds diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index f20aad051da960..ae54f6c46891c6 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -944,6 +944,173 @@ def choose_lines(source, number, seed=None, generator=random): """ +class ZlibDecompressorTest(): + # Test adopted from test_bz2.py + TEXT = HAMLET_SCENE + DATA = zlib.compress(HAMLET_SCENE) + BAD_DATA = b"Not a valid deflate block" + def test_Constructor(self): + self.assertRaises(TypeError, zlib._ZlibDecompressor, 42) + + def testDecompress(self): + zlibd = zlib._ZlibDecompressor() + self.assertRaises(TypeError, zlibd.decompress) + text = zlibd.decompress(self.DATA) + self.assertEqual(text, self.TEXT) + + def testDecompressChunks10(self): + zlibd = zlib._ZlibDecompressor() + text = b'' + n = 0 + while True: + str = self.DATA[n*10:(n+1)*10] + if not str: + break + text += zlibd.decompress(str) + n += 1 + self.assertEqual(text, self.TEXT) + + def testDecompressUnusedData(self): + zlibd = zlib._ZlibDecompressor() + unused_data = b"this is unused data" + text = zlibd.decompress(self.DATA+unused_data) + self.assertEqual(text, self.TEXT) + self.assertEqual(zlibd.unused_data, unused_data) + + def testEOFError(self): + zlibd = zlib._ZlibDecompressor() + text = zlibd.decompress(self.DATA) + self.assertRaises(EOFError, zlibd.decompress, b"anything") + self.assertRaises(EOFError, zlibd.decompress, b"") + + @support.skip_if_pgo_task + @bigmemtest(size=_4G + 100, memuse=3.3) + def testDecompress4G(self, size): + # "Test zlib._ZlibDecompressor.decompress() with >4GiB input" + blocksize = 10 * 1024 * 1024 + block = random.randbytes(blocksize) + try: + data = block * (size // blocksize + 1) + compressed = zlib.compress(data) + zlibd = zlib._ZlibDecompressor() + decompressed = zlibd.decompress(compressed) + self.assertTrue(decompressed == data) + finally: + data = None + compressed = None + decompressed = None + + def testPickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(zlib._ZlibDecompressor(), proto) + + def testDecompressorChunksMaxsize(self): + zlibd = zlib._ZlibDecompressor() + max_length = 100 + out = [] + + # Feed some input + len_ = len(self.BIG_DATA) - 64 + out.append(zlibd.decompress(self.BIG_DATA[:len_], + max_length=max_length)) + self.assertFalse(zlibd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data without providing more input + out.append(zlibd.decompress(b'', max_length=max_length)) + self.assertFalse(zlibd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data while providing more input + out.append(zlibd.decompress(self.BIG_DATA[len_:], + max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + # Retrieve remaining uncompressed data + while not zlibd.eof: + out.append(zlibd.decompress(b'', max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + out = b"".join(out) + self.assertEqual(out, self.BIG_TEXT) + self.assertEqual(zlibd.unused_data, b"") + + def test_decompressor_inputbuf_1(self): + # Test reusing input buffer after moving existing + # contents to beginning + zlibd = zlib._ZlibDecompressor() + out = [] + + # Create input buffer and fill it + self.assertEqual(zlibd.decompress(self.DATA[:100], + max_length=0), b'') + + # Retrieve some results, freeing capacity at beginning + # of input buffer + out.append(zlibd.decompress(b'', 2)) + + # Add more data that fits into input buffer after + # moving existing data to beginning + out.append(zlibd.decompress(self.DATA[100:105], 15)) + + # Decompress rest of data + out.append(zlibd.decompress(self.DATA[105:])) + self.assertEqual(b''.join(out), self.TEXT) + + def test_decompressor_inputbuf_2(self): + # Test reusing input buffer by appending data at the + # end right away + zlibd = zlib._ZlibDecompressor() + out = [] + + # Create input buffer and empty it + self.assertEqual(zlibd.decompress(self.DATA[:200], + max_length=0), b'') + out.append(zlibd.decompress(b'')) + + # Fill buffer with new data + out.append(zlibd.decompress(self.DATA[200:280], 2)) + + # Append some more data, not enough to require resize + out.append(zlibd.decompress(self.DATA[280:300], 2)) + + # Decompress rest of data + out.append(zlibd.decompress(self.DATA[300:])) + self.assertEqual(b''.join(out), self.TEXT) + + def test_decompressor_inputbuf_3(self): + # Test reusing input buffer after extending it + + zlibd = zlib._ZlibDecompressor() + out = [] + + # Create almost full input buffer + out.append(zlibd.decompress(self.DATA[:200], 5)) + + # Add even more data to it, requiring resize + out.append(zlibd.decompress(self.DATA[200:300], 5)) + + # Decompress rest of data + out.append(zlibd.decompress(self.DATA[300:])) + self.assertEqual(b''.join(out), self.TEXT) + + def test_failure(self): + zlibd = zlib._ZlibDecompressor() + self.assertRaises(Exception, zlibd.decompress, self.BAD_DATA * 30) + # Previously, a second call could crash due to internal inconsistency + self.assertRaises(Exception, zlibd.decompress, self.BAD_DATA * 30) + + @support.refcount_test + def test_refleaks_in___init__(self): + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + zlibd = zlib._ZlibDecompressor() + refs_before = gettotalrefcount() + for i in range(100): + zlibd.__init__() + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + class CustomInt: def __index__(self): return 100 diff --git a/Lib/token.py b/Lib/token.py index 9d0c0bf0fb0368..95b107c6643b3f 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,5 +1,5 @@ """Token constants.""" -# Auto-generated by Tools/scripts/generate_token.py +# Auto-generated by Tools/build/generate_token.py __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] diff --git a/Lib/traceback.py b/Lib/traceback.py index c46ddaf51a00d4..bb7856a5142e03 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -712,6 +712,13 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, suggestion = _compute_suggestion_error(exc_value, exc_traceback) if suggestion: self._str += f". Did you mean: '{suggestion}'?" + if issubclass(exc_type, NameError): + wrong_name = getattr(exc_value, "name", None) + if wrong_name is not None and wrong_name in sys.stdlib_module_names: + if suggestion: + self._str += f" Or did you forget to import '{wrong_name}'" + else: + self._str += f". Did you forget to import '{wrong_name}'" if lookup_lines: self._load_lines() self.__suppress_context__ = \ diff --git a/Makefile.pre.in b/Makefile.pre.in index 7e2567173bc2ce..5b4bf15eb8aa58 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -285,7 +285,7 @@ BUILDPYTHON= python$(BUILDEXE) HOSTRUNNER= @HOSTRUNNER@ PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ -UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py +UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ # Single-platform builds depend on $(BUILDPYTHON). Cross builds use an # external "build Python" and have an empty PYTHON_FOR_BUILD_DEPS. @@ -705,7 +705,7 @@ coverage-report: regen-token regen-frozen .PHONY=clinic clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) @@ -907,7 +907,7 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt # dependency on BUILDPYTHON ensures that the target is run last checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) - @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/scripts/check_extension_modules.py + @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py rundsymutil: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) @if [ ! -z $(DSYMUTIL) ] ; then \ @@ -961,13 +961,13 @@ regen-test-frozenmain: $(BUILDPYTHON) .PHONY: regen-test-levenshtein regen-test-levenshtein: # Regenerate Lib/test/levenshtein_examples.json - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_levenshtein_examples.py Lib/test/levenshtein_examples.json + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_levenshtein_examples.py Lib/test/levenshtein_examples.json .PHONY: regen-re regen-re: $(BUILDPYTHON) # Regenerate Lib/re/_casefix.py - # using Tools/scripts/generate_re_casefix.py - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py + # using Tools/build/generate_re_casefix.py + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) @@ -1013,7 +1013,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # 2) deepfreeze modules with external build Python. # -# FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py. +# FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py. FROZEN_FILES_IN = \ Lib/importlib/_bootstrap.py \ Lib/importlib/_bootstrap_external.py \ @@ -1149,11 +1149,11 @@ Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS) # END: freezing modules -Tools/scripts/freeze_modules.py: $(FREEZE_MODULE) +Tools/build/freeze_modules.py: $(FREEZE_MODULE) .PHONY: regen-frozen -regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py +regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py @echo "The Makefile was updated, you may need to re-run make." ############################################################################ @@ -1162,11 +1162,11 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN) .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) +DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) # BEGIN: deepfreeze modules Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \ + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ Python/frozen_modules/zipimport.h:zipimport \ @@ -1203,8 +1203,8 @@ regen-importlib: regen-frozen # Global objects .PHONY: regen-global-objects -regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py +regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py @echo "Note: Global objects can be added or removed by other tools (e.g. deepfreeze), " @echo " so be sure to re-run regen-global-objects after those tools." @@ -1220,7 +1220,7 @@ check-abidump: all abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms regen-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml ############################################################################ # Regenerate all generated files @@ -1331,8 +1331,8 @@ regen-ast: .PHONY: regen-opcode regen-opcode: # Regenerate Include/opcode.h from Lib/opcode.py - # using Tools/scripts/generate_opcode_h.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \ + # using Tools/build/generate_opcode_h.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ $(srcdir)/Include/opcode.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new @@ -1342,23 +1342,23 @@ regen-opcode: .PHONY: regen-token regen-token: # Regenerate Doc/library/token-list.inc from Grammar/Tokens - # using Tools/scripts/generate_token.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \ + # using Tools/build/generate_token.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py rst \ $(srcdir)/Grammar/Tokens \ $(srcdir)/Doc/library/token-list.inc # Regenerate Include/internal/pycore_token.h from Grammar/Tokens - # using Tools/scripts/generate_token.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \ + # using Tools/build/generate_token.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py h \ $(srcdir)/Grammar/Tokens \ $(srcdir)/Include/internal/pycore_token.h # Regenerate Parser/token.c from Grammar/Tokens - # using Tools/scripts/generate_token.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \ + # using Tools/build/generate_token.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py c \ $(srcdir)/Grammar/Tokens \ $(srcdir)/Parser/token.c # Regenerate Lib/token.py from Grammar/Tokens - # using Tools/scripts/generate_token.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py py \ + # using Tools/build/generate_token.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py py \ $(srcdir)/Grammar/Tokens \ $(srcdir)/Lib/token.py @@ -1375,16 +1375,16 @@ regen-keyword: .PHONY: regen-stdlib-module-names regen-stdlib-module-names: all Programs/_testembed # Regenerate Python/stdlib_module_names.h - # using Tools/scripts/generate_stdlib_module_names.py + # using Tools/build/generate_stdlib_module_names.py $(RUNSHARED) ./$(BUILDPYTHON) \ - $(srcdir)/Tools/scripts/generate_stdlib_module_names.py \ + $(srcdir)/Tools/build/generate_stdlib_module_names.py \ > $(srcdir)/Python/stdlib_module_names.h.new $(UPDATE_FILE) $(srcdir)/Python/stdlib_module_names.h $(srcdir)/Python/stdlib_module_names.h.new regen-sre: # Regenerate Modules/_sre/sre_constants.h and Modules/_sre/sre_targets.h - # from Lib/re/_constants.py using Tools/scripts/generate_sre_constants.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_sre_constants.py \ + # from Lib/re/_constants.py using Tools/build/generate_sre_constants.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_sre_constants.py \ $(srcdir)/Lib/re/_constants.py \ $(srcdir)/Modules/_sre/sre_constants.h \ $(srcdir)/Modules/_sre/sre_targets.h @@ -2511,7 +2511,7 @@ distclean: clobber docclean # Check that all symbols exported by libpython start with "Py" or "_Py" smelly: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/smelly.py + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/smelly.py # Find files with funny names funny: @@ -2549,7 +2549,7 @@ patchcheck: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/patchcheck/patchcheck.py check-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml .PHONY: update-config update-config: diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-13-23-23-01.gh-issue-98254.bC8IKt.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-13-23-23-01.gh-issue-98254.bC8IKt.rst new file mode 100644 index 00000000000000..af5d93ff24e96a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-13-23-23-01.gh-issue-98254.bC8IKt.rst @@ -0,0 +1,3 @@ +Modules from the standard library are now potentially suggested as part of the +error messages displayed by the interpreter when an :exc:`NameError` is raised +to the top level. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst b/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst new file mode 100644 index 00000000000000..292e16998acdc0 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-10-16-17-34-45.gh-issue-85525.DvkD0v.rst @@ -0,0 +1 @@ +Remove extra row diff --git a/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst b/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst new file mode 100644 index 00000000000000..e7fda89741942f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2022-10-15-21-20-40.gh-issue-97527.otAHJM.rst @@ -0,0 +1,3 @@ +Fix a bug in the previous bugfix that caused IDLE to not start when run with +3.10.8, 3.12.0a1, and at least Microsoft Python 3.10.2288.0 installed +without the Lib/test package. 3.11.0 was never affected. diff --git a/Misc/NEWS.d/next/Library/2022-09-30-09-22-37.gh-issue-95534.ndEfPj.rst b/Misc/NEWS.d/next/Library/2022-09-30-09-22-37.gh-issue-95534.ndEfPj.rst new file mode 100644 index 00000000000000..131d4ed69991a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-09-30-09-22-37.gh-issue-95534.ndEfPj.rst @@ -0,0 +1 @@ +:meth:`gzip.GzipFile.read` reads 10% faster. diff --git a/Misc/NEWS.d/next/Library/2022-10-05-16-10-24.gh-issue-97930.NPSrzE.rst b/Misc/NEWS.d/next/Library/2022-10-05-16-10-24.gh-issue-97930.NPSrzE.rst new file mode 100644 index 00000000000000..860f6adeb201f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-05-16-10-24.gh-issue-97930.NPSrzE.rst @@ -0,0 +1,3 @@ +Apply changes from importlib_resources 5.8 and 5.9: ``Traversable.joinpath`` +provides a concrete implementation. ``as_file`` now supports directories of +resources. diff --git a/Misc/NEWS.d/next/Library/2022-10-12-11-20-54.gh-issue-94597.GYJZlb.rst b/Misc/NEWS.d/next/Library/2022-10-12-11-20-54.gh-issue-94597.GYJZlb.rst new file mode 100644 index 00000000000000..5ea1358b7d8a35 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-12-11-20-54.gh-issue-94597.GYJZlb.rst @@ -0,0 +1 @@ +Deprecated :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` and :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` methods to be removed in Python 3.14. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst b/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst new file mode 100644 index 00000000000000..3fe41d53c980b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-16-06-18-59.gh-issue-98307.b2_CDu.rst @@ -0,0 +1,2 @@ +A :meth:`~logging.handlers.SysLogHandler.createSocket` method was added to +:class:`~logging.handlers.SysLogHandler`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst new file mode 100644 index 00000000000000..6b214616c0a920 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-05-23-25-59.gh-issue-95731.N2KohU.rst @@ -0,0 +1 @@ +Fix handling of module docstrings in :file:`Tools/i18n/pygettext.py`. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index a8920d999e818f..e78646fdea59cf 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2,7 +2,7 @@ # Please append new items at the end. # The syntax of this file is not fixed. -# It is designed to be read only by Tools/stable_abi.py, which can change +# It is designed to be read only by Tools/build/stable_abi.py, which can change # without notice. # For the history of the stable ABI prior to this file, diff --git a/Modules/_sre/sre_constants.h b/Modules/_sre/sre_constants.h index c6335147368626..f030815c6c008d 100644 --- a/Modules/_sre/sre_constants.h +++ b/Modules/_sre/sre_constants.h @@ -3,7 +3,7 @@ * * regular expression matching engine * - * Auto-generated by Tools/scripts/generate_sre_constants.py from + * Auto-generated by Tools/build/generate_sre_constants.py from * Lib/re/_constants.py. * * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. diff --git a/Modules/_sre/sre_targets.h b/Modules/_sre/sre_targets.h index 25b6edd436bb73..62761a0000d836 100644 --- a/Modules/_sre/sre_targets.h +++ b/Modules/_sre/sre_targets.h @@ -3,7 +3,7 @@ * * regular expression matching engine * - * Auto-generated by Tools/scripts/generate_sre_constants.py from + * Auto-generated by Tools/build/generate_sre_constants.py from * Lib/re/_constants.py. * * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b3db00dba03d46..194b2de2f0d6e2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5469,6 +5469,12 @@ frame_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(lasti); } +static PyObject * +eval_get_func_name(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncName(func)); +} + static PyObject * get_feature_macros(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -5652,6 +5658,42 @@ test_macros(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +static PyObject * +function_get_code(PyObject *self, PyObject *func) +{ + PyObject *code = PyFunction_GetCode(func); + if (code != NULL) { + Py_INCREF(code); + return code; + } else { + return NULL; + } +} + +static PyObject * +function_get_globals(PyObject *self, PyObject *func) +{ + PyObject *globals = PyFunction_GetGlobals(func); + if (globals != NULL) { + Py_INCREF(globals); + return globals; + } else { + return NULL; + } +} + +static PyObject * +function_get_module(PyObject *self, PyObject *func) +{ + PyObject *module = PyFunction_GetModule(func); + if (module != NULL) { + Py_INCREF(module); + return module; + } else { + return NULL; + } +} + // type watchers @@ -6047,6 +6089,7 @@ static PyMethodDef TestMethods[] = { {"frame_getgenerator", frame_getgenerator, METH_O, NULL}, {"frame_getbuiltins", frame_getbuiltins, METH_O, NULL}, {"frame_getlasti", frame_getlasti, METH_O, NULL}, + {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, {"settrace_to_record", settrace_to_record, METH_O, NULL}, @@ -6057,6 +6100,9 @@ static PyMethodDef TestMethods[] = { {"watch_dict", watch_dict, METH_VARARGS, NULL}, {"unwatch_dict", unwatch_dict, METH_VARARGS, NULL}, {"get_dict_watcher_events", get_dict_watcher_events, METH_NOARGS, NULL}, + {"function_get_code", function_get_code, METH_O, NULL}, + {"function_get_globals", function_get_globals, METH_O, NULL}, + {"function_get_module", function_get_module, METH_O, NULL}, {"add_type_watcher", add_type_watcher, METH_O, NULL}, {"clear_type_watcher", clear_type_watcher, METH_O, NULL}, {"watch_type", watch_type, METH_VARARGS, NULL}, diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index a04b954a57f353..65412b2435ade1 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -897,6 +897,104 @@ zlib_Decompress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args return return_value; } +PyDoc_STRVAR(zlib_ZlibDecompressor_decompress__doc__, +"decompress($self, /, data, max_length=-1)\n" +"--\n" +"\n" +"Decompress *data*, returning uncompressed data as bytes.\n" +"\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" +"decompressed data. If this limit is reached and further output can be\n" +"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" +"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"\n" +"If all of the input data was decompressed and returned (either because this\n" +"was less than *max_length* bytes, or because *max_length* was negative),\n" +"*self.needs_input* will be set to True.\n" +"\n" +"Attempting to decompress data after the end of stream is reached raises an\n" +"EOFError. Any data found after the end of the stream is ignored and saved in\n" +"the unused_data attribute."); + +#define ZLIB_ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ + {"decompress", _PyCFunction_CAST(zlib_ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib_ZlibDecompressor_decompress__doc__}, + +static PyObject * +zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, + Py_buffer *data, Py_ssize_t max_length); + +static PyObject * +zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(data), &_Py_ID(max_length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"data", "max_length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "decompress", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_buffer data = {NULL, NULL}; + Py_ssize_t max_length = -1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!PyBuffer_IsContiguous(&data, 'C')) { + _PyArg_BadArgument("decompress", "argument 'data'", "contiguous buffer", args[0]); + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + max_length = ival; + } +skip_optional_pos: + return_value = zlib_ZlibDecompressor_decompress_impl(self, &data, max_length); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + PyDoc_STRVAR(zlib_adler32__doc__, "adler32($module, data, value=1, /)\n" "--\n" @@ -1031,4 +1129,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=9e5f9911d0c273e1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=57ff7b511ab23132 input=a9049054013a1b77]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 2fc39a3bd56201..2a490ed5d183a7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -8,6 +8,11 @@ #include "Python.h" #include "structmember.h" // PyMemberDef #include "zlib.h" +#include "stdbool.h" + +#if defined(ZLIB_VERNUM) && ZLIB_VERNUM < 0x1221 +#error "At least zlib version 1.2.2.1 is required" +#endif // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" @@ -171,9 +176,6 @@ OutputBuffer_WindowOnError(_BlocksOutputBuffer *buffer, _Uint32Window *window) } } while (0) #define LEAVE_ZLIB(obj) PyThread_release_lock((obj)->lock); -#if defined(ZLIB_VERNUM) && ZLIB_VERNUM >= 0x1221 -# define AT_LEAST_ZLIB_1_2_2_1 -#endif /* The following parameters are copied from zutil.h, version 0.95 */ #define DEFLATED 8 @@ -185,12 +187,14 @@ OutputBuffer_WindowOnError(_BlocksOutputBuffer *buffer, _Uint32Window *window) /* Initial buffer size. */ #define DEF_BUF_SIZE (16*1024) +#define DEF_MAX_INITIAL_BUF_SIZE (16 * 1024 * 1024) static PyModuleDef zlibmodule; typedef struct { PyTypeObject *Comptype; PyTypeObject *Decomptype; + PyTypeObject *ZlibDecompressorType; PyObject *ZlibError; } zlibstate; @@ -209,7 +213,7 @@ typedef struct PyObject *unused_data; PyObject *unconsumed_tail; char eof; - int is_initialised; + bool is_initialised; PyObject *zdict; PyThread_type_lock lock; } compobject; @@ -320,7 +324,7 @@ static PyObject * zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits) /*[clinic end generated code: output=46bd152fadd66df2 input=c4d06ee5782a7e3f]*/ { - PyObject *RetVal; + PyObject *return_value; int flush; z_stream zst; _BlocksOutputBuffer buffer = {.list = NULL}; @@ -387,11 +391,11 @@ zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits) err = deflateEnd(&zst); if (err == Z_OK) { - RetVal = OutputBuffer_Finish(&buffer, zst.avail_out); - if (RetVal == NULL) { + return_value = OutputBuffer_Finish(&buffer, zst.avail_out); + if (return_value == NULL) { goto error; } - return RetVal; + return return_value; } else zlib_error(state, zst, err, "while finishing compression"); @@ -419,7 +423,7 @@ zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, Py_ssize_t bufsize) /*[clinic end generated code: output=77c7e35111dc8c42 input=a9ac17beff1f893f]*/ { - PyObject *RetVal; + PyObject *return_value; Byte *ibuf; Py_ssize_t ibuflen; int err, flush; @@ -514,9 +518,9 @@ zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, goto error; } - RetVal = OutputBuffer_WindowFinish(&buffer, &window, zst.avail_out); - if (RetVal != NULL) { - return RetVal; + return_value = OutputBuffer_WindowFinish(&buffer, &window, zst.avail_out); + if (return_value != NULL) { + return return_value; } error: @@ -675,18 +679,10 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) case Z_OK: self->is_initialised = 1; if (self->zdict != NULL && wbits < 0) { -#ifdef AT_LEAST_ZLIB_1_2_2_1 if (set_inflate_zdict(state, self) < 0) { Py_DECREF(self); return NULL; } -#else - PyErr_Format(state->ZlibError, - "zlib version %s does not allow raw inflate with dictionary", - ZLIB_VERSION); - Py_DECREF(self); - return NULL; -#endif } return (PyObject *)self; case Z_STREAM_ERROR: @@ -753,7 +749,7 @@ zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data) /*[clinic end generated code: output=6731b3f0ff357ca6 input=04d00f65ab01d260]*/ { - PyObject *RetVal; + PyObject *return_value; int err; _BlocksOutputBuffer buffer = {.list = NULL}; zlibstate *state = PyType_GetModuleState(cls); @@ -791,17 +787,17 @@ zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, } while (ibuflen != 0); - RetVal = OutputBuffer_Finish(&buffer, self->zst.avail_out); - if (RetVal != NULL) { + return_value = OutputBuffer_Finish(&buffer, self->zst.avail_out); + if (return_value != NULL) { goto success; } error: OutputBuffer_OnError(&buffer); - RetVal = NULL; + return_value = NULL; success: LEAVE_ZLIB(self); - return RetVal; + return return_value; } /* Helper for objdecompress() and flush(). Saves any unconsumed input data in @@ -875,7 +871,7 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, { int err = Z_OK; Py_ssize_t ibuflen; - PyObject *RetVal; + PyObject *return_value; _BlocksOutputBuffer buffer = {.list = NULL}; PyObject *module = PyType_GetModule(cls); @@ -953,17 +949,17 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, goto abort; } - RetVal = OutputBuffer_Finish(&buffer, self->zst.avail_out); - if (RetVal != NULL) { + return_value = OutputBuffer_Finish(&buffer, self->zst.avail_out); + if (return_value != NULL) { goto success; } abort: OutputBuffer_OnError(&buffer); - RetVal = NULL; + return_value = NULL; success: LEAVE_ZLIB(self); - return RetVal; + return return_value; } /*[clinic input] @@ -985,7 +981,7 @@ zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode) /*[clinic end generated code: output=c7efd13efd62add2 input=286146e29442eb6c]*/ { int err; - PyObject *RetVal; + PyObject *return_value; _BlocksOutputBuffer buffer = {.list = NULL}; zlibstate *state = PyType_GetModuleState(cls); @@ -1042,17 +1038,17 @@ zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode) goto error; } - RetVal = OutputBuffer_Finish(&buffer, self->zst.avail_out); - if (RetVal != NULL) { + return_value = OutputBuffer_Finish(&buffer, self->zst.avail_out); + if (return_value != NULL) { goto success; } error: OutputBuffer_OnError(&buffer); - RetVal = NULL; + return_value = NULL; success: LEAVE_ZLIB(self); - return RetVal; + return return_value; } #ifdef HAVE_ZLIB_COPY @@ -1071,14 +1067,14 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls) { zlibstate *state = PyType_GetModuleState(cls); - compobject *retval = newcompobject(state->Comptype); - if (!retval) return NULL; + compobject *return_value = newcompobject(state->Comptype); + if (!return_value) return NULL; /* Copy the zstream state * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe */ ENTER_ZLIB(self); - int err = deflateCopy(&retval->zst, &self->zst); + int err = deflateCopy(&return_value->zst, &self->zst); switch (err) { case Z_OK: break; @@ -1094,22 +1090,22 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls) goto error; } Py_INCREF(self->unused_data); - Py_XSETREF(retval->unused_data, self->unused_data); + Py_XSETREF(return_value->unused_data, self->unused_data); Py_INCREF(self->unconsumed_tail); - Py_XSETREF(retval->unconsumed_tail, self->unconsumed_tail); + Py_XSETREF(return_value->unconsumed_tail, self->unconsumed_tail); Py_XINCREF(self->zdict); - Py_XSETREF(retval->zdict, self->zdict); - retval->eof = self->eof; + Py_XSETREF(return_value->zdict, self->zdict); + return_value->eof = self->eof; /* Mark it as being initialized */ - retval->is_initialised = 1; + return_value->is_initialised = 1; LEAVE_ZLIB(self); - return (PyObject *)retval; + return (PyObject *)return_value; error: LEAVE_ZLIB(self); - Py_XDECREF(retval); + Py_XDECREF(return_value); return NULL; } @@ -1158,14 +1154,14 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls) { zlibstate *state = PyType_GetModuleState(cls); - compobject *retval = newcompobject(state->Decomptype); - if (!retval) return NULL; + compobject *return_value = newcompobject(state->Decomptype); + if (!return_value) return NULL; /* Copy the zstream state * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe */ ENTER_ZLIB(self); - int err = inflateCopy(&retval->zst, &self->zst); + int err = inflateCopy(&return_value->zst, &self->zst); switch (err) { case Z_OK: break; @@ -1182,22 +1178,22 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls) } Py_INCREF(self->unused_data); - Py_XSETREF(retval->unused_data, self->unused_data); + Py_XSETREF(return_value->unused_data, self->unused_data); Py_INCREF(self->unconsumed_tail); - Py_XSETREF(retval->unconsumed_tail, self->unconsumed_tail); + Py_XSETREF(return_value->unconsumed_tail, self->unconsumed_tail); Py_XINCREF(self->zdict); - Py_XSETREF(retval->zdict, self->zdict); - retval->eof = self->eof; + Py_XSETREF(return_value->zdict, self->zdict); + return_value->eof = self->eof; /* Mark it as being initialized */ - retval->is_initialised = 1; + return_value->is_initialised = 1; LEAVE_ZLIB(self); - return (PyObject *)retval; + return (PyObject *)return_value; error: LEAVE_ZLIB(self); - Py_XDECREF(retval); + Py_XDECREF(return_value); return NULL; } @@ -1252,7 +1248,7 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, { int err, flush; Py_buffer data; - PyObject *RetVal; + PyObject *return_value; Py_ssize_t ibuflen; _BlocksOutputBuffer buffer = {.list = NULL}; _Uint32Window window; // output buffer's UINT32_MAX sliding window @@ -1306,13 +1302,6 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, case Z_STREAM_END: break; default: - if (err == Z_NEED_DICT && self->zdict != NULL) { - if (set_inflate_zdict(state, self) < 0) { - goto abort; - } - else - break; - } goto save; } @@ -1336,18 +1325,459 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, } } - RetVal = OutputBuffer_WindowFinish(&buffer, &window, self->zst.avail_out); - if (RetVal != NULL) { + return_value = OutputBuffer_WindowFinish(&buffer, &window, self->zst.avail_out); + if (return_value != NULL) { goto success; } abort: OutputBuffer_WindowOnError(&buffer, &window); - RetVal = NULL; + return_value = NULL; success: PyBuffer_Release(&data); LEAVE_ZLIB(self); - return RetVal; + return return_value; +} + + +typedef struct { + PyObject_HEAD + z_stream zst; + PyObject *zdict; + PyThread_type_lock lock; + PyObject *unused_data; + uint8_t *input_buffer; + Py_ssize_t input_buffer_size; + /* zst>avail_in is only 32 bit, so we store the true length + separately. Conversion and looping is encapsulated in + decompress_buf() */ + Py_ssize_t avail_in_real; + bool is_initialised; + char eof; /* T_BOOL expects a char */ + char needs_input; +} ZlibDecompressor; + +/*[clinic input] +class zlib.ZlibDecompressor "ZlibDecompressor *" "&ZlibDecompressorType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=0658178ab94645df]*/ + +static void +ZlibDecompressor_dealloc(ZlibDecompressor *self) +{ + PyObject *type = (PyObject *)Py_TYPE(self); + PyThread_free_lock(self->lock); + if (self->is_initialised) { + inflateEnd(&self->zst); + } + PyMem_Free(self->input_buffer); + Py_CLEAR(self->unused_data); + Py_CLEAR(self->zdict); + PyObject_Free(self); + Py_DECREF(type); +} + +static int +set_inflate_zdict_ZlibDecompressor(zlibstate *state, ZlibDecompressor *self) +{ + Py_buffer zdict_buf; + if (PyObject_GetBuffer(self->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) { + return -1; + } + if ((size_t)zdict_buf.len > UINT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "zdict length does not fit in an unsigned int"); + PyBuffer_Release(&zdict_buf); + return -1; + } + int err; + err = inflateSetDictionary(&self->zst, + zdict_buf.buf, (unsigned int)zdict_buf.len); + PyBuffer_Release(&zdict_buf); + if (err != Z_OK) { + zlib_error(state, self->zst, err, "while setting zdict"); + return -1; + } + return 0; +} + +static Py_ssize_t +arrange_output_buffer_with_maximum(uint32_t *avail_out, + uint8_t **next_out, + PyObject **buffer, + Py_ssize_t length, + Py_ssize_t max_length) +{ + Py_ssize_t occupied; + + if (*buffer == NULL) { + if (!(*buffer = PyBytes_FromStringAndSize(NULL, length))) + return -1; + occupied = 0; + } + else { + occupied = *next_out - (uint8_t *)PyBytes_AS_STRING(*buffer); + + if (length == occupied) { + Py_ssize_t new_length; + assert(length <= max_length); + /* can not scale the buffer over max_length */ + if (length == max_length) + return -2; + if (length <= (max_length >> 1)) + new_length = length << 1; + else + new_length = max_length; + if (_PyBytes_Resize(buffer, new_length) < 0) + return -1; + length = new_length; + } + } + + *avail_out = (uint32_t)Py_MIN((size_t)(length - occupied), UINT32_MAX); + *next_out = (uint8_t *)PyBytes_AS_STRING(*buffer) + occupied; + + return length; +} + +/* Decompress data of length self->avail_in_real in self->state.next_in. The + output buffer is allocated dynamically and returned. If the max_length is + of sufficiently low size, max_length is allocated immediately. At most + max_length bytes are returned, so some of the input may not be consumed. + self->state.next_in and self->avail_in_real are updated to reflect the + consumed input. */ +static PyObject* +decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length) +{ + /* data_size is strictly positive, but because we repeatedly have to + compare against max_length and PyBytes_GET_SIZE we declare it as + signed */ + PyObject *return_value = NULL; + Py_ssize_t hard_limit; + Py_ssize_t obuflen; + zlibstate *state = PyType_GetModuleState(Py_TYPE(self)); + + int err = Z_OK; + + /* When sys.maxsize is passed as default use DEF_BUF_SIZE as start buffer. + In this particular case the data may not necessarily be very big, so + it is better to grow dynamically.*/ + if ((max_length < 0) || max_length == PY_SSIZE_T_MAX) { + hard_limit = PY_SSIZE_T_MAX; + obuflen = DEF_BUF_SIZE; + } else { + /* Assume that decompressor is used in file decompression with a fixed + block size of max_length. In that case we will reach max_length almost + always (except at the end of the file). So it makes sense to allocate + max_length. */ + hard_limit = max_length; + obuflen = max_length; + if (obuflen > DEF_MAX_INITIAL_BUF_SIZE){ + // Safeguard against memory overflow. + obuflen = DEF_MAX_INITIAL_BUF_SIZE; + } + } + + do { + arrange_input_buffer(&(self->zst), &(self->avail_in_real)); + + do { + obuflen = arrange_output_buffer_with_maximum(&(self->zst.avail_out), + &(self->zst.next_out), + &return_value, + obuflen, + hard_limit); + if (obuflen == -1){ + PyErr_SetString(PyExc_MemoryError, + "Insufficient memory for buffer allocation"); + goto error; + } + else if (obuflen == -2) { + break; + } + Py_BEGIN_ALLOW_THREADS + err = inflate(&self->zst, Z_SYNC_FLUSH); + Py_END_ALLOW_THREADS + switch (err) { + case Z_OK: /* fall through */ + case Z_BUF_ERROR: /* fall through */ + case Z_STREAM_END: + break; + default: + if (err == Z_NEED_DICT) { + goto error; + } + else { + break; + } + } + } while (self->zst.avail_out == 0); + } while(err != Z_STREAM_END && self->avail_in_real != 0); + + if (err == Z_STREAM_END) { + self->eof = 1; + self->is_initialised = 0; + /* Unlike the Decompress object we call inflateEnd here as there are no + backwards compatibility issues */ + err = inflateEnd(&self->zst); + if (err != Z_OK) { + zlib_error(state, self->zst, err, "while finishing decompression"); + goto error; + } + } else if (err != Z_OK && err != Z_BUF_ERROR) { + zlib_error(state, self->zst, err, "while decompressing data"); + } + + self->avail_in_real += self->zst.avail_in; + + if (_PyBytes_Resize(&return_value, self->zst.next_out - + (uint8_t *)PyBytes_AS_STRING(return_value)) != 0) { + goto error; + } + + goto success; +error: + Py_CLEAR(return_value); +success: + return return_value; +} + + +static PyObject * +decompress(ZlibDecompressor *self, uint8_t *data, + size_t len, Py_ssize_t max_length) +{ + bool input_buffer_in_use; + PyObject *result; + + /* Prepend unconsumed input if necessary */ + if (self->zst.next_in != NULL) { + size_t avail_now, avail_total; + + /* Number of bytes we can append to input buffer */ + avail_now = (self->input_buffer + self->input_buffer_size) + - (self->zst.next_in + self->avail_in_real); + + /* Number of bytes we can append if we move existing + contents to beginning of buffer (overwriting + consumed input) */ + avail_total = self->input_buffer_size - self->avail_in_real; + + if (avail_total < len) { + size_t offset = self->zst.next_in - self->input_buffer; + uint8_t *tmp; + size_t new_size = self->input_buffer_size + len - avail_now; + + /* Assign to temporary variable first, so we don't + lose address of allocated buffer if realloc fails */ + tmp = PyMem_Realloc(self->input_buffer, new_size); + if (tmp == NULL) { + PyErr_SetNone(PyExc_MemoryError); + return NULL; + } + self->input_buffer = tmp; + self->input_buffer_size = new_size; + + self->zst.next_in = self->input_buffer + offset; + } + else if (avail_now < len) { + memmove(self->input_buffer, self->zst.next_in, + self->avail_in_real); + self->zst.next_in = self->input_buffer; + } + memcpy((void*)(self->zst.next_in + self->avail_in_real), data, len); + self->avail_in_real += len; + input_buffer_in_use = 1; + } + else { + self->zst.next_in = data; + self->avail_in_real = len; + input_buffer_in_use = 0; + } + + result = decompress_buf(self, max_length); + if(result == NULL) { + self->zst.next_in = NULL; + return NULL; + } + + if (self->eof) { + self->needs_input = 0; + + if (self->avail_in_real > 0) { + PyObject *unused_data = PyBytes_FromStringAndSize( + (char *)self->zst.next_in, self->avail_in_real); + if (unused_data == NULL) { + goto error; + } + Py_XSETREF(self->unused_data, unused_data); + } + } + else if (self->avail_in_real == 0) { + self->zst.next_in = NULL; + self->needs_input = 1; + } + else { + self->needs_input = 0; + + /* If we did not use the input buffer, we now have + to copy the tail from the caller's buffer into the + input buffer */ + if (!input_buffer_in_use) { + + /* Discard buffer if it's too small + (resizing it may needlessly copy the current contents) */ + if (self->input_buffer != NULL && + self->input_buffer_size < self->avail_in_real) { + PyMem_Free(self->input_buffer); + self->input_buffer = NULL; + } + + /* Allocate if necessary */ + if (self->input_buffer == NULL) { + self->input_buffer = PyMem_Malloc(self->avail_in_real); + if (self->input_buffer == NULL) { + PyErr_SetNone(PyExc_MemoryError); + goto error; + } + self->input_buffer_size = self->avail_in_real; + } + + /* Copy tail */ + memcpy(self->input_buffer, self->zst.next_in, self->avail_in_real); + self->zst.next_in = self->input_buffer; + } + } + return result; + +error: + Py_XDECREF(result); + return NULL; +} + +/*[clinic input] +zlib.ZlibDecompressor.decompress + + data: Py_buffer + max_length: Py_ssize_t=-1 + +Decompress *data*, returning uncompressed data as bytes. + +If *max_length* is nonnegative, returns at most *max_length* bytes of +decompressed data. If this limit is reached and further output can be +produced, *self.needs_input* will be set to ``False``. In this case, the next +call to *decompress()* may provide *data* as b'' to obtain more of the output. + +If all of the input data was decompressed and returned (either because this +was less than *max_length* bytes, or because *max_length* was negative), +*self.needs_input* will be set to True. + +Attempting to decompress data after the end of stream is reached raises an +EOFError. Any data found after the end of the stream is ignored and saved in +the unused_data attribute. +[clinic start generated code]*/ + +static PyObject * +zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, + Py_buffer *data, Py_ssize_t max_length) +/*[clinic end generated code: output=990d32787b775f85 input=0b29d99715250b96]*/ + +{ + PyObject *result = NULL; + + ENTER_ZLIB(self); + if (self->eof) { + PyErr_SetString(PyExc_EOFError, "End of stream already reached"); + } + else { + result = decompress(self, data->buf, data->len, max_length); + } + LEAVE_ZLIB(self); + return result; +} + +PyDoc_STRVAR(ZlibDecompressor__new____doc__, +"_ZlibDecompressor(wbits=15, zdict=b\'\')\n" +"--\n" +"\n" +"Create a decompressor object for decompressing data incrementally.\n" +"\n" +" wbits = 15\n" +" zdict\n" +" The predefined compression dictionary. This is a sequence of bytes\n" +" (such as a bytes object) containing subsequences that are expected\n" +" to occur frequently in the data that is to be compressed. Those\n" +" subsequences that are expected to be most common should come at the\n" +" end of the dictionary. This must be the same dictionary as used by the\n" +" compressor that produced the input data.\n" +"\n"); + +static PyObject * +ZlibDecompressor__new__(PyTypeObject *cls, + PyObject *args, + PyObject *kwargs) +{ + static char *keywords[] = {"wbits", "zdict", NULL}; + static char *format = "|iO:_ZlibDecompressor"; + int wbits = MAX_WBITS; + PyObject *zdict = NULL; + zlibstate *state = PyType_GetModuleState(cls); + + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, format, keywords, &wbits, &zdict)) { + return NULL; + } + ZlibDecompressor *self = PyObject_New(ZlibDecompressor, cls); + self->eof = 0; + self->needs_input = 1; + self->avail_in_real = 0; + self->input_buffer = NULL; + self->input_buffer_size = 0; + if (zdict != NULL) { + Py_INCREF(zdict); + } + self->zdict = zdict; + self->zst.opaque = NULL; + self->zst.zalloc = PyZlib_Malloc; + self->zst.zfree = PyZlib_Free; + self->zst.next_in = NULL; + self->zst.avail_in = 0; + self->unused_data = PyBytes_FromStringAndSize(NULL, 0); + if (self->unused_data == NULL) { + Py_CLEAR(self); + return NULL; + } + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + Py_DECREF(self); + PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); + return NULL; + } + int err = inflateInit2(&(self->zst), wbits); + switch (err) { + case Z_OK: + self->is_initialised = 1; + if (self->zdict != NULL && wbits < 0) { + if (set_inflate_zdict_ZlibDecompressor(state, self) < 0) { + Py_DECREF(self); + return NULL; + } + } + return (PyObject *)self; + case Z_STREAM_ERROR: + Py_DECREF(self); + PyErr_SetString(PyExc_ValueError, "Invalid initialization option"); + return NULL; + case Z_MEM_ERROR: + Py_DECREF(self); + PyErr_SetString(PyExc_MemoryError, + "Can't allocate memory for decompression object"); + return NULL; + default: + zlib_error(state, self->zst, err, "while creating decompression object"); + Py_DECREF(self); + return NULL; + } } #include "clinic/zlibmodule.c.h" @@ -1372,6 +1802,11 @@ static PyMethodDef Decomp_methods[] = {NULL, NULL} }; +static PyMethodDef ZlibDecompressor_methods[] = { + ZLIB_ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF + {NULL} +}; + #define COMP_OFF(x) offsetof(compobject, x) static PyMemberDef Decomp_members[] = { {"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY}, @@ -1380,6 +1815,26 @@ static PyMemberDef Decomp_members[] = { {NULL}, }; +PyDoc_STRVAR(ZlibDecompressor_eof__doc__, +"True if the end-of-stream marker has been reached."); + +PyDoc_STRVAR(ZlibDecompressor_unused_data__doc__, +"Data found after the end of the compressed stream."); + +PyDoc_STRVAR(ZlibDecompressor_needs_input_doc, +"True if more input is needed before more decompressed data can be produced."); + +static PyMemberDef ZlibDecompressor_members[] = { + {"eof", T_BOOL, offsetof(ZlibDecompressor, eof), + READONLY, ZlibDecompressor_eof__doc__}, + {"unused_data", T_OBJECT_EX, offsetof(ZlibDecompressor, unused_data), + READONLY, ZlibDecompressor_unused_data__doc__}, + {"needs_input", T_BOOL, offsetof(ZlibDecompressor, needs_input), READONLY, + ZlibDecompressor_needs_input_doc}, + {NULL}, +}; + + /*[clinic input] zlib.adler32 @@ -1497,6 +1952,25 @@ static PyType_Spec Decomptype_spec = { .slots = Decomptype_slots, }; +static PyType_Slot ZlibDecompressor_type_slots[] = { + {Py_tp_dealloc, ZlibDecompressor_dealloc}, + {Py_tp_members, ZlibDecompressor_members}, + {Py_tp_new, ZlibDecompressor__new__}, + {Py_tp_doc, (char *)ZlibDecompressor__new____doc__}, + {Py_tp_methods, ZlibDecompressor_methods}, + {0, 0}, +}; + +static PyType_Spec ZlibDecompressor_type_spec = { + .name = "zlib._ZlibDecompressor", + .basicsize = sizeof(ZlibDecompressor), + // Calling PyType_GetModuleState() on a subclass is not safe. + // ZlibDecompressor_type_spec does not have Py_TPFLAGS_BASETYPE flag + // which prevents to create a subclass. + // So calling PyType_GetModuleState() in this file is always safe. + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .slots = ZlibDecompressor_type_slots, +}; PyDoc_STRVAR(zlib_module_documentation, "The functions in this module allow compression and decompression using the\n" "zlib library, which is based on GNU zip.\n" @@ -1518,6 +1992,7 @@ zlib_clear(PyObject *mod) zlibstate *state = get_zlib_state(mod); Py_CLEAR(state->Comptype); Py_CLEAR(state->Decomptype); + Py_CLEAR(state->ZlibDecompressorType); Py_CLEAR(state->ZlibError); return 0; } @@ -1528,6 +2003,7 @@ zlib_traverse(PyObject *mod, visitproc visit, void *arg) zlibstate *state = get_zlib_state(mod); Py_VISIT(state->Comptype); Py_VISIT(state->Decomptype); + Py_VISIT(state->ZlibDecompressorType); Py_VISIT(state->ZlibError); return 0; } @@ -1555,6 +2031,12 @@ zlib_exec(PyObject *mod) return -1; } + state->ZlibDecompressorType = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &ZlibDecompressor_type_spec, NULL); + if (state->ZlibDecompressorType == NULL) { + return -1; + } + state->ZlibError = PyErr_NewException("zlib.error", NULL, NULL); if (state->ZlibError == NULL) { return -1; @@ -1565,6 +2047,12 @@ zlib_exec(PyObject *mod) Py_DECREF(state->ZlibError); return -1; } + Py_INCREF(state->ZlibDecompressorType); + if (PyModule_AddObject(mod, "_ZlibDecompressor", + (PyObject *)state->ZlibDecompressorType) < 0) { + Py_DECREF(state->ZlibDecompressorType); + return -1; + } #define ZLIB_ADD_INT_MACRO(c) \ do { \ diff --git a/PC/python3dll.c b/PC/python3dll.c index 89bbd05932b853..c1b88c66903b10 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -1,7 +1,7 @@ /* Re-export stable Python ABI */ -/* Generated by Tools/scripts/stable_abi.py */ +/* Generated by Tools/build/stable_abi.py */ #ifdef _M_IX86 #define DECORATE "_" diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 39939a7ba98345..49e5cc89a26122 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -395,7 +395,7 @@ DependsOnTargets="FindPythonForBuild" Condition="$(Configuration) != 'PGUpdate'"> - -C - <_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> + <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - @@ -69,7 +69,7 @@ Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)" DependsOnTargets="FindPythonForBuild"> - @@ -85,7 +85,7 @@ - diff --git a/Parser/token.c b/Parser/token.c index fa03fbc450b2bc..6299ad2f563144 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -1,4 +1,4 @@ -/* Auto-generated by Tools/scripts/generate_token.py */ +/* Auto-generated by Tools/build/generate_token.py */ #include "Python.h" #include "pycore_token.h" diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6ecbf0c72b5ff4..bbac0c4e1a8a45 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -2,7 +2,7 @@ /* Frozen modules bootstrap * * Limited and restricted Python interpreter to run - * "Tools/scripts/deepfreeze.py" on systems with no or older Python + * "Tools/build/deepfreeze.py" on systems with no or older Python * interpreter. */ diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 3d27b79c237c36..d6d737d48d6070 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -2,7 +2,7 @@ modules into frozen modules (like Lib/importlib/_bootstrap.py into Python/importlib.h). - This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen". + This is used directly by Tools/build/freeze_modules.py, and indirectly by "make regen-frozen". See Python/frozen.c for more info. diff --git a/Python/compile.c b/Python/compile.c index 2da36d0f6316e0..5fbf6fe10d132e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -118,17 +118,17 @@ (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && (c->u->u_ste->ste_type == ModuleBlock)) -struct location { +typedef struct location_ { int lineno; int end_lineno; int col_offset; int end_col_offset; -}; +} location; #define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const struct location){(LNO), (END_LNO), (COL), (END_COL)}) + ((const location){(LNO), (END_LNO), (COL), (END_COL)}) -static struct location NO_LOCATION = {-1, -1, -1, -1}; +static location NO_LOCATION = {-1, -1, -1, -1}; typedef struct jump_target_label_ { int id; @@ -153,7 +153,7 @@ static struct jump_target_label_ NO_LABEL = {-1}; struct instr { int i_opcode; int i_oparg; - struct location i_loc; + location i_loc; /* The following fields should not be set by the front-end: */ struct basicblock_ *i_target; /* target block (if jump instruction) */ struct basicblock_ *i_except; /* target block when exception is raised */ @@ -386,7 +386,6 @@ struct compiler_unit { struct fblockinfo u_fblock[CO_MAXBLOCKS]; int u_firstlineno; /* the first lineno of the block */ - struct location u_loc; /* line/column info of the current stmt */ }; /* This struct captures the global state of a compilation. @@ -418,7 +417,7 @@ struct compiler { }; #define CFG_BUILDER(c) (&((c)->u->u_cfg_builder)) -#define COMPILER_LOC(c) ((c)->u->u_loc) + typedef struct { // A list of strings corresponding to name captures. It is used to track: @@ -449,12 +448,12 @@ static int basicblock_next_instr(basicblock *); static basicblock *cfg_builder_new_block(cfg_builder *g); static int cfg_builder_maybe_start_new_block(cfg_builder *g); -static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, struct location loc); +static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc); static void compiler_free(struct compiler *); -static int compiler_error(struct compiler *, const char *, ...); -static int compiler_warn(struct compiler *, const char *, ...); -static int compiler_nameop(struct compiler *, identifier, expr_context_ty); +static int compiler_error(struct compiler *, location loc, const char *, ...); +static int compiler_warn(struct compiler *, location loc, const char *, ...); +static int compiler_nameop(struct compiler *, location, identifier, expr_context_ty); static PyCodeObject *compiler_mod(struct compiler *, mod_ty); static int compiler_visit_stmt(struct compiler *, stmt_ty); @@ -472,31 +471,33 @@ static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_async_with(struct compiler *, stmt_ty, int); static int compiler_async_for(struct compiler *, stmt_ty); static int compiler_call_simple_kw_helper(struct compiler *c, + location loc, asdl_keyword_seq *keywords, Py_ssize_t nkwelts); -static int compiler_call_helper(struct compiler *c, int n, - asdl_expr_seq *args, +static int compiler_call_helper(struct compiler *c, location loc, + int n, asdl_expr_seq *args, asdl_keyword_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); static int compiler_try_star_except(struct compiler *, stmt_ty); static int compiler_set_qualname(struct compiler *); static int compiler_sync_comprehension_generator( - struct compiler *c, + struct compiler *c, location *ploc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type); static int compiler_async_comprehension_generator( - struct compiler *c, + struct compiler *c, location *ploc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type); -static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); +static int compiler_pattern(struct compiler *, location *, + pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); -static int compiler_pattern_subpattern(struct compiler *, pattern_ty, - pattern_context *); +static int compiler_pattern_subpattern(struct compiler *, location *, + pattern_ty, pattern_context *); static void remove_redundant_nops(basicblock *bb); @@ -997,18 +998,15 @@ basicblock_next_instr(basicblock *b) - before the "except" and "finally" clauses */ -#define SET_LOC(c, x) \ - (c)->u->u_loc.lineno = (x)->lineno; \ - (c)->u->u_loc.end_lineno = (x)->end_lineno; \ - (c)->u->u_loc.col_offset = (x)->col_offset; \ - (c)->u->u_loc.end_col_offset = (x)->end_col_offset; +#define SET_LOC(c, x) // Artificial instructions -#define UNSET_LOC(c) \ - (c)->u->u_loc.lineno = -1; \ - (c)->u->u_loc.end_lineno = -1; \ - (c)->u->u_loc.col_offset = -1; \ - (c)->u->u_loc.end_col_offset = -1; +#define UNSET_LOC(c) + +#define LOC(x) LOCATION((x)->lineno, \ + (x)->end_lineno, \ + (x)->col_offset, \ + (x)->end_col_offset) /* Return the stack effect of opcode with argument oparg. @@ -1290,7 +1288,7 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) */ static int -basicblock_addop(basicblock *b, int opcode, int oparg, struct location loc) +basicblock_addop(basicblock *b, int opcode, int oparg, location loc) { assert(IS_WITHIN_OPCODE_RANGE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); @@ -1336,7 +1334,7 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) } static int -cfg_builder_addop(cfg_builder *g, int opcode, int oparg, struct location loc) +cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) { if (cfg_builder_maybe_start_new_block(g) != 0) { return -1; @@ -1345,7 +1343,7 @@ cfg_builder_addop(cfg_builder *g, int opcode, int oparg, struct location loc) } static int -cfg_builder_addop_noarg(cfg_builder *g, int opcode, struct location loc) +cfg_builder_addop_noarg(cfg_builder *g, int opcode, location loc) { assert(!HAS_ARG(opcode)); return cfg_builder_addop(g, opcode, 0, loc); @@ -1503,27 +1501,27 @@ compiler_add_const(struct compiler *c, PyObject *o) } static int -compiler_addop_load_const(struct compiler *c, PyObject *o) +compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) { Py_ssize_t arg = compiler_add_const(c, o); if (arg < 0) return 0; - return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, COMPILER_LOC(c)); + return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, loc); } static int -compiler_addop_o(struct compiler *c, int opcode, PyObject *dict, - PyObject *o) +compiler_addop_o(struct compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); if (arg < 0) return 0; - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, COMPILER_LOC(c)); + return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); } static int -compiler_addop_name(struct compiler *c, int opcode, PyObject *dict, - PyObject *o) +compiler_addop_name(struct compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg; @@ -1542,14 +1540,14 @@ compiler_addop_name(struct compiler *c, int opcode, PyObject *dict, arg <<= 1; arg |= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, COMPILER_LOC(c)); + return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); } /* Add an opcode with an integer argument. Returns 0 on failure, 1 on success. */ static int -cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, struct location loc) +cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) { /* oparg value is unsigned, but a signed C int is usually used to store it in the C code (like Python/ceval.c). @@ -1564,7 +1562,8 @@ cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, struct locatio } static int -cfg_builder_addop_j(cfg_builder *g, int opcode, jump_target_label target, struct location loc) +cfg_builder_addop_j(cfg_builder *g, location loc, + int opcode, jump_target_label target) { assert(IS_LABEL(target)); assert(IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); @@ -1572,102 +1571,84 @@ cfg_builder_addop_j(cfg_builder *g, int opcode, jump_target_label target, struct } -#define ADDOP(C, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), COMPILER_LOC(C))) \ +#define ADDOP(C, LOC, OP) { \ + if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) \ return 0; \ } -#define ADDOP_NOLINE(C, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), NO_LOCATION)) \ - return 0; \ -} - -#define ADDOP_IN_SCOPE(C, OP) { \ - if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), COMPILER_LOC(C))) { \ +#define ADDOP_IN_SCOPE(C, LOC, OP) { \ + if (!cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) { \ compiler_exit_scope(c); \ return 0; \ } \ } -#define ADDOP_LOAD_CONST(C, O) { \ - if (!compiler_addop_load_const((C), (O))) \ +#define ADDOP_LOAD_CONST(C, LOC, O) { \ + if (!compiler_addop_load_const((C), (LOC), (O))) \ return 0; \ } /* Same as ADDOP_LOAD_CONST, but steals a reference. */ -#define ADDOP_LOAD_CONST_NEW(C, O) { \ +#define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ PyObject *__new_const = (O); \ if (__new_const == NULL) { \ return 0; \ } \ - if (!compiler_addop_load_const((C), __new_const)) { \ + if (!compiler_addop_load_const((C), (LOC), __new_const)) { \ Py_DECREF(__new_const); \ return 0; \ } \ Py_DECREF(__new_const); \ } -#define ADDOP_N(C, OP, O, TYPE) { \ +#define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) { \ + if (!compiler_addop_o((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) { \ Py_DECREF((O)); \ return 0; \ } \ Py_DECREF((O)); \ } -#define ADDOP_NAME(C, OP, O, TYPE) { \ - if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \ +#define ADDOP_NAME(C, LOC, OP, O, TYPE) { \ + if (!compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) \ return 0; \ } -#define ADDOP_I(C, OP, O) { \ - if (!cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), COMPILER_LOC(C))) \ +#define ADDOP_I(C, LOC, OP, O) { \ + if (!cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) \ return 0; \ } -#define ADDOP_I_NOLINE(C, OP, O) { \ - if (!cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), NO_LOCATION)) \ +#define ADDOP_JUMP(C, LOC, OP, O) { \ + if (!cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) \ return 0; \ } -#define ADDOP_JUMP(C, OP, O) { \ - if (!cfg_builder_addop_j(CFG_BUILDER(C), (OP), (O), COMPILER_LOC(C))) \ +#define ADDOP_COMPARE(C, LOC, CMP) { \ + if (!compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) \ return 0; \ } -/* Add a jump with no line number. - * Used for artificial jumps that have no corresponding - * token in the source code. */ -#define ADDOP_JUMP_NOLINE(C, OP, O) { \ - if (!cfg_builder_addop_j(CFG_BUILDER(C), (OP), (O), NO_LOCATION)) \ - return 0; \ -} +#define ADDOP_BINARY(C, LOC, BINOP) \ + RETURN_IF_FALSE(addop_binary((C), (LOC), (BINOP), false)) -#define ADDOP_COMPARE(C, CMP) { \ - if (!compiler_addcompare((C), (cmpop_ty)(CMP))) \ - return 0; \ -} +#define ADDOP_INPLACE(C, LOC, BINOP) \ + RETURN_IF_FALSE(addop_binary((C), (LOC), (BINOP), true)) + +#define ADD_YIELD_FROM(C, LOC, await) \ + RETURN_IF_FALSE(compiler_add_yield_from((C), (LOC), (await))) -#define ADDOP_BINARY(C, BINOP) \ - RETURN_IF_FALSE(addop_binary((C), (BINOP), false)) +#define POP_EXCEPT_AND_RERAISE(C, LOC) \ + RETURN_IF_FALSE(compiler_pop_except_and_reraise((C), (LOC))) -#define ADDOP_INPLACE(C, BINOP) \ - RETURN_IF_FALSE(addop_binary((C), (BINOP), true)) +#define ADDOP_YIELD(C, LOC) \ + RETURN_IF_FALSE(addop_yield((C), (LOC))) /* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. */ -#define ADD_YIELD_FROM(C, await) \ - RETURN_IF_FALSE(compiler_add_yield_from((C), (await))) - -#define POP_EXCEPT_AND_RERAISE(C) \ - RETURN_IF_FALSE(compiler_pop_except_and_reraise((C))) - -#define ADDOP_YIELD(C) \ - RETURN_IF_FALSE(addop_yield(C)) - #define VISIT(C, TYPE, V) {\ if (!compiler_visit_ ## TYPE((C), (V))) \ return 0; \ @@ -1711,6 +1692,8 @@ static int compiler_enter_scope(struct compiler *c, identifier name, int scope_type, void *key, int lineno) { + location loc = LOCATION(lineno, lineno, 0, 0); + struct compiler_unit *u; u = (struct compiler_unit *)PyObject_Calloc(1, sizeof( @@ -1758,7 +1741,6 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_nfblocks = 0; u->u_firstlineno = lineno; - u->u_loc = LOCATION(lineno, lineno, 0, 0); u->u_consts = PyDict_New(); if (!u->u_consts) { compiler_unit_free(u); @@ -1794,16 +1776,16 @@ compiler_enter_scope(struct compiler *c, identifier name, } if (u->u_scope_type == COMPILER_SCOPE_MODULE) { - c->u->u_loc.lineno = 0; + loc.lineno = 0; } else { if (!compiler_set_qualname(c)) return 0; } - ADDOP_I(c, RESUME, 0); + ADDOP_I(c, loc, RESUME, 0); if (u->u_scope_type == COMPILER_SCOPE_MODULE) { - c->u->u_loc.lineno = -1; + loc.lineno = -1; } return 1; } @@ -1911,12 +1893,13 @@ find_ann(asdl_stmt_seq *stmts) */ static int -compiler_push_fblock(struct compiler *c, enum fblocktype t, jump_target_label block_label, +compiler_push_fblock(struct compiler *c, location loc, + enum fblocktype t, jump_target_label block_label, jump_target_label exit, void *datum) { struct fblockinfo *f; if (c->u->u_nfblocks >= CO_MAXBLOCKS) { - return compiler_error(c, "too many statically nested blocks"); + return compiler_error(c, loc, "too many statically nested blocks"); } f = &c->u->u_fblock[c->u->u_nfblocks++]; f->fb_type = t; @@ -1937,40 +1920,41 @@ compiler_pop_fblock(struct compiler *c, enum fblocktype t, jump_target_label blo } static int -compiler_call_exit_with_nones(struct compiler *c) { - ADDOP_LOAD_CONST(c, Py_None); - ADDOP_LOAD_CONST(c, Py_None); - ADDOP_LOAD_CONST(c, Py_None); - ADDOP_I(c, CALL, 2); +compiler_call_exit_with_nones(struct compiler *c, location loc) +{ + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, CALL, 2); return 1; } static int -compiler_add_yield_from(struct compiler *c, int await) +compiler_add_yield_from(struct compiler *c, location loc, int await) { NEW_JUMP_TARGET_LABEL(c, send); NEW_JUMP_TARGET_LABEL(c, fail); NEW_JUMP_TARGET_LABEL(c, exit); USE_LABEL(c, send); - ADDOP_JUMP(c, SEND, exit); + ADDOP_JUMP(c, loc, SEND, exit); // Set up a virtual try/except to handle when StopIteration is raised during // a close or throw call. The only way YIELD_VALUE raises if they do! - ADDOP_JUMP(c, SETUP_FINALLY, fail); - ADDOP_I(c, YIELD_VALUE, 0); - ADDOP_NOLINE(c, POP_BLOCK); - ADDOP_I(c, RESUME, await ? 3 : 2); - ADDOP_JUMP(c, JUMP_NO_INTERRUPT, send); + ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); + ADDOP_I(c, loc, YIELD_VALUE, 0); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_I(c, loc, RESUME, await ? 3 : 2); + ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send); USE_LABEL(c, fail); - ADDOP(c, CLEANUP_THROW); + ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); return 1; } static int -compiler_pop_except_and_reraise(struct compiler *c) +compiler_pop_except_and_reraise(struct compiler *c, location loc) { /* Stack contents * [exc_info, lasti, exc] COPY 3 @@ -1979,9 +1963,9 @@ compiler_pop_except_and_reraise(struct compiler *c) * (exception_unwind clears the stack) */ - ADDOP_I(c, COPY, 3); - ADDOP(c, POP_EXCEPT); - ADDOP_I(c, RERAISE, 1); + ADDOP_I(c, loc, COPY, 3); + ADDOP(c, loc, POP_EXCEPT); + ADDOP_I(c, loc, RERAISE, 1); return 1; } @@ -1991,8 +1975,8 @@ compiler_pop_except_and_reraise(struct compiler *c) * be popped. */ static int -compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info, - int preserve_tos) +compiler_unwind_fblock(struct compiler *c, location *ploc, + struct fblockinfo *info, int preserve_tos) { switch (info->fb_type) { case WHILE_LOOP: @@ -2004,20 +1988,20 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info, case FOR_LOOP: /* Pop the iterator */ if (preserve_tos) { - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, *ploc, SWAP, 2); } - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); return 1; case TRY_EXCEPT: - ADDOP(c, POP_BLOCK); + ADDOP(c, *ploc, POP_BLOCK); return 1; case FINALLY_TRY: /* This POP_BLOCK gets the line number of the unwinding statement */ - ADDOP(c, POP_BLOCK); + ADDOP(c, *ploc, POP_BLOCK); if (preserve_tos) { - if (!compiler_push_fblock(c, POP_VALUE, NO_LABEL, NO_LABEL, NULL)) { + if (!compiler_push_fblock(c, *ploc, POP_VALUE, NO_LABEL, NO_LABEL, NULL)) { return 0; } } @@ -2030,78 +2014,84 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info, * statement causing the unwinding, so make the unwinding * instruction artificial */ UNSET_LOC(c); + *ploc = NO_LOCATION; return 1; case FINALLY_END: if (preserve_tos) { - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, *ploc, SWAP, 2); } - ADDOP(c, POP_TOP); /* exc_value */ + ADDOP(c, *ploc, POP_TOP); /* exc_value */ if (preserve_tos) { - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, *ploc, SWAP, 2); } - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); + ADDOP(c, *ploc, POP_BLOCK); + ADDOP(c, *ploc, POP_EXCEPT); return 1; case WITH: case ASYNC_WITH: SET_LOC(c, (stmt_ty)info->fb_datum); - ADDOP(c, POP_BLOCK); + *ploc = LOC((stmt_ty)info->fb_datum); + ADDOP(c, *ploc, POP_BLOCK); if (preserve_tos) { - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, *ploc, SWAP, 2); } - if(!compiler_call_exit_with_nones(c)) { + if(!compiler_call_exit_with_nones(c, *ploc)) { return 0; } if (info->fb_type == ASYNC_WITH) { - ADDOP_I(c, GET_AWAITABLE, 2); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); + ADDOP_I(c, *ploc, GET_AWAITABLE, 2); + ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADD_YIELD_FROM(c, *ploc, 1); } - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); /* The exit block should appear to execute after the * statement causing the unwinding, so make the unwinding * instruction artificial */ UNSET_LOC(c); + *ploc = NO_LOCATION; return 1; - case HANDLER_CLEANUP: + case HANDLER_CLEANUP: { if (info->fb_datum) { - ADDOP(c, POP_BLOCK); + ADDOP(c, *ploc, POP_BLOCK); } if (preserve_tos) { - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, *ploc, SWAP, 2); } - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); + ADDOP(c, *ploc, POP_BLOCK); + ADDOP(c, *ploc, POP_EXCEPT); if (info->fb_datum) { - ADDOP_LOAD_CONST(c, Py_None); - compiler_nameop(c, info->fb_datum, Store); - compiler_nameop(c, info->fb_datum, Del); + ADDOP_LOAD_CONST(c, *ploc, Py_None); + compiler_nameop(c, *ploc, info->fb_datum, Store); + compiler_nameop(c, *ploc, info->fb_datum, Del); } return 1; - - case POP_VALUE: + } + case POP_VALUE: { if (preserve_tos) { - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, *ploc, SWAP, 2); } - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); return 1; + } } Py_UNREACHABLE(); } /** Unwind block stack. If loop is not NULL, then stop when the first loop is encountered. */ static int -compiler_unwind_fblock_stack(struct compiler *c, int preserve_tos, struct fblockinfo **loop) { +compiler_unwind_fblock_stack(struct compiler *c, location *ploc, + int preserve_tos, struct fblockinfo **loop) +{ if (c->u->u_nfblocks == 0) { return 1; } struct fblockinfo *top = &c->u->u_fblock[c->u->u_nfblocks-1]; if (top->fb_type == EXCEPTION_GROUP_HANDLER) { return compiler_error( - c, "'break', 'continue' and 'return' cannot appear in an except* block"); + c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block"); } if (loop != NULL && (top->fb_type == WHILE_LOOP || top->fb_type == FOR_LOOP)) { *loop = top; @@ -2109,10 +2099,10 @@ compiler_unwind_fblock_stack(struct compiler *c, int preserve_tos, struct fblock } struct fblockinfo copy = *top; c->u->u_nfblocks--; - if (!compiler_unwind_fblock(c, ©, preserve_tos)) { + if (!compiler_unwind_fblock(c, ploc, ©, preserve_tos)) { return 0; } - if (!compiler_unwind_fblock_stack(c, preserve_tos, loop)) { + if (!compiler_unwind_fblock_stack(c, ploc, preserve_tos, loop)) { return 0; } c->u->u_fblock[c->u->u_nfblocks] = copy; @@ -2124,7 +2114,7 @@ compiler_unwind_fblock_stack(struct compiler *c, int preserve_tos, struct fblock and for annotations. */ static int -compiler_body(struct compiler *c, asdl_stmt_seq *stmts) +compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) { int i = 0; stmt_ty st; @@ -2137,10 +2127,11 @@ compiler_body(struct compiler *c, asdl_stmt_seq *stmts) if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && asdl_seq_LEN(stmts)) { st = (stmt_ty)asdl_seq_GET(stmts, 0); SET_LOC(c, st); + loc = LOC(st); } /* Every annotated class and module should have __annotations__. */ if (find_ann(stmts)) { - ADDOP(c, SETUP_ANNOTATIONS); + ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) return 1; @@ -2153,7 +2144,7 @@ compiler_body(struct compiler *c, asdl_stmt_seq *stmts) assert(st->kind == Expr_kind); VISIT(c, expr, st->v.Expr.value); UNSET_LOC(c); - if (!compiler_nameop(c, &_Py_ID(__doc__), Store)) + if (!compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)) return 0; } } @@ -2172,17 +2163,17 @@ compiler_mod(struct compiler *c, mod_ty mod) mod, 1)) { return NULL; } - c->u->u_loc.lineno = 1; + location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: - if (!compiler_body(c, mod->v.Module.body)) { + if (!compiler_body(c, loc, mod->v.Module.body)) { compiler_exit_scope(c); return 0; } break; case Interactive_kind: if (find_ann(mod->v.Interactive.body)) { - ADDOP(c, SETUP_ANNOTATIONS); + ADDOP(c, loc, SETUP_ANNOTATIONS); } c->c_interactive = 1; VISIT_SEQ_IN_SCOPE(c, stmt, mod->v.Interactive.body); @@ -2239,8 +2230,8 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) } static int -compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, - PyObject *qualname) +compiler_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags, PyObject *qualname) { if (qualname == NULL) qualname = co->co_name; @@ -2286,13 +2277,13 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, Py_DECREF(freevars); return 0; } - ADDOP_I(c, LOAD_CLOSURE, arg); + ADDOP_I(c, loc, LOAD_CLOSURE, arg); } flags |= 0x08; - ADDOP_I(c, BUILD_TUPLE, co->co_nfreevars); + ADDOP_I(c, loc, BUILD_TUPLE, co->co_nfreevars); } - ADDOP_LOAD_CONST(c, (PyObject*)co); - ADDOP_I(c, MAKE_FUNCTION, flags); + ADDOP_LOAD_CONST(c, loc, (PyObject*)co); + ADDOP_I(c, loc, MAKE_FUNCTION, flags); return 1; } @@ -2316,18 +2307,17 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) if (!decos) return 1; - struct location old_loc = c->u->u_loc; for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { SET_LOC(c, (expr_ty)asdl_seq_GET(decos, i)); - ADDOP_I(c, CALL, 0); + location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); + ADDOP_I(c, loc, CALL, 0); } - c->u->u_loc = old_loc; return 1; } static int -compiler_visit_kwonlydefaults(struct compiler *c, asdl_arg_seq *kwonlyargs, - asdl_expr_seq *kw_defaults) +compiler_visit_kwonlydefaults(struct compiler *c, location loc, + asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) { /* Push a dict of keyword-only default values. @@ -2368,8 +2358,8 @@ compiler_visit_kwonlydefaults(struct compiler *c, asdl_arg_seq *kwonlyargs, Py_ssize_t default_count = PyList_GET_SIZE(keys); PyObject *keys_tuple = PyList_AsTuple(keys); Py_DECREF(keys); - ADDOP_LOAD_CONST_NEW(c, keys_tuple); - ADDOP_I(c, BUILD_CONST_KEY_MAP, default_count); + ADDOP_LOAD_CONST_NEW(c, loc, keys_tuple); + ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, default_count); assert(default_count > 0); return 1; } @@ -2385,23 +2375,23 @@ compiler_visit_kwonlydefaults(struct compiler *c, asdl_arg_seq *kwonlyargs, static int compiler_visit_annexpr(struct compiler *c, expr_ty annotation) { - ADDOP_LOAD_CONST_NEW(c, _PyAST_ExprAsUnicode(annotation)); + location loc = LOC(annotation); + ADDOP_LOAD_CONST_NEW(c, loc, _PyAST_ExprAsUnicode(annotation)); return 1; } static int compiler_visit_argannotation(struct compiler *c, identifier id, - expr_ty annotation, Py_ssize_t *annotations_len) + expr_ty annotation, Py_ssize_t *annotations_len, location loc) { if (!annotation) { return 1; } - PyObject *mangled = _Py_Mangle(c->u->u_private, id); if (!mangled) { return 0; } - ADDOP_LOAD_CONST(c, mangled); + ADDOP_LOAD_CONST(c, loc, mangled); Py_DECREF(mangled); if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) { @@ -2414,7 +2404,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, // (Note that in theory we could end up here even for an argument // other than *args, but in practice the grammar doesn't allow it.) VISIT(c, expr, annotation->v.Starred.value); - ADDOP_I(c, UNPACK_SEQUENCE, (Py_ssize_t) 1); + ADDOP_I(c, loc, UNPACK_SEQUENCE, (Py_ssize_t) 1); } else { VISIT(c, expr, annotation); @@ -2426,7 +2416,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, static int compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, - Py_ssize_t *annotations_len) + Py_ssize_t *annotations_len, location loc) { int i; for (i = 0; i < asdl_seq_LEN(args); i++) { @@ -2435,15 +2425,16 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, c, arg->arg, arg->annotation, - annotations_len)) + annotations_len, + loc)) return 0; } return 1; } static int -compiler_visit_annotations(struct compiler *c, arguments_ty args, - expr_ty returns) +compiler_visit_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) { /* Push arg annotation names and values. The expressions are evaluated out-of-order wrt the source code. @@ -2452,28 +2443,28 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args, */ Py_ssize_t annotations_len = 0; - if (!compiler_visit_argannotations(c, args->args, &annotations_len)) + if (!compiler_visit_argannotations(c, args->args, &annotations_len, loc)) return 0; - if (!compiler_visit_argannotations(c, args->posonlyargs, &annotations_len)) + if (!compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)) return 0; if (args->vararg && args->vararg->annotation && !compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len)) + args->vararg->annotation, &annotations_len, loc)) return 0; - if (!compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len)) + if (!compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)) return 0; if (args->kwarg && args->kwarg->annotation && !compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len)) + args->kwarg->annotation, &annotations_len, loc)) return 0; if (!compiler_visit_argannotation(c, &_Py_ID(return), returns, - &annotations_len)) { + &annotations_len, loc)) { return 0; } if (annotations_len) { - ADDOP_I(c, BUILD_TUPLE, annotations_len); + ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); return 1; } @@ -2481,24 +2472,27 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args, } static int -compiler_visit_defaults(struct compiler *c, arguments_ty args) +compiler_visit_defaults(struct compiler *c, arguments_ty args, + location loc) { VISIT_SEQ(c, expr, args->defaults); - ADDOP_I(c, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); + ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); return 1; } static Py_ssize_t -compiler_default_arguments(struct compiler *c, arguments_ty args) +compiler_default_arguments(struct compiler *c, location loc, + arguments_ty args) { Py_ssize_t funcflags = 0; if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - if (!compiler_visit_defaults(c, args)) + if (!compiler_visit_defaults(c, args, loc)) return -1; funcflags |= 0x01; } if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs, + int res = compiler_visit_kwonlydefaults(c, loc, + args->kwonlyargs, args->kw_defaults); if (res == 0) { return -1; @@ -2511,15 +2505,15 @@ compiler_default_arguments(struct compiler *c, arguments_ty args) } static int -forbidden_name(struct compiler *c, identifier name, expr_context_ty ctx) +forbidden_name(struct compiler *c, location loc, identifier name, + expr_context_ty ctx) { - if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { - compiler_error(c, "cannot assign to __debug__"); + compiler_error(c, loc, "cannot assign to __debug__"); return 1; } if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { - compiler_error(c, "cannot delete __debug__"); + compiler_error(c, loc, "cannot delete __debug__"); return 1; } return 0; @@ -2529,7 +2523,7 @@ static int compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) { if (arg != NULL) { - if (forbidden_name(c, arg->arg, Store)) + if (forbidden_name(c, LOC(arg), arg->arg, Store)) return 0; } return 1; @@ -2611,12 +2605,12 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno; } - funcflags = compiler_default_arguments(c, args); + location loc = LOC(s); + funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { return 0; } - - annotations = compiler_visit_annotations(c, args, returns); + annotations = compiler_visit_annotations(c, loc, args, returns); if (annotations == 0) { return 0; } @@ -2652,8 +2646,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) Py_XDECREF(co); return 0; } - - if (!compiler_make_closure(c, co, funcflags, qualname)) { + if (!compiler_make_closure(c, loc, co, funcflags, qualname)) { Py_DECREF(qualname); Py_DECREF(co); return 0; @@ -2663,7 +2656,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!compiler_apply_decorators(c, decos)) return 0; - return compiler_nameop(c, name, Store); + return compiler_nameop(c, loc, name, Store); } static int @@ -2691,7 +2684,6 @@ compiler_class(struct compiler *c, stmt_ty s) is the keyword arguments and **kwds argument This borrows from compiler_call. */ - /* 1. compile the class body into a code object */ if (!compiler_enter_scope(c, s->v.ClassDef.name, COMPILER_SCOPE_CLASS, (void *)s, firstlineno)) { @@ -2699,27 +2691,28 @@ compiler_class(struct compiler *c, stmt_ty s) } /* this block represents what we do in the new scope */ { + location loc = LOCATION(firstlineno, firstlineno, 0, 0); /* use the class name for name mangling */ Py_INCREF(s->v.ClassDef.name); Py_XSETREF(c->u->u_private, s->v.ClassDef.name); /* load (global) __name__ ... */ - if (!compiler_nameop(c, &_Py_ID(__name__), Load)) { + if (!compiler_nameop(c, loc, &_Py_ID(__name__), Load)) { compiler_exit_scope(c); return 0; } /* ... and store it as __module__ */ - if (!compiler_nameop(c, &_Py_ID(__module__), Store)) { + if (!compiler_nameop(c, loc, &_Py_ID(__module__), Store)) { compiler_exit_scope(c); return 0; } assert(c->u->u_qualname); - ADDOP_LOAD_CONST(c, c->u->u_qualname); - if (!compiler_nameop(c, &_Py_ID(__qualname__), Store)) { + ADDOP_LOAD_CONST(c, loc, c->u->u_qualname); + if (!compiler_nameop(c, loc, &_Py_ID(__qualname__), Store)) { compiler_exit_scope(c); return 0; } /* compile the body proper */ - if (!compiler_body(c, s->v.ClassDef.body)) { + if (!compiler_body(c, loc, s->v.ClassDef.body)) { compiler_exit_scope(c); return 0; } @@ -2734,10 +2727,9 @@ compiler_class(struct compiler *c, stmt_ty s) return 0; } assert(i == 0); - - ADDOP_I(c, LOAD_CLOSURE, i); - ADDOP_I(c, COPY, 1); - if (!compiler_nameop(c, &_Py_ID(__classcell__), Store)) { + ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); + ADDOP_I(c, NO_LOCATION, COPY, 1); + if (!compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store)) { compiler_exit_scope(c); return 0; } @@ -2745,9 +2737,9 @@ compiler_class(struct compiler *c, stmt_ty s) else { /* No methods referenced __class__, so just return None */ assert(PyDict_GET_SIZE(c->u->u_cellvars) == 0); - ADDOP_LOAD_CONST(c, Py_None); + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); } - ADDOP_IN_SCOPE(c, RETURN_VALUE); + ADDOP_IN_SCOPE(c, NO_LOCATION, RETURN_VALUE); /* create the code object */ co = assemble(c, 1); } @@ -2756,29 +2748,32 @@ compiler_class(struct compiler *c, stmt_ty s) if (co == NULL) return 0; + location loc = LOC(s); /* 2. load the 'build_class' function */ - ADDOP(c, PUSH_NULL); - ADDOP(c, LOAD_BUILD_CLASS); + ADDOP(c, loc, PUSH_NULL); + ADDOP(c, loc, LOAD_BUILD_CLASS); /* 3. load a function (or closure) made from the code object */ - if (!compiler_make_closure(c, co, 0, NULL)) { + if (!compiler_make_closure(c, loc, co, 0, NULL)) { Py_DECREF(co); return 0; } Py_DECREF(co); /* 4. load class name */ - ADDOP_LOAD_CONST(c, s->v.ClassDef.name); + ADDOP_LOAD_CONST(c, loc, s->v.ClassDef.name); /* 5. generate the rest of the code for the call */ - if (!compiler_call_helper(c, 2, s->v.ClassDef.bases, s->v.ClassDef.keywords)) + if (!compiler_call_helper(c, loc, 2, + s->v.ClassDef.bases, + s->v.ClassDef.keywords)) return 0; /* 6. apply decorators */ if (!compiler_apply_decorators(c, decos)) return 0; /* 7. store into */ - if (!compiler_nameop(c, s->v.ClassDef.name, Store)) + if (!compiler_nameop(c, loc, s->v.ClassDef.name, Store)) return 0; return 1; } @@ -2816,7 +2811,7 @@ check_compare(struct compiler *c, expr_ty e) const char *msg = (op == Is) ? "\"is\" with a literal. Did you mean \"==\"?" : "\"is not\" with a literal. Did you mean \"!=\"?"; - return compiler_warn(c, msg); + return compiler_warn(c, LOC(e), msg); } } left = right; @@ -2824,7 +2819,8 @@ check_compare(struct compiler *c, expr_ty e) return 1; } -static int compiler_addcompare(struct compiler *c, cmpop_ty op) +static int compiler_addcompare(struct compiler *c, location loc, + cmpop_ty op) { int cmp; switch (op) { @@ -2847,33 +2843,34 @@ static int compiler_addcompare(struct compiler *c, cmpop_ty op) cmp = Py_GE; break; case Is: - ADDOP_I(c, IS_OP, 0); + ADDOP_I(c, loc, IS_OP, 0); return 1; case IsNot: - ADDOP_I(c, IS_OP, 1); + ADDOP_I(c, loc, IS_OP, 1); return 1; case In: - ADDOP_I(c, CONTAINS_OP, 0); + ADDOP_I(c, loc, CONTAINS_OP, 0); return 1; case NotIn: - ADDOP_I(c, CONTAINS_OP, 1); + ADDOP_I(c, loc, CONTAINS_OP, 1); return 1; default: Py_UNREACHABLE(); } - ADDOP_I(c, COMPARE_OP, cmp); + ADDOP_I(c, loc, COMPARE_OP, cmp); return 1; } static int -compiler_jump_if(struct compiler *c, expr_ty e, jump_target_label next, int cond) +compiler_jump_if(struct compiler *c, location *ploc, + expr_ty e, jump_target_label next, int cond) { switch (e->kind) { case UnaryOp_kind: if (e->v.UnaryOp.op == Not) - return compiler_jump_if(c, e->v.UnaryOp.operand, next, !cond); + return compiler_jump_if(c, ploc, e->v.UnaryOp.operand, next, !cond); /* fallback to general implementation */ break; case BoolOp_kind: { @@ -2887,10 +2884,10 @@ compiler_jump_if(struct compiler *c, expr_ty e, jump_target_label next, int cond next2 = new_next2; } for (i = 0; i < n; ++i) { - if (!compiler_jump_if(c, (expr_ty)asdl_seq_GET(s, i), next2, cond2)) + if (!compiler_jump_if(c, ploc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)) return 0; } - if (!compiler_jump_if(c, (expr_ty)asdl_seq_GET(s, n), next, cond)) + if (!compiler_jump_if(c, ploc, (expr_ty)asdl_seq_GET(s, n), next, cond)) return 0; if (!SAME_LABEL(next2, next)) { USE_LABEL(c, next2); @@ -2900,14 +2897,14 @@ compiler_jump_if(struct compiler *c, expr_ty e, jump_target_label next, int cond case IfExp_kind: { NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next2); - if (!compiler_jump_if(c, e->v.IfExp.test, next2, 0)) + if (!compiler_jump_if(c, ploc, e->v.IfExp.test, next2, 0)) return 0; - if (!compiler_jump_if(c, e->v.IfExp.body, next, cond)) + if (!compiler_jump_if(c, ploc, e->v.IfExp.body, next, cond)) return 0; - ADDOP_JUMP_NOLINE(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, next2); - if (!compiler_jump_if(c, e->v.IfExp.orelse, next, cond)) + if (!compiler_jump_if(c, ploc, e->v.IfExp.orelse, next, cond)) return 0; USE_LABEL(c, end); @@ -2915,6 +2912,7 @@ compiler_jump_if(struct compiler *c, expr_ty e, jump_target_label next, int cond } case Compare_kind: { SET_LOC(c, e); + *ploc = LOC(e); Py_ssize_t i, n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n > 0) { if (!check_compare(c, e)) { @@ -2925,21 +2923,21 @@ compiler_jump_if(struct compiler *c, expr_ty e, jump_target_label next, int cond for (i = 0; i < n; i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); - ADDOP_I(c, SWAP, 2); - ADDOP_I(c, COPY, 2); - ADDOP_COMPARE(c, asdl_seq_GET(e->v.Compare.ops, i)); - ADDOP_JUMP(c, POP_JUMP_IF_FALSE, cleanup); + ADDOP_I(c, *ploc, SWAP, 2); + ADDOP_I(c, *ploc, COPY, 2); + ADDOP_COMPARE(c, *ploc, asdl_seq_GET(e->v.Compare.ops, i)); + ADDOP_JUMP(c, *ploc, POP_JUMP_IF_FALSE, cleanup); } VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); - ADDOP_COMPARE(c, asdl_seq_GET(e->v.Compare.ops, n)); - ADDOP_JUMP(c, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); + ADDOP_COMPARE(c, *ploc, asdl_seq_GET(e->v.Compare.ops, n)); + ADDOP_JUMP(c, *ploc, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP_NOLINE(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, cleanup); - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); if (!cond) { - ADDOP_JUMP_NOLINE(c, JUMP, next); + ADDOP_JUMP(c, NO_LOCATION, JUMP, next); } USE_LABEL(c, end); @@ -2955,7 +2953,7 @@ compiler_jump_if(struct compiler *c, expr_ty e, jump_target_label next, int cond /* general implementation */ VISIT(c, expr, e); - ADDOP_JUMP(c, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); + ADDOP_JUMP(c, *ploc, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); return 1; } @@ -2966,10 +2964,12 @@ compiler_ifexp(struct compiler *c, expr_ty e) NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, next); - if (!compiler_jump_if(c, e->v.IfExp.test, next, 0)) + location loc = LOC(e); + if (!compiler_jump_if(c, &loc, e->v.IfExp.test, next, 0)) { return 0; + } VISIT(c, expr, e->v.IfExp.body); - ADDOP_JUMP_NOLINE(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, next); VISIT(c, expr, e->v.IfExp.orelse); @@ -2990,7 +2990,8 @@ compiler_lambda(struct compiler *c, expr_ty e) if (!compiler_check_debug_args(c, args)) return 0; - funcflags = compiler_default_arguments(c, args); + location loc = LOC(e); + funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { return 0; } @@ -3013,7 +3014,8 @@ compiler_lambda(struct compiler *c, expr_ty e) co = assemble(c, 0); } else { - ADDOP_IN_SCOPE(c, RETURN_VALUE); + location loc = LOCATION(e->lineno, e->lineno, 0, 0); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); co = assemble(c, 1); } qualname = c->u->u_qualname; @@ -3024,7 +3026,7 @@ compiler_lambda(struct compiler *c, expr_ty e) return 0; } - if (!compiler_make_closure(c, co, funcflags, qualname)) { + if (!compiler_make_closure(c, loc, co, funcflags, qualname)) { Py_DECREF(qualname); Py_DECREF(co); return 0; @@ -3048,12 +3050,13 @@ compiler_if(struct compiler *c, stmt_ty s) else { next = end; } - if (!compiler_jump_if(c, s->v.If.test, next, 0)) { + location loc = LOC(s); + if (!compiler_jump_if(c, &loc, s->v.If.test, next, 0)) { return 0; } VISIT_SEQ(c, stmt, s->v.If.body); if (asdl_seq_LEN(s->v.If.orelse)) { - ADDOP_JUMP_NOLINE(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, next); VISIT_SEQ(c, stmt, s->v.If.orelse); @@ -3066,26 +3069,27 @@ compiler_if(struct compiler *c, stmt_ty s) static int compiler_for(struct compiler *c, stmt_ty s) { + location loc = LOC(s); NEW_JUMP_TARGET_LABEL(c, start); NEW_JUMP_TARGET_LABEL(c, body); NEW_JUMP_TARGET_LABEL(c, cleanup); NEW_JUMP_TARGET_LABEL(c, end); - if (!compiler_push_fblock(c, FOR_LOOP, start, end, NULL)) { + if (!compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)) { return 0; } VISIT(c, expr, s->v.For.iter); - ADDOP(c, GET_ITER); + ADDOP(c, loc, GET_ITER); USE_LABEL(c, start); - ADDOP_JUMP(c, FOR_ITER, cleanup); + ADDOP_JUMP(c, loc, FOR_ITER, cleanup); USE_LABEL(c, body); VISIT(c, expr, s->v.For.target); VISIT_SEQ(c, stmt, s->v.For.body); /* Mark jump as artificial */ UNSET_LOC(c); - ADDOP_JUMP(c, JUMP, start); + ADDOP_JUMP(c, NO_LOCATION, JUMP, start); USE_LABEL(c, cleanup); @@ -3101,10 +3105,11 @@ compiler_for(struct compiler *c, stmt_ty s) static int compiler_async_for(struct compiler *c, stmt_ty s) { + location loc = LOC(s); if (IS_TOP_LEVEL_AWAIT(c)){ c->u->u_ste->ste_coroutine = 1; } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) { - return compiler_error(c, "'async for' outside async function"); + return compiler_error(c, loc, "'async for' outside async function"); } NEW_JUMP_TARGET_LABEL(c, start); @@ -3112,25 +3117,25 @@ compiler_async_for(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, end); VISIT(c, expr, s->v.AsyncFor.iter); - ADDOP(c, GET_AITER); + ADDOP(c, loc, GET_AITER); USE_LABEL(c, start); - if (!compiler_push_fblock(c, FOR_LOOP, start, end, NULL)) { + if (!compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)) { return 0; } /* SETUP_FINALLY to guard the __anext__ call */ - ADDOP_JUMP(c, SETUP_FINALLY, except); - ADDOP(c, GET_ANEXT); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); - ADDOP(c, POP_BLOCK); /* for SETUP_FINALLY */ + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); + ADDOP(c, loc, GET_ANEXT); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + ADDOP(c, loc, POP_BLOCK); /* for SETUP_FINALLY */ /* Success block for __anext__ */ VISIT(c, expr, s->v.AsyncFor.target); VISIT_SEQ(c, stmt, s->v.AsyncFor.body); /* Mark jump as artificial */ UNSET_LOC(c); - ADDOP_JUMP(c, JUMP, start); + ADDOP_JUMP(c, NO_LOCATION, JUMP, start); compiler_pop_fblock(c, FOR_LOOP, start); @@ -3140,7 +3145,8 @@ compiler_async_for(struct compiler *c, stmt_ty s) /* Use same line number as the iterator, * as the END_ASYNC_FOR succeeds the `for`, not the body. */ SET_LOC(c, s->v.AsyncFor.iter); - ADDOP(c, END_ASYNC_FOR); + loc = LOC(s->v.AsyncFor.iter); + ADDOP(c, loc, END_ASYNC_FOR); /* `else` block */ VISIT_SEQ(c, stmt, s->v.For.orelse); @@ -3152,23 +3158,25 @@ compiler_async_for(struct compiler *c, stmt_ty s) static int compiler_while(struct compiler *c, stmt_ty s) { + location loc = LOC(s); NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, body); NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, anchor); USE_LABEL(c, loop); - if (!compiler_push_fblock(c, WHILE_LOOP, loop, end, NULL)) { + if (!compiler_push_fblock(c, loc, WHILE_LOOP, loop, end, NULL)) { return 0; } - if (!compiler_jump_if(c, s->v.While.test, anchor, 0)) { + if (!compiler_jump_if(c, &loc, s->v.While.test, anchor, 0)) { return 0; } USE_LABEL(c, body); VISIT_SEQ(c, stmt, s->v.While.body); SET_LOC(c, s); - if (!compiler_jump_if(c, s->v.While.test, body, 1)) { + loc = LOC(s); + if (!compiler_jump_if(c, &loc, s->v.While.test, body, 1)) { return 0; } @@ -3186,79 +3194,95 @@ compiler_while(struct compiler *c, stmt_ty s) static int compiler_return(struct compiler *c, stmt_ty s) { + location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && (s->v.Return.value->kind != Constant_kind)); if (c->u->u_ste->ste_type != FunctionBlock) - return compiler_error(c, "'return' outside function"); + return compiler_error(c, loc, "'return' outside function"); if (s->v.Return.value != NULL && c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) { return compiler_error( - c, "'return' with value in async generator"); + c, loc, "'return' with value in async generator"); } + if (preserve_tos) { VISIT(c, expr, s->v.Return.value); } else { /* Emit instruction with line number for return value */ if (s->v.Return.value != NULL) { SET_LOC(c, s->v.Return.value); - ADDOP(c, NOP); + loc = LOC(s->v.Return.value); + ADDOP(c, loc, NOP); } } if (s->v.Return.value == NULL || s->v.Return.value->lineno != s->lineno) { SET_LOC(c, s); - ADDOP(c, NOP); + loc = LOC(s); + ADDOP(c, loc, NOP); } - if (!compiler_unwind_fblock_stack(c, preserve_tos, NULL)) + if (!compiler_unwind_fblock_stack(c, &loc, preserve_tos, NULL)) return 0; if (s->v.Return.value == NULL) { - ADDOP_LOAD_CONST(c, Py_None); + ADDOP_LOAD_CONST(c, loc, Py_None); } else if (!preserve_tos) { - ADDOP_LOAD_CONST(c, s->v.Return.value->v.Constant.value); + ADDOP_LOAD_CONST(c, loc, s->v.Return.value->v.Constant.value); } - ADDOP(c, RETURN_VALUE); + ADDOP(c, loc, RETURN_VALUE); return 1; } static int -compiler_break(struct compiler *c) +compiler_break(struct compiler *c, location loc) { struct fblockinfo *loop = NULL; /* Emit instruction with line number */ - ADDOP(c, NOP); - if (!compiler_unwind_fblock_stack(c, 0, &loop)) { + ADDOP(c, loc, NOP); + if (!compiler_unwind_fblock_stack(c, &loc, 0, &loop)) { return 0; } if (loop == NULL) { - return compiler_error(c, "'break' outside loop"); + return compiler_error(c, loc, "'break' outside loop"); } - if (!compiler_unwind_fblock(c, loop, 0)) { + if (!compiler_unwind_fblock(c, &loc, loop, 0)) { return 0; } - ADDOP_JUMP(c, JUMP, loop->fb_exit); + ADDOP_JUMP(c, loc, JUMP, loop->fb_exit); return 1; } static int -compiler_continue(struct compiler *c) +compiler_continue(struct compiler *c, location loc) { struct fblockinfo *loop = NULL; /* Emit instruction with line number */ - ADDOP(c, NOP); - if (!compiler_unwind_fblock_stack(c, 0, &loop)) { + ADDOP(c, loc, NOP); + if (!compiler_unwind_fblock_stack(c, &loc, 0, &loop)) { return 0; } if (loop == NULL) { - return compiler_error(c, "'continue' not properly in loop"); + return compiler_error(c, loc, "'continue' not properly in loop"); } - ADDOP_JUMP(c, JUMP, loop->fb_block); + ADDOP_JUMP(c, loc, JUMP, loop->fb_block); return 1; } +static location +location_of_last_executing_statement(asdl_stmt_seq *stmts) +{ + for (Py_ssize_t i = asdl_seq_LEN(stmts) - 1; i >= 0; i++) { + location loc = LOC((stmt_ty)asdl_seq_GET(stmts, i)); + if (loc.lineno > 0) { + return loc; + } + } + return NO_LOCATION; +} + /* Code generated for "try: finally: " is as follows: SETUP_FINALLY L @@ -3291,16 +3315,18 @@ compiler_continue(struct compiler *c) static int compiler_try_finally(struct compiler *c, stmt_ty s) { + location loc = LOC(s); + NEW_JUMP_TARGET_LABEL(c, body); NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, exit); NEW_JUMP_TARGET_LABEL(c, cleanup); /* `try` block */ - ADDOP_JUMP(c, SETUP_FINALLY, end); + ADDOP_JUMP(c, loc, SETUP_FINALLY, end); USE_LABEL(c, body); - if (!compiler_push_fblock(c, FINALLY_TRY, body, end, s->v.Try.finalbody)) + if (!compiler_push_fblock(c, loc, FINALLY_TRY, body, end, s->v.Try.finalbody)) return 0; if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { if (!compiler_try_except(c, s)) @@ -3309,25 +3335,29 @@ compiler_try_finally(struct compiler *c, stmt_ty s) else { VISIT_SEQ(c, stmt, s->v.Try.body); } - ADDOP_NOLINE(c, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); compiler_pop_fblock(c, FINALLY_TRY, body); VISIT_SEQ(c, stmt, s->v.Try.finalbody); - ADDOP_JUMP_NOLINE(c, JUMP, exit); + + ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); /* `finally` block */ USE_LABEL(c, end); UNSET_LOC(c); - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup); - ADDOP(c, PUSH_EXC_INFO); - if (!compiler_push_fblock(c, FINALLY_END, end, NO_LABEL, NULL)) + loc = NO_LOCATION; + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + if (!compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)) return 0; VISIT_SEQ(c, stmt, s->v.Try.finalbody); + loc = location_of_last_executing_statement(s->v.Try.finalbody); compiler_pop_fblock(c, FINALLY_END, end); - ADDOP_I(c, RERAISE, 0); + + ADDOP_I(c, loc, RERAISE, 0); USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c); + POP_EXCEPT_AND_RERAISE(c, loc); USE_LABEL(c, exit); return 1; @@ -3336,15 +3366,17 @@ compiler_try_finally(struct compiler *c, stmt_ty s) static int compiler_try_star_finally(struct compiler *c, stmt_ty s) { + location loc = LOC(s); + NEW_JUMP_TARGET_LABEL(c, body); NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, exit); NEW_JUMP_TARGET_LABEL(c, cleanup); /* `try` block */ - ADDOP_JUMP(c, SETUP_FINALLY, end); + ADDOP_JUMP(c, loc, SETUP_FINALLY, end); USE_LABEL(c, body); - if (!compiler_push_fblock(c, FINALLY_TRY, body, end, s->v.TryStar.finalbody)) { + if (!compiler_push_fblock(c, loc, FINALLY_TRY, body, end, s->v.TryStar.finalbody)) { return 0; } if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { @@ -3355,26 +3387,30 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) else { VISIT_SEQ(c, stmt, s->v.TryStar.body); } - ADDOP_NOLINE(c, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); compiler_pop_fblock(c, FINALLY_TRY, body); VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); - ADDOP_JUMP_NOLINE(c, JUMP, exit); + + ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); /* `finally` block */ USE_LABEL(c, end); UNSET_LOC(c); - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup); - ADDOP(c, PUSH_EXC_INFO); - if (!compiler_push_fblock(c, FINALLY_END, end, NO_LABEL, NULL)) { + loc = NO_LOCATION; + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + if (!compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)) { return 0; } VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); + loc = location_of_last_executing_statement(s->v.Try.finalbody); + compiler_pop_fblock(c, FINALLY_END, end); - ADDOP_I(c, RERAISE, 0); + ADDOP_I(c, loc, RERAISE, 0); USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c); + POP_EXCEPT_AND_RERAISE(c, loc); USE_LABEL(c, exit); return 1; @@ -3412,6 +3448,7 @@ compiler_try_star_finally(struct compiler *c, stmt_ty s) static int compiler_try_except(struct compiler *c, stmt_ty s) { + location loc = LOC(s); Py_ssize_t i, n; NEW_JUMP_TARGET_LABEL(c, body); @@ -3419,47 +3456,48 @@ compiler_try_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, end); NEW_JUMP_TARGET_LABEL(c, cleanup); - ADDOP_JUMP(c, SETUP_FINALLY, except); + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); USE_LABEL(c, body); - if (!compiler_push_fblock(c, TRY_EXCEPT, body, NO_LABEL, NULL)) + if (!compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)) return 0; VISIT_SEQ(c, stmt, s->v.Try.body); compiler_pop_fblock(c, TRY_EXCEPT, body); - ADDOP_NOLINE(c, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); if (s->v.Try.orelse && asdl_seq_LEN(s->v.Try.orelse)) { VISIT_SEQ(c, stmt, s->v.Try.orelse); } - ADDOP_JUMP_NOLINE(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); n = asdl_seq_LEN(s->v.Try.handlers); USE_LABEL(c, except); UNSET_LOC(c); - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup); - ADDOP(c, PUSH_EXC_INFO); + ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); + ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)) + if (!compiler_push_fblock(c, loc, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)) return 0; for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( s->v.Try.handlers, i); SET_LOC(c, handler); + location loc = LOC(handler); if (!handler->v.ExceptHandler.type && i < n-1) { - return compiler_error(c, "default 'except:' must be last"); + return compiler_error(c, loc, "default 'except:' must be last"); } NEW_JUMP_TARGET_LABEL(c, next_except); except = next_except; if (handler->v.ExceptHandler.type) { VISIT(c, expr, handler->v.ExceptHandler.type); - ADDOP(c, CHECK_EXC_MATCH); - ADDOP_JUMP(c, POP_JUMP_IF_FALSE, except); + ADDOP(c, loc, CHECK_EXC_MATCH); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, except); } if (handler->v.ExceptHandler.name) { NEW_JUMP_TARGET_LABEL(c, cleanup_end); NEW_JUMP_TARGET_LABEL(c, cleanup_body); - compiler_nameop(c, handler->v.ExceptHandler.name, Store); + compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store); /* try: @@ -3473,24 +3511,26 @@ compiler_try_except(struct compiler *c, stmt_ty s) */ /* second try: */ - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup_end); + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, HANDLER_CLEANUP, cleanup_body, NO_LABEL, handler->v.ExceptHandler.name)) + if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)) { return 0; + } /* second # body */ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); /* name = None; del name; # Mark as artificial */ UNSET_LOC(c); - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); - ADDOP_LOAD_CONST(c, Py_None); - compiler_nameop(c, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, handler->v.ExceptHandler.name, Del); - ADDOP_JUMP(c, JUMP, end); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); /* except: */ USE_LABEL(c, cleanup_end); @@ -3498,26 +3538,26 @@ compiler_try_except(struct compiler *c, stmt_ty s) /* name = None; del name; # Mark as artificial */ UNSET_LOC(c); - ADDOP_LOAD_CONST(c, Py_None); - compiler_nameop(c, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, handler->v.ExceptHandler.name, Del); + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); - ADDOP_I(c, RERAISE, 1); + ADDOP_I(c, NO_LOCATION, RERAISE, 1); } else { NEW_JUMP_TARGET_LABEL(c, cleanup_body); - ADDOP(c, POP_TOP); /* exc_value */ + ADDOP(c, loc, POP_TOP); /* exc_value */ USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, HANDLER_CLEANUP, cleanup_body, NO_LABEL, NULL)) + if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, NO_LABEL, NULL)) return 0; VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); UNSET_LOC(c); - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); - ADDOP_JUMP(c, JUMP, end); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); } USE_LABEL(c, except); @@ -3525,10 +3565,10 @@ compiler_try_except(struct compiler *c, stmt_ty s) /* Mark as artificial */ UNSET_LOC(c); compiler_pop_fblock(c, EXCEPTION_HANDLER, NO_LABEL); - ADDOP_I(c, RERAISE, 0); + ADDOP_I(c, NO_LOCATION, RERAISE, 0); USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c); + POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, end); return 1; @@ -3586,6 +3626,8 @@ compiler_try_except(struct compiler *c, stmt_ty s) static int compiler_try_star_except(struct compiler *c, stmt_ty s) { + location loc = LOC(s); + NEW_JUMP_TARGET_LABEL(c, body); NEW_JUMP_TARGET_LABEL(c, except); NEW_JUMP_TARGET_LABEL(c, orelse); @@ -3593,25 +3635,25 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup); NEW_JUMP_TARGET_LABEL(c, reraise_star); - ADDOP_JUMP(c, SETUP_FINALLY, except); + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); USE_LABEL(c, body); - if (!compiler_push_fblock(c, TRY_EXCEPT, body, NO_LABEL, NULL)) { + if (!compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)) { return 0; } VISIT_SEQ(c, stmt, s->v.TryStar.body); compiler_pop_fblock(c, TRY_EXCEPT, body); - ADDOP_NOLINE(c, POP_BLOCK); - ADDOP_JUMP_NOLINE(c, JUMP, orelse); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_JUMP(c, NO_LOCATION, JUMP, orelse); Py_ssize_t n = asdl_seq_LEN(s->v.TryStar.handlers); USE_LABEL(c, except); UNSET_LOC(c); - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup); - ADDOP(c, PUSH_EXC_INFO); + ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); + ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, EXCEPTION_GROUP_HANDLER, + if (!compiler_push_fblock(c, loc, EXCEPTION_GROUP_HANDLER, NO_LABEL, NO_LABEL, "except handler")) { return 0; } @@ -3619,6 +3661,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( s->v.TryStar.handlers, i); SET_LOC(c, handler); + location loc = LOC(handler); NEW_JUMP_TARGET_LABEL(c, next_except); except = next_except; NEW_JUMP_TARGET_LABEL(c, handle_match); @@ -3628,7 +3671,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) [exc] COPY 1 [orig, exc] */ - ADDOP_I(c, COPY, 1); + ADDOP_I(c, loc, COPY, 1); /* create empty list for exceptions raised/reraise in the except* blocks */ /* @@ -3636,16 +3679,16 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) [orig, exc, []] SWAP 2 [orig, [], exc] */ - ADDOP_I(c, BUILD_LIST, 0); - ADDOP_I(c, SWAP, 2); + ADDOP_I(c, loc, BUILD_LIST, 0); + ADDOP_I(c, loc, SWAP, 2); } if (handler->v.ExceptHandler.type) { VISIT(c, expr, handler->v.ExceptHandler.type); - ADDOP(c, CHECK_EG_MATCH); - ADDOP_I(c, COPY, 1); - ADDOP_JUMP(c, POP_JUMP_IF_NOT_NONE, handle_match); - ADDOP(c, POP_TOP); // match - ADDOP_JUMP(c, JUMP, except); + ADDOP(c, loc, CHECK_EG_MATCH); + ADDOP_I(c, loc, COPY, 1); + ADDOP_JUMP(c, loc, POP_JUMP_IF_NOT_NONE, handle_match); + ADDOP(c, loc, POP_TOP); // match + ADDOP_JUMP(c, loc, JUMP, except); } USE_LABEL(c, handle_match); @@ -3654,10 +3697,10 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, cleanup_body); if (handler->v.ExceptHandler.name) { - compiler_nameop(c, handler->v.ExceptHandler.name, Store); + compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store); } else { - ADDOP(c, POP_TOP); // match + ADDOP(c, loc, POP_TOP); // match } /* @@ -3671,24 +3714,25 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) del name */ /* second try: */ - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup_end); + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); USE_LABEL(c, cleanup_body); - if (!compiler_push_fblock(c, HANDLER_CLEANUP, cleanup_body, NO_LABEL, handler->v.ExceptHandler.name)) + if (!compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, NO_LABEL, handler->v.ExceptHandler.name)) { return 0; + } /* second # body */ VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); /* name = None; del name; # Mark as artificial */ UNSET_LOC(c); - ADDOP(c, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); if (handler->v.ExceptHandler.name) { - ADDOP_LOAD_CONST(c, Py_None); - compiler_nameop(c, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, handler->v.ExceptHandler.name, Del); + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); } - ADDOP_JUMP(c, JUMP, except); + ADDOP_JUMP(c, NO_LOCATION, JUMP, except); /* except: */ USE_LABEL(c, cleanup_end); @@ -3697,22 +3741,22 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) UNSET_LOC(c); if (handler->v.ExceptHandler.name) { - ADDOP_LOAD_CONST(c, Py_None); - compiler_nameop(c, handler->v.ExceptHandler.name, Store); - compiler_nameop(c, handler->v.ExceptHandler.name, Del); + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store); + compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del); } /* add exception raised to the res list */ - ADDOP_I(c, LIST_APPEND, 3); // exc - ADDOP(c, POP_TOP); // lasti - ADDOP_JUMP(c, JUMP, except); + ADDOP_I(c, NO_LOCATION, LIST_APPEND, 3); // exc + ADDOP(c, NO_LOCATION, POP_TOP); // lasti + ADDOP_JUMP(c, NO_LOCATION, JUMP, except); USE_LABEL(c, except); if (i == n - 1) { /* Add exc to the list (if not None it's the unhandled part of the EG) */ - ADDOP_I(c, LIST_APPEND, 1); - ADDOP_JUMP(c, JUMP, reraise_star); + ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); + ADDOP_JUMP(c, NO_LOCATION, JUMP, reraise_star); } } /* Mark as artificial */ @@ -3721,24 +3765,24 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, reraise); USE_LABEL(c, reraise_star); - ADDOP(c, PREP_RERAISE_STAR); - ADDOP_I(c, COPY, 1); - ADDOP_JUMP(c, POP_JUMP_IF_NOT_NONE, reraise); + ADDOP(c, NO_LOCATION, PREP_RERAISE_STAR); + ADDOP_I(c, NO_LOCATION, COPY, 1); + ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_NOT_NONE, reraise); /* Nothing to reraise */ - ADDOP(c, POP_TOP); - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); - ADDOP_JUMP(c, JUMP, end); + ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, reraise); - ADDOP(c, POP_BLOCK); - ADDOP_I(c, SWAP, 2); - ADDOP(c, POP_EXCEPT); - ADDOP_I(c, RERAISE, 0); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_I(c, NO_LOCATION, SWAP, 2); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_I(c, NO_LOCATION, RERAISE, 0); USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c); + POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, orelse); VISIT_SEQ(c, stmt, s->v.TryStar.orelse); @@ -3767,7 +3811,8 @@ compiler_try_star(struct compiler *c, stmt_ty s) } static int -compiler_import_as(struct compiler *c, identifier name, identifier asname) +compiler_import_as(struct compiler *c, location loc, + identifier name, identifier asname) { /* The IMPORT_NAME opcode was already generated. This function merely needs to bind the result to a name. @@ -3790,25 +3835,26 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); if (!attr) return 0; - ADDOP_N(c, IMPORT_FROM, attr, names); + ADDOP_N(c, loc, IMPORT_FROM, attr, names); if (dot == -1) { break; } - ADDOP_I(c, SWAP, 2); - ADDOP(c, POP_TOP); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, POP_TOP); } - if (!compiler_nameop(c, asname, Store)) { + if (!compiler_nameop(c, loc, asname, Store)) { return 0; } - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); return 1; } - return compiler_nameop(c, asname, Store); + return compiler_nameop(c, loc, asname, Store); } static int compiler_import(struct compiler *c, stmt_ty s) { + location loc = LOC(s); /* The Import node stores a module name like a.b.c as a single string. This is convenient for all cases except import a.b.c as d @@ -3823,12 +3869,12 @@ compiler_import(struct compiler *c, stmt_ty s) alias_ty alias = (alias_ty)asdl_seq_GET(s->v.Import.names, i); int r; - ADDOP_LOAD_CONST(c, zero); - ADDOP_LOAD_CONST(c, Py_None); - ADDOP_NAME(c, IMPORT_NAME, alias->name, names); + ADDOP_LOAD_CONST(c, loc, zero); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_NAME(c, loc, IMPORT_NAME, alias->name, names); if (alias->asname) { - r = compiler_import_as(c, alias->name, alias->asname); + r = compiler_import_as(c, loc, alias->name, alias->asname); if (!r) return r; } @@ -3841,7 +3887,7 @@ compiler_import(struct compiler *c, stmt_ty s) if (tmp == NULL) return 0; } - r = compiler_nameop(c, tmp, Store); + r = compiler_nameop(c, loc, tmp, Store); if (dot != -1) { Py_DECREF(tmp); } @@ -3855,10 +3901,11 @@ compiler_import(struct compiler *c, stmt_ty s) static int compiler_from_import(struct compiler *c, stmt_ty s) { + location loc = LOC(s); Py_ssize_t i, n = asdl_seq_LEN(s->v.ImportFrom.names); PyObject *names; - ADDOP_LOAD_CONST_NEW(c, PyLong_FromLong(s->v.ImportFrom.level)); + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(s->v.ImportFrom.level)); names = PyTuple_New(n); if (!names) @@ -3874,17 +3921,17 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { Py_DECREF(names); - return compiler_error(c, "from __future__ imports must occur " + return compiler_error(c, loc, "from __future__ imports must occur " "at the beginning of the file"); } - ADDOP_LOAD_CONST_NEW(c, names); + ADDOP_LOAD_CONST_NEW(c, loc, names); if (s->v.ImportFrom.module) { - ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names); + ADDOP_NAME(c, loc, IMPORT_NAME, s->v.ImportFrom.module, names); } else { _Py_DECLARE_STR(empty, ""); - ADDOP_NAME(c, IMPORT_NAME, &_Py_STR(empty), names); + ADDOP_NAME(c, loc, IMPORT_NAME, &_Py_STR(empty), names); } for (i = 0; i < n; i++) { alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); @@ -3892,27 +3939,28 @@ compiler_from_import(struct compiler *c, stmt_ty s) if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { assert(n == 1); - ADDOP(c, IMPORT_STAR); + ADDOP(c, loc, IMPORT_STAR); return 1; } - ADDOP_NAME(c, IMPORT_FROM, alias->name, names); + ADDOP_NAME(c, loc, IMPORT_FROM, alias->name, names); store_name = alias->name; if (alias->asname) store_name = alias->asname; - if (!compiler_nameop(c, store_name, Store)) { + if (!compiler_nameop(c, loc, store_name, Store)) { return 0; } } /* remove imported module */ - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); return 1; } static int compiler_assert(struct compiler *c, stmt_ty s) { + location loc = LOC(s); /* Always emit a warning if the test is a non-zero length tuple */ if ((s->v.Assert.test->kind == Tuple_kind && asdl_seq_LEN(s->v.Assert.test->v.Tuple.elts) > 0) || @@ -3920,8 +3968,8 @@ compiler_assert(struct compiler *c, stmt_ty s) PyTuple_Check(s->v.Assert.test->v.Constant.value) && PyTuple_Size(s->v.Assert.test->v.Constant.value) > 0)) { - if (!compiler_warn(c, "assertion is always true, " - "perhaps remove parentheses?")) + if (!compiler_warn(c, loc, "assertion is always true, " + "perhaps remove parentheses?")) { return 0; } @@ -3929,38 +3977,38 @@ compiler_assert(struct compiler *c, stmt_ty s) if (c->c_optimize) return 1; NEW_JUMP_TARGET_LABEL(c, end); - if (!compiler_jump_if(c, s->v.Assert.test, end, 1)) + if (!compiler_jump_if(c, &loc, s->v.Assert.test, end, 1)) return 0; - ADDOP(c, LOAD_ASSERTION_ERROR); + ADDOP(c, loc, LOAD_ASSERTION_ERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); - ADDOP_I(c, CALL, 0); + ADDOP_I(c, loc, CALL, 0); } - ADDOP_I(c, RAISE_VARARGS, 1); + ADDOP_I(c, loc, RAISE_VARARGS, 1); USE_LABEL(c, end); return 1; } static int -compiler_visit_stmt_expr(struct compiler *c, expr_ty value) +compiler_stmt_expr(struct compiler *c, location loc, expr_ty value) { if (c->c_interactive && c->c_nestlevel <= 1) { VISIT(c, expr, value); - ADDOP(c, PRINT_EXPR); + ADDOP(c, loc, PRINT_EXPR); return 1; } if (value->kind == Constant_kind) { /* ignore constant statement */ - ADDOP(c, NOP); + ADDOP(c, loc, NOP); return 1; } VISIT(c, expr, value); /* Mark POP_TOP as artificial */ UNSET_LOC(c); - ADDOP(c, POP_TOP); + ADDOP(c, NO_LOCATION, POP_TOP); return 1; } @@ -3968,7 +4016,6 @@ static int compiler_visit_stmt(struct compiler *c, stmt_ty s) { Py_ssize_t i, n; - /* Always assign a lineno to the next instruction for a stmt. */ SET_LOC(c, s); @@ -3983,16 +4030,19 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) VISIT_SEQ(c, expr, s->v.Delete.targets) break; case Assign_kind: + { n = asdl_seq_LEN(s->v.Assign.targets); VISIT(c, expr, s->v.Assign.value); + location loc = LOC(s); for (i = 0; i < n; i++) { if (i < n - 1) { - ADDOP_I(c, COPY, 1); + ADDOP_I(c, loc, COPY, 1); } VISIT(c, expr, (expr_ty)asdl_seq_GET(s->v.Assign.targets, i)); } break; + } case AugAssign_kind: return compiler_augassign(c, s); case AnnAssign_kind: @@ -4006,6 +4056,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Match_kind: return compiler_match(c, s); case Raise_kind: + { n = 0; if (s->v.Raise.exc) { VISIT(c, expr, s->v.Raise.exc); @@ -4015,8 +4066,10 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) n++; } } - ADDOP_I(c, RAISE_VARARGS, (int)n); + location loc = LOC(s); + ADDOP_I(c, loc, RAISE_VARARGS, (int)n); break; + } case Try_kind: return compiler_try(c, s); case TryStar_kind: @@ -4031,14 +4084,26 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Nonlocal_kind: break; case Expr_kind: - return compiler_visit_stmt_expr(c, s->v.Expr.value); + { + location loc = LOC(s); + return compiler_stmt_expr(c, loc, s->v.Expr.value); + } case Pass_kind: - ADDOP(c, NOP); + { + location loc = LOC(s); + ADDOP(c, loc, NOP); break; + } case Break_kind: - return compiler_break(c); + { + location loc = LOC(s); + return compiler_break(c, loc); + } case Continue_kind: - return compiler_continue(c); + { + location loc = LOC(s); + return compiler_continue(c, loc); + } case With_kind: return compiler_with(c, s, 0); case AsyncFunctionDef_kind: @@ -4072,7 +4137,8 @@ unaryop(unaryop_ty op) } static int -addop_binary(struct compiler *c, operator_ty binop, bool inplace) +addop_binary(struct compiler *c, location loc, operator_ty binop, + bool inplace) { int oparg; switch (binop) { @@ -4120,23 +4186,24 @@ addop_binary(struct compiler *c, operator_ty binop, bool inplace) inplace ? "inplace" : "binary", binop); return 0; } - ADDOP_I(c, BINARY_OP, oparg); + ADDOP_I(c, loc, BINARY_OP, oparg); return 1; } static int -addop_yield(struct compiler *c) { +addop_yield(struct compiler *c, location loc) { if (c->u->u_ste->ste_generator && c->u->u_ste->ste_coroutine) { - ADDOP(c, ASYNC_GEN_WRAP); + ADDOP(c, loc, ASYNC_GEN_WRAP); } - ADDOP_I(c, YIELD_VALUE, 0); - ADDOP_I(c, RESUME, 1); + ADDOP_I(c, loc, YIELD_VALUE, 0); + ADDOP_I(c, loc, RESUME, 1); return 1; } static int -compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) +compiler_nameop(struct compiler *c, location loc, + identifier name, expr_context_ty ctx) { int op, scope; Py_ssize_t arg; @@ -4149,7 +4216,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) !_PyUnicode_EqualToASCIIString(name, "True") && !_PyUnicode_EqualToASCIIString(name, "False")); - if (forbidden_name(c, name, ctx)) + if (forbidden_name(c, loc, name, ctx)) return 0; mangled = _Py_Mangle(c->u->u_private, name); @@ -4203,7 +4270,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) case Store: op = STORE_FAST; break; case Del: op = DELETE_FAST; break; } - ADDOP_N(c, op, mangled, varnames); + ADDOP_N(c, loc, op, mangled, varnames); return 1; case OP_GLOBAL: switch (ctx) { @@ -4230,7 +4297,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) if (op == LOAD_GLOBAL) { arg <<= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), op, arg, COMPILER_LOC(c)); + return cfg_builder_addop_i(CFG_BUILDER(c), op, arg, loc); } static int @@ -4240,6 +4307,7 @@ compiler_boolop(struct compiler *c, expr_ty e) Py_ssize_t i, n; asdl_expr_seq *s; + location loc = LOC(e); assert(e->kind == BoolOp_kind); if (e->v.BoolOp.op == And) jumpi = JUMP_IF_FALSE_OR_POP; @@ -4251,7 +4319,7 @@ compiler_boolop(struct compiler *c, expr_ty e) assert(n >= 0); for (i = 0; i < n; ++i) { VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i)); - ADDOP_JUMP(c, jumpi, end); + ADDOP_JUMP(c, loc, jumpi, end); NEW_JUMP_TARGET_LABEL(c, next); USE_LABEL(c, next); @@ -4263,7 +4331,8 @@ compiler_boolop(struct compiler *c, expr_ty e) } static int -starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed, +starunpack_helper(struct compiler *c, location loc, + asdl_expr_seq *elts, int pushed, int build, int add, int extend, int tuple) { Py_ssize_t n = asdl_seq_LEN(elts); @@ -4279,7 +4348,7 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed, PyTuple_SET_ITEM(folded, i, val); } if (tuple && !pushed) { - ADDOP_LOAD_CONST_NEW(c, folded); + ADDOP_LOAD_CONST_NEW(c, loc, folded); } else { if (add == SET_ADD) { Py_SETREF(folded, PyFrozenSet_New(folded)); @@ -4287,11 +4356,11 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed, return 0; } } - ADDOP_I(c, build, pushed); - ADDOP_LOAD_CONST_NEW(c, folded); - ADDOP_I(c, extend, 1); + ADDOP_I(c, loc, build, pushed); + ADDOP_LOAD_CONST_NEW(c, loc, folded); + ADDOP_I(c, loc, extend, 1); if (tuple) { - ADDOP(c, LIST_TO_TUPLE); + ADDOP(c, loc, LIST_TO_TUPLE); } } return 1; @@ -4312,43 +4381,43 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed, VISIT(c, expr, elt); } if (tuple) { - ADDOP_I(c, BUILD_TUPLE, n+pushed); + ADDOP_I(c, loc, BUILD_TUPLE, n+pushed); } else { - ADDOP_I(c, build, n+pushed); + ADDOP_I(c, loc, build, n+pushed); } return 1; } int sequence_built = 0; if (big) { - ADDOP_I(c, build, pushed); + ADDOP_I(c, loc, build, pushed); sequence_built = 1; } for (Py_ssize_t i = 0; i < n; i++) { expr_ty elt = asdl_seq_GET(elts, i); if (elt->kind == Starred_kind) { if (sequence_built == 0) { - ADDOP_I(c, build, i+pushed); + ADDOP_I(c, loc, build, i+pushed); sequence_built = 1; } VISIT(c, expr, elt->v.Starred.value); - ADDOP_I(c, extend, 1); + ADDOP_I(c, loc, extend, 1); } else { VISIT(c, expr, elt); if (sequence_built) { - ADDOP_I(c, add, 1); + ADDOP_I(c, loc, add, 1); } } } assert(sequence_built); if (tuple) { - ADDOP(c, LIST_TO_TUPLE); + ADDOP(c, loc, LIST_TO_TUPLE); } return 1; } static int -unpack_helper(struct compiler *c, asdl_expr_seq *elts) +unpack_helper(struct compiler *c, location loc, asdl_expr_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); int seen_star = 0; @@ -4357,28 +4426,28 @@ unpack_helper(struct compiler *c, asdl_expr_seq *elts) if (elt->kind == Starred_kind && !seen_star) { if ((i >= (1 << 8)) || (n-i-1 >= (INT_MAX >> 8))) - return compiler_error(c, + return compiler_error(c, loc, "too many expressions in " "star-unpacking assignment"); - ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8))); + ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); seen_star = 1; } else if (elt->kind == Starred_kind) { - return compiler_error(c, + return compiler_error(c, loc, "multiple starred expressions in assignment"); } } if (!seen_star) { - ADDOP_I(c, UNPACK_SEQUENCE, n); + ADDOP_I(c, loc, UNPACK_SEQUENCE, n); } return 1; } static int -assignment_helper(struct compiler *c, asdl_expr_seq *elts) +assignment_helper(struct compiler *c, location loc, asdl_expr_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); - RETURN_IF_FALSE(unpack_helper(c, elts)); + RETURN_IF_FALSE(unpack_helper(c, loc, elts)); for (Py_ssize_t i = 0; i < n; i++) { expr_ty elt = asdl_seq_GET(elts, i); VISIT(c, expr, elt->kind != Starred_kind ? elt : elt->v.Starred.value); @@ -4389,13 +4458,14 @@ assignment_helper(struct compiler *c, asdl_expr_seq *elts) static int compiler_list(struct compiler *c, expr_ty e) { + location loc = LOC(e); asdl_expr_seq *elts = e->v.List.elts; if (e->v.List.ctx == Store) { - return assignment_helper(c, elts); + return assignment_helper(c, loc, elts); } else if (e->v.List.ctx == Load) { - return starunpack_helper(c, elts, 0, BUILD_LIST, - LIST_APPEND, LIST_EXTEND, 0); + return starunpack_helper(c, loc, elts, 0, + BUILD_LIST, LIST_APPEND, LIST_EXTEND, 0); } else VISIT_SEQ(c, expr, elts); @@ -4405,13 +4475,14 @@ compiler_list(struct compiler *c, expr_ty e) static int compiler_tuple(struct compiler *c, expr_ty e) { + location loc = LOC(e); asdl_expr_seq *elts = e->v.Tuple.elts; if (e->v.Tuple.ctx == Store) { - return assignment_helper(c, elts); + return assignment_helper(c, loc, elts); } else if (e->v.Tuple.ctx == Load) { - return starunpack_helper(c, elts, 0, BUILD_LIST, - LIST_APPEND, LIST_EXTEND, 1); + return starunpack_helper(c, loc, elts, 0, + BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1); } else VISIT_SEQ(c, expr, elts); @@ -4421,8 +4492,9 @@ compiler_tuple(struct compiler *c, expr_ty e) static int compiler_set(struct compiler *c, expr_ty e) { - return starunpack_helper(c, e->v.Set.elts, 0, BUILD_SET, - SET_ADD, SET_UPDATE, 0); + location loc = LOC(e); + return starunpack_helper(c, loc, e->v.Set.elts, 0, + BUILD_SET, SET_ADD, SET_UPDATE, 0); } static int @@ -4443,6 +4515,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end Py_ssize_t i, n = end - begin; PyObject *keys, *key; int big = n*2 > STACK_USE_GUIDELINE; + location loc = LOC(e); if (n > 1 && !big && are_all_items_const(e->v.Dict.keys, begin, end)) { for (i = begin; i < end; i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); @@ -4456,22 +4529,22 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end Py_INCREF(key); PyTuple_SET_ITEM(keys, i - begin, key); } - ADDOP_LOAD_CONST_NEW(c, keys); - ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + ADDOP_LOAD_CONST_NEW(c, loc, keys); + ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); return 1; } if (big) { - ADDOP_I(c, BUILD_MAP, 0); + ADDOP_I(c, loc, BUILD_MAP, 0); } for (i = begin; i < end; i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); if (big) { - ADDOP_I(c, MAP_ADD, 1); + ADDOP_I(c, loc, MAP_ADD, 1); } } if (!big) { - ADDOP_I(c, BUILD_MAP, n); + ADDOP_I(c, loc, BUILD_MAP, n); } return 1; } @@ -4479,6 +4552,7 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end static int compiler_dict(struct compiler *c, expr_ty e) { + location loc = LOC(e); Py_ssize_t i, n, elements; int have_dict; int is_unpacking = 0; @@ -4493,17 +4567,17 @@ compiler_dict(struct compiler *c, expr_ty e) return 0; } if (have_dict) { - ADDOP_I(c, DICT_UPDATE, 1); + ADDOP_I(c, loc, DICT_UPDATE, 1); } have_dict = 1; elements = 0; } if (have_dict == 0) { - ADDOP_I(c, BUILD_MAP, 0); + ADDOP_I(c, loc, BUILD_MAP, 0); have_dict = 1; } VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); - ADDOP_I(c, DICT_UPDATE, 1); + ADDOP_I(c, loc, DICT_UPDATE, 1); } else { if (elements*2 > STACK_USE_GUIDELINE) { @@ -4511,7 +4585,7 @@ compiler_dict(struct compiler *c, expr_ty e) return 0; } if (have_dict) { - ADDOP_I(c, DICT_UPDATE, 1); + ADDOP_I(c, loc, DICT_UPDATE, 1); } have_dict = 1; elements = 0; @@ -4526,12 +4600,12 @@ compiler_dict(struct compiler *c, expr_ty e) return 0; } if (have_dict) { - ADDOP_I(c, DICT_UPDATE, 1); + ADDOP_I(c, loc, DICT_UPDATE, 1); } have_dict = 1; } if (!have_dict) { - ADDOP_I(c, BUILD_MAP, 0); + ADDOP_I(c, loc, BUILD_MAP, 0); } return 1; } @@ -4539,6 +4613,7 @@ compiler_dict(struct compiler *c, expr_ty e) static int compiler_compare(struct compiler *c, expr_ty e) { + location loc = LOC(e); Py_ssize_t i, n; if (!check_compare(c, e)) { @@ -4549,26 +4624,26 @@ compiler_compare(struct compiler *c, expr_ty e) n = asdl_seq_LEN(e->v.Compare.ops) - 1; if (n == 0) { VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0)); - ADDOP_COMPARE(c, asdl_seq_GET(e->v.Compare.ops, 0)); + ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, 0)); } else { NEW_JUMP_TARGET_LABEL(c, cleanup); for (i = 0; i < n; i++) { VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); - ADDOP_I(c, SWAP, 2); - ADDOP_I(c, COPY, 2); - ADDOP_COMPARE(c, asdl_seq_GET(e->v.Compare.ops, i)); - ADDOP_JUMP(c, JUMP_IF_FALSE_OR_POP, cleanup); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_I(c, loc, COPY, 2); + ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, i)); + ADDOP_JUMP(c, loc, JUMP_IF_FALSE_OR_POP, cleanup); } VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); - ADDOP_COMPARE(c, asdl_seq_GET(e->v.Compare.ops, n)); + ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, n)); NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP_NOLINE(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); USE_LABEL(c, cleanup); - ADDOP_I(c, SWAP, 2); - ADDOP(c, POP_TOP); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, POP_TOP); USE_LABEL(c, end); } @@ -4618,10 +4693,12 @@ check_caller(struct compiler *c, expr_ty e) case SetComp_kind: case GeneratorExp_kind: case JoinedStr_kind: - case FormattedValue_kind: - return compiler_warn(c, "'%.200s' object is not callable; " - "perhaps you missed a comma?", - infer_type(e)->tp_name); + case FormattedValue_kind: { + location loc = LOC(e); + return compiler_warn(c, loc, "'%.200s' object is not callable; " + "perhaps you missed a comma?", + infer_type(e)->tp_name); + } default: return 1; } @@ -4645,10 +4722,12 @@ check_subscripter(struct compiler *c, expr_ty e) case Set_kind: case SetComp_kind: case GeneratorExp_kind: - case Lambda_kind: - return compiler_warn(c, "'%.200s' object is not subscriptable; " - "perhaps you missed a comma?", - infer_type(e)->tp_name); + case Lambda_kind: { + location loc = LOC(e); + return compiler_warn(c, loc, "'%.200s' object is not subscriptable; " + "perhaps you missed a comma?", + infer_type(e)->tp_name); + } default: return 1; } @@ -4677,12 +4756,14 @@ check_index(struct compiler *c, expr_ty e, expr_ty s) case List_kind: case ListComp_kind: case JoinedStr_kind: - case FormattedValue_kind: - return compiler_warn(c, "%.200s indices must be integers or slices, " - "not %.200s; " - "perhaps you missed a comma?", - infer_type(e)->tp_name, - index_type->tp_name); + case FormattedValue_kind: { + location loc = LOC(e); + return compiler_warn(c, loc, "%.200s indices must be integers " + "or slices, not %.200s; " + "perhaps you missed a comma?", + infer_type(e)->tp_name, + index_type->tp_name); + } default: return 1; } @@ -4707,29 +4788,30 @@ is_import_originated(struct compiler *c, expr_ty e) // If an attribute access spans multiple lines, update the current start // location to point to the attribute name. -static void -update_start_location_to_match_attr(struct compiler *c, expr_ty attr) +static location +update_start_location_to_match_attr(struct compiler *c, location loc, + expr_ty attr) { assert(attr->kind == Attribute_kind); - struct location *loc = &c->u->u_loc; - if (loc->lineno != attr->end_lineno) { - loc->lineno = attr->end_lineno; + if (loc.lineno != attr->end_lineno) { + loc.lineno = attr->end_lineno; int len = (int)PyUnicode_GET_LENGTH(attr->v.Attribute.attr); if (len <= attr->end_col_offset) { - loc->col_offset = attr->end_col_offset - len; + loc.col_offset = attr->end_col_offset - len; } else { // GH-94694: Somebody's compiling weird ASTs. Just drop the columns: - loc->col_offset = -1; - loc->end_col_offset = -1; + loc.col_offset = -1; + loc.end_col_offset = -1; } // Make sure the end position still follows the start position, even for // weird ASTs: - loc->end_lineno = Py_MAX(loc->lineno, loc->end_lineno); - if (loc->lineno == loc->end_lineno) { - loc->end_col_offset = Py_MAX(loc->col_offset, loc->end_col_offset); + loc.end_lineno = Py_MAX(loc.lineno, loc.end_lineno); + if (loc.lineno == loc.end_lineno) { + loc.end_col_offset = Py_MAX(loc.col_offset, loc.end_col_offset); } } + return loc; } // Return 1 if the method call was optimized, -1 if not, and 0 on error. @@ -4774,19 +4856,21 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) /* Alright, we can optimize the code. */ VISIT(c, expr, meth->v.Attribute.value); SET_LOC(c, meth); - update_start_location_to_match_attr(c, meth); - ADDOP_NAME(c, LOAD_METHOD, meth->v.Attribute.attr, names); + location loc = LOC(meth); + loc = update_start_location_to_match_attr(c, loc, meth); + ADDOP_NAME(c, loc, LOAD_METHOD, meth->v.Attribute.attr, names); VISIT_SEQ(c, expr, e->v.Call.args); if (kwdsl) { VISIT_SEQ(c, keyword, kwds); - if (!compiler_call_simple_kw_helper(c, kwds, kwdsl)) { + if (!compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)) { return 0; }; } SET_LOC(c, e); - update_start_location_to_match_attr(c, meth); - ADDOP_I(c, CALL, argsl + kwdsl); + loc = LOC(e); + loc = update_start_location_to_match_attr(c, loc, meth); + ADDOP_I(c, loc, CALL, argsl + kwdsl); return 1; } @@ -4799,14 +4883,15 @@ validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) if (key->arg == NULL) { continue; } - if (forbidden_name(c, key->arg, Store)) { + location loc = LOC(key); + if (forbidden_name(c, loc, key->arg, Store)) { return -1; } for (Py_ssize_t j = i + 1; j < nkeywords; j++) { keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { SET_LOC(c, other); - compiler_error(c, "keyword argument repeated: %U", key->arg); + compiler_error(c, LOC(other), "keyword argument repeated: %U", key->arg); return -1; } } @@ -4828,10 +4913,12 @@ compiler_call(struct compiler *c, expr_ty e) return 0; } SET_LOC(c, e->v.Call.func); - ADDOP(c, PUSH_NULL); + location loc = LOC(e->v.Call.func); + ADDOP(c, loc, PUSH_NULL); SET_LOC(c, e); VISIT(c, expr, e->v.Call.func); - return compiler_call_helper(c, 0, + loc = LOC(e); + return compiler_call_helper(c, loc, 0, e->v.Call.args, e->v.Call.keywords); } @@ -4839,23 +4926,23 @@ compiler_call(struct compiler *c, expr_ty e) static int compiler_joined_str(struct compiler *c, expr_ty e) { - + location loc = LOC(e); Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values); if (value_count > STACK_USE_GUIDELINE) { _Py_DECLARE_STR(empty, ""); - ADDOP_LOAD_CONST_NEW(c, Py_NewRef(&_Py_STR(empty))); - ADDOP_NAME(c, LOAD_METHOD, &_Py_ID(join), names); - ADDOP_I(c, BUILD_LIST, 0); + ADDOP_LOAD_CONST_NEW(c, loc, Py_NewRef(&_Py_STR(empty))); + ADDOP_NAME(c, loc, LOAD_METHOD, &_Py_ID(join), names); + ADDOP_I(c, loc, BUILD_LIST, 0); for (Py_ssize_t i = 0; i < asdl_seq_LEN(e->v.JoinedStr.values); i++) { VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); - ADDOP_I(c, LIST_APPEND, 1); + ADDOP_I(c, loc, LIST_APPEND, 1); } - ADDOP_I(c, CALL, 1); + ADDOP_I(c, loc, CALL, 1); } else { VISIT_SEQ(c, expr, e->v.JoinedStr.values); if (asdl_seq_LEN(e->v.JoinedStr.values) != 1) { - ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); + ADDOP_I(c, loc, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values)); } } return 1; @@ -4902,13 +4989,16 @@ compiler_formatted_value(struct compiler *c, expr_ty e) } /* And push our opcode and oparg */ - ADDOP_I(c, FORMAT_VALUE, oparg); + location loc = LOC(e); + ADDOP_I(c, loc, FORMAT_VALUE, oparg); return 1; } static int -compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t begin, Py_ssize_t end) +compiler_subkwargs(struct compiler *c, location loc, + asdl_keyword_seq *keywords, + Py_ssize_t begin, Py_ssize_t end) { Py_ssize_t i, n = end - begin; keyword_ty kw; @@ -4929,23 +5019,23 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be Py_INCREF(key); PyTuple_SET_ITEM(keys, i - begin, key); } - ADDOP_LOAD_CONST_NEW(c, keys); - ADDOP_I(c, BUILD_CONST_KEY_MAP, n); + ADDOP_LOAD_CONST_NEW(c, loc, keys); + ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); return 1; } if (big) { - ADDOP_I_NOLINE(c, BUILD_MAP, 0); + ADDOP_I(c, NO_LOCATION, BUILD_MAP, 0); } for (i = begin; i < end; i++) { kw = asdl_seq_GET(keywords, i); - ADDOP_LOAD_CONST(c, kw->arg); + ADDOP_LOAD_CONST(c, loc, kw->arg); VISIT(c, expr, kw->value); if (big) { - ADDOP_I_NOLINE(c, MAP_ADD, 1); + ADDOP_I(c, NO_LOCATION, MAP_ADD, 1); } } if (!big) { - ADDOP_I(c, BUILD_MAP, n); + ADDOP_I(c, loc, BUILD_MAP, n); } return 1; } @@ -4955,9 +5045,8 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be * Returns 1 on success, 0 on error. */ static int -compiler_call_simple_kw_helper(struct compiler *c, - asdl_keyword_seq *keywords, - Py_ssize_t nkwelts) +compiler_call_simple_kw_helper(struct compiler *c, location loc, + asdl_keyword_seq *keywords, Py_ssize_t nkwelts) { PyObject *names; names = PyTuple_New(nkwelts); @@ -4974,14 +5063,14 @@ compiler_call_simple_kw_helper(struct compiler *c, return 0; } Py_DECREF(names); - ADDOP_I(c, KW_NAMES, arg); + ADDOP_I(c, loc, KW_NAMES, arg); return 1; } /* shared code between compiler_call and compiler_class */ static int -compiler_call_helper(struct compiler *c, +compiler_call_helper(struct compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, asdl_keyword_seq *keywords) @@ -5019,11 +5108,11 @@ compiler_call_helper(struct compiler *c, } if (nkwelts) { VISIT_SEQ(c, keyword, keywords); - if (!compiler_call_simple_kw_helper(c, keywords, nkwelts)) { + if (!compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)) { return 0; }; } - ADDOP_I(c, CALL, n + nelts + nkwelts); + ADDOP_I(c, loc, CALL, n + nelts + nkwelts); return 1; ex_call: @@ -5032,7 +5121,7 @@ compiler_call_helper(struct compiler *c, if (n ==0 && nelts == 1 && ((expr_ty)asdl_seq_GET(args, 0))->kind == Starred_kind) { VISIT(c, expr, ((expr_ty)asdl_seq_GET(args, 0))->v.Starred.value); } - else if (starunpack_helper(c, args, n, BUILD_LIST, + else if (starunpack_helper(c, loc, args, n, BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1) == 0) { return 0; } @@ -5047,21 +5136,21 @@ compiler_call_helper(struct compiler *c, if (kw->arg == NULL) { /* A keyword argument unpacking. */ if (nseen) { - if (!compiler_subkwargs(c, keywords, i - nseen, i)) { + if (!compiler_subkwargs(c, loc, keywords, i - nseen, i)) { return 0; } if (have_dict) { - ADDOP_I(c, DICT_MERGE, 1); + ADDOP_I(c, loc, DICT_MERGE, 1); } have_dict = 1; nseen = 0; } if (!have_dict) { - ADDOP_I(c, BUILD_MAP, 0); + ADDOP_I(c, loc, BUILD_MAP, 0); have_dict = 1; } VISIT(c, expr, kw->value); - ADDOP_I(c, DICT_MERGE, 1); + ADDOP_I(c, loc, DICT_MERGE, 1); } else { nseen++; @@ -5069,17 +5158,17 @@ compiler_call_helper(struct compiler *c, } if (nseen) { /* Pack up any trailing keyword arguments. */ - if (!compiler_subkwargs(c, keywords, nkwelts - nseen, nkwelts)) { + if (!compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)) { return 0; } if (have_dict) { - ADDOP_I(c, DICT_MERGE, 1); + ADDOP_I(c, loc, DICT_MERGE, 1); } have_dict = 1; } assert(have_dict); } - ADDOP_I(c, CALL_FUNCTION_EX, nkwelts > 0); + ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0); return 1; } @@ -5099,7 +5188,7 @@ compiler_call_helper(struct compiler *c, static int -compiler_comprehension_generator(struct compiler *c, +compiler_comprehension_generator(struct compiler *c, location *ploc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type) @@ -5108,15 +5197,15 @@ compiler_comprehension_generator(struct compiler *c, gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); if (gen->is_async) { return compiler_async_comprehension_generator( - c, generators, gen_index, depth, elt, val, type); + c, ploc, generators, gen_index, depth, elt, val, type); } else { return compiler_sync_comprehension_generator( - c, generators, gen_index, depth, elt, val, type); + c, ploc, generators, gen_index, depth, elt, val, type); } } static int -compiler_sync_comprehension_generator(struct compiler *c, +compiler_sync_comprehension_generator(struct compiler *c, location *ploc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type) @@ -5136,7 +5225,7 @@ compiler_sync_comprehension_generator(struct compiler *c, if (gen_index == 0) { /* Receive outermost iter as an implicit argument */ c->u->u_argcount = 1; - ADDOP_I(c, LOAD_FAST, 0); + ADDOP_I(c, *ploc, LOAD_FAST, 0); } else { /* Sub-iter - calculate on the fly */ @@ -5163,13 +5252,13 @@ compiler_sync_comprehension_generator(struct compiler *c, } if (IS_LABEL(start)) { VISIT(c, expr, gen->iter); - ADDOP(c, GET_ITER); + ADDOP(c, *ploc, GET_ITER); } } if (IS_LABEL(start)) { depth++; USE_LABEL(c, start); - ADDOP_JUMP(c, FOR_ITER, anchor); + ADDOP_JUMP(c, *ploc, FOR_ITER, anchor); } VISIT(c, expr, gen->target); @@ -5177,12 +5266,12 @@ compiler_sync_comprehension_generator(struct compiler *c, n = asdl_seq_LEN(gen->ifs); for (i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - if (!compiler_jump_if(c, e, if_cleanup, 0)) + if (!compiler_jump_if(c, ploc, e, if_cleanup, 0)) return 0; } if (++gen_index < asdl_seq_LEN(generators)) - if (!compiler_comprehension_generator(c, + if (!compiler_comprehension_generator(c, ploc, generators, gen_index, depth, elt, val, type)) return 0; @@ -5193,23 +5282,23 @@ compiler_sync_comprehension_generator(struct compiler *c, switch (type) { case COMP_GENEXP: VISIT(c, expr, elt); - ADDOP_YIELD(c); - ADDOP(c, POP_TOP); + ADDOP_YIELD(c, *ploc); + ADDOP(c, *ploc, POP_TOP); break; case COMP_LISTCOMP: VISIT(c, expr, elt); - ADDOP_I(c, LIST_APPEND, depth + 1); + ADDOP_I(c, *ploc, LIST_APPEND, depth + 1); break; case COMP_SETCOMP: VISIT(c, expr, elt); - ADDOP_I(c, SET_ADD, depth + 1); + ADDOP_I(c, *ploc, SET_ADD, depth + 1); break; case COMP_DICTCOMP: /* With '{k: v}', k is evaluated before v, so we do the same. */ VISIT(c, expr, elt); VISIT(c, expr, val); - ADDOP_I(c, MAP_ADD, depth + 1); + ADDOP_I(c, *ploc, MAP_ADD, depth + 1); break; default: return 0; @@ -5218,7 +5307,7 @@ compiler_sync_comprehension_generator(struct compiler *c, USE_LABEL(c, if_cleanup); if (IS_LABEL(start)) { - ADDOP_JUMP(c, JUMP, start); + ADDOP_JUMP(c, *ploc, JUMP, start); USE_LABEL(c, anchor); } @@ -5227,7 +5316,7 @@ compiler_sync_comprehension_generator(struct compiler *c, } static int -compiler_async_comprehension_generator(struct compiler *c, +compiler_async_comprehension_generator(struct compiler *c, location *ploc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type) @@ -5243,38 +5332,38 @@ compiler_async_comprehension_generator(struct compiler *c, if (gen_index == 0) { /* Receive outermost iter as an implicit argument */ c->u->u_argcount = 1; - ADDOP_I(c, LOAD_FAST, 0); + ADDOP_I(c, *ploc, LOAD_FAST, 0); } else { /* Sub-iter - calculate on the fly */ VISIT(c, expr, gen->iter); - ADDOP(c, GET_AITER); + ADDOP(c, *ploc, GET_AITER); } USE_LABEL(c, start); /* Runtime will push a block here, so we need to account for that */ - if (!compiler_push_fblock(c, ASYNC_COMPREHENSION_GENERATOR, start, - NO_LABEL, NULL)) { + if (!compiler_push_fblock(c, *ploc, ASYNC_COMPREHENSION_GENERATOR, + start, NO_LABEL, NULL)) { return 0; } - ADDOP_JUMP(c, SETUP_FINALLY, except); - ADDOP(c, GET_ANEXT); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); - ADDOP(c, POP_BLOCK); + ADDOP_JUMP(c, *ploc, SETUP_FINALLY, except); + ADDOP(c, *ploc, GET_ANEXT); + ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADD_YIELD_FROM(c, *ploc, 1); + ADDOP(c, *ploc, POP_BLOCK); VISIT(c, expr, gen->target); n = asdl_seq_LEN(gen->ifs); for (i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - if (!compiler_jump_if(c, e, if_cleanup, 0)) + if (!compiler_jump_if(c, ploc, e, if_cleanup, 0)) return 0; } depth++; if (++gen_index < asdl_seq_LEN(generators)) - if (!compiler_comprehension_generator(c, + if (!compiler_comprehension_generator(c, ploc, generators, gen_index, depth, elt, val, type)) return 0; @@ -5285,23 +5374,23 @@ compiler_async_comprehension_generator(struct compiler *c, switch (type) { case COMP_GENEXP: VISIT(c, expr, elt); - ADDOP_YIELD(c); - ADDOP(c, POP_TOP); + ADDOP_YIELD(c, *ploc); + ADDOP(c, *ploc, POP_TOP); break; case COMP_LISTCOMP: VISIT(c, expr, elt); - ADDOP_I(c, LIST_APPEND, depth + 1); + ADDOP_I(c, *ploc, LIST_APPEND, depth + 1); break; case COMP_SETCOMP: VISIT(c, expr, elt); - ADDOP_I(c, SET_ADD, depth + 1); + ADDOP_I(c, *ploc, SET_ADD, depth + 1); break; case COMP_DICTCOMP: /* With '{k: v}', k is evaluated before v, so we do the same. */ VISIT(c, expr, elt); VISIT(c, expr, val); - ADDOP_I(c, MAP_ADD, depth + 1); + ADDOP_I(c, *ploc, MAP_ADD, depth + 1); break; default: return 0; @@ -5309,14 +5398,14 @@ compiler_async_comprehension_generator(struct compiler *c, } USE_LABEL(c, if_cleanup); - ADDOP_JUMP(c, JUMP, start); + ADDOP_JUMP(c, *ploc, JUMP, start); compiler_pop_fblock(c, ASYNC_COMPREHENSION_GENERATOR, start); USE_LABEL(c, except); //UNSET_LOC(c); - ADDOP(c, END_ASYNC_FOR); + ADDOP(c, *ploc, END_ASYNC_FOR); return 1; } @@ -5340,6 +5429,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, goto error; } SET_LOC(c, e); + location loc = LOC(e); is_async_generator = c->u->u_ste->ste_coroutine; @@ -5348,8 +5438,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, scope_type != COMPILER_SCOPE_COMPREHENSION && !is_top_level_await) { - compiler_error(c, "asynchronous comprehension outside of " - "an asynchronous function"); + compiler_error(c, loc, "asynchronous comprehension outside of " + "an asynchronous function"); goto error_in_scope; } @@ -5371,15 +5461,15 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, goto error_in_scope; } - ADDOP_I(c, op, 0); + ADDOP_I(c, loc, op, 0); } - if (!compiler_comprehension_generator(c, generators, 0, 0, elt, - val, type)) + if (!compiler_comprehension_generator(c, &loc, generators, 0, 0, + elt, val, type)) goto error_in_scope; if (type != COMP_GENEXP) { - ADDOP(c, RETURN_VALUE); + ADDOP(c, loc, RETURN_VALUE); } co = assemble(c, 1); @@ -5392,7 +5482,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, if (co == NULL) goto error; - if (!compiler_make_closure(c, co, 0, qualname)) { + loc = LOC(e); + if (!compiler_make_closure(c, loc, co, 0, qualname)) { goto error; } Py_DECREF(qualname); @@ -5400,18 +5491,19 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, VISIT(c, expr, outermost->iter); + loc = LOC(e); if (outermost->is_async) { - ADDOP(c, GET_AITER); + ADDOP(c, loc, GET_AITER); } else { - ADDOP(c, GET_ITER); + ADDOP(c, loc, GET_ITER); } - ADDOP_I(c, CALL, 0); + ADDOP_I(c, loc, CALL, 0); if (is_async_generator && type != COMP_GENEXP) { - ADDOP_I(c, GET_AWAITABLE, 0); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); + ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); } return 1; @@ -5477,20 +5569,20 @@ static int compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { UNSET_LOC(c); NEW_JUMP_TARGET_LABEL(c, suppress); - ADDOP_JUMP(c, POP_JUMP_IF_TRUE, suppress); - ADDOP_I(c, RERAISE, 2); + ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_TRUE, suppress); + ADDOP_I(c, NO_LOCATION, RERAISE, 2); USE_LABEL(c, suppress); - ADDOP(c, POP_TOP); /* exc_value */ - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); - ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); + ADDOP(c, NO_LOCATION, POP_TOP); /* exc_value */ + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_TOP); NEW_JUMP_TARGET_LABEL(c, exit); - ADDOP_JUMP(c, JUMP, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP, exit); USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c); + POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); USE_LABEL(c, exit); return 1; @@ -5523,13 +5615,14 @@ compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { static int compiler_async_with(struct compiler *c, stmt_ty s, int pos) { + location loc = LOC(s); withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); assert(s->kind == AsyncWith_kind); if (IS_TOP_LEVEL_AWAIT(c)){ c->u->u_ste->ste_coroutine = 1; } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){ - return compiler_error(c, "'async with' outside async function"); + return compiler_error(c, loc, "'async with' outside async function"); } NEW_JUMP_TARGET_LABEL(c, block); @@ -5540,16 +5633,16 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) /* Evaluate EXPR */ VISIT(c, expr, item->context_expr); - ADDOP(c, BEFORE_ASYNC_WITH); - ADDOP_I(c, GET_AWAITABLE, 1); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); + ADDOP(c, loc, BEFORE_ASYNC_WITH); + ADDOP_I(c, loc, GET_AWAITABLE, 1); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); - ADDOP_JUMP(c, SETUP_WITH, final); + ADDOP_JUMP(c, loc, SETUP_WITH, final); /* SETUP_WITH pushes a finally block. */ USE_LABEL(c, block); - if (!compiler_push_fblock(c, ASYNC_WITH, block, final, s)) { + if (!compiler_push_fblock(c, loc, ASYNC_WITH, block, final, s)) { return 0; } @@ -5558,43 +5651,46 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) } else { /* Discard result from context.__aenter__() */ - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); } pos++; - if (pos == asdl_seq_LEN(s->v.AsyncWith.items)) + if (pos == asdl_seq_LEN(s->v.AsyncWith.items)) { /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.AsyncWith.body) - else if (!compiler_async_with(c, s, pos)) + } + else if (!compiler_async_with(c, s, pos)) { return 0; + } compiler_pop_fblock(c, ASYNC_WITH, block); - ADDOP(c, POP_BLOCK); + + SET_LOC(c, s); + ADDOP(c, loc, POP_BLOCK); /* End of body; start the cleanup */ /* For successful outcome: * call __exit__(None, None, None) */ - SET_LOC(c, s); - if(!compiler_call_exit_with_nones(c)) + if(!compiler_call_exit_with_nones(c, loc)) return 0; - ADDOP_I(c, GET_AWAITABLE, 2); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); + ADDOP_I(c, loc, GET_AWAITABLE, 2); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); - ADDOP_JUMP(c, JUMP, exit); + ADDOP_JUMP(c, loc, JUMP, exit); /* For exceptional outcome: */ USE_LABEL(c, final); - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup); - ADDOP(c, PUSH_EXC_INFO); - ADDOP(c, WITH_EXCEPT_START); - ADDOP_I(c, GET_AWAITABLE, 2); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + ADDOP(c, loc, WITH_EXCEPT_START); + ADDOP_I(c, loc, GET_AWAITABLE, 2); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); compiler_with_except_finish(c, cleanup); USE_LABEL(c, exit); @@ -5638,12 +5734,13 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* Evaluate EXPR */ VISIT(c, expr, item->context_expr); /* Will push bound __exit__ */ - ADDOP(c, BEFORE_WITH); - ADDOP_JUMP(c, SETUP_WITH, final); + location loc = LOC(s); + ADDOP(c, loc, BEFORE_WITH); + ADDOP_JUMP(c, loc, SETUP_WITH, final); /* SETUP_WITH pushes a finally block. */ USE_LABEL(c, block); - if (!compiler_push_fblock(c, WITH, block, final, s)) { + if (!compiler_push_fblock(c, loc, WITH, block, final, s)) { return 0; } @@ -5652,7 +5749,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) } else { /* Discard result from context.__enter__() */ - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); } pos++; @@ -5665,7 +5762,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) /* Mark all following code as artificial */ UNSET_LOC(c); - ADDOP(c, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); compiler_pop_fblock(c, WITH, block); /* End of body; start the cleanup. */ @@ -5674,17 +5771,18 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) * call __exit__(None, None, None) */ SET_LOC(c, s); - if (!compiler_call_exit_with_nones(c)) + loc = LOC(s); + if (!compiler_call_exit_with_nones(c, loc)) return 0; - ADDOP(c, POP_TOP); - ADDOP_JUMP(c, JUMP, exit); + ADDOP(c, loc, POP_TOP); + ADDOP_JUMP(c, loc, JUMP, exit); /* For exceptional outcome: */ USE_LABEL(c, final); - ADDOP_JUMP(c, SETUP_CLEANUP, cleanup); - ADDOP(c, PUSH_EXC_INFO); - ADDOP(c, WITH_EXCEPT_START); + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + ADDOP(c, loc, WITH_EXCEPT_START); compiler_with_except_finish(c, cleanup); USE_LABEL(c, exit); @@ -5694,10 +5792,11 @@ compiler_with(struct compiler *c, stmt_ty s, int pos) static int compiler_visit_expr1(struct compiler *c, expr_ty e) { + location loc = LOC(e); switch (e->kind) { case NamedExpr_kind: VISIT(c, expr, e->v.NamedExpr.value); - ADDOP_I(c, COPY, 1); + ADDOP_I(c, loc, COPY, 1); VISIT(c, expr, e->v.NamedExpr.target); break; case BoolOp_kind: @@ -5705,11 +5804,11 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case BinOp_kind: VISIT(c, expr, e->v.BinOp.left); VISIT(c, expr, e->v.BinOp.right); - ADDOP_BINARY(c, e->v.BinOp.op); + ADDOP_BINARY(c, loc, e->v.BinOp.op); break; case UnaryOp_kind: VISIT(c, expr, e->v.UnaryOp.operand); - ADDOP(c, unaryop(e->v.UnaryOp.op)); + ADDOP(c, loc, unaryop(e->v.UnaryOp.op)); break; case Lambda_kind: return compiler_lambda(c, e); @@ -5729,50 +5828,50 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) return compiler_dictcomp(c, e); case Yield_kind: if (c->u->u_ste->ste_type != FunctionBlock) - return compiler_error(c, "'yield' outside function"); + return compiler_error(c, loc, "'yield' outside function"); if (e->v.Yield.value) { VISIT(c, expr, e->v.Yield.value); } else { - ADDOP_LOAD_CONST(c, Py_None); + ADDOP_LOAD_CONST(c, loc, Py_None); } - ADDOP_YIELD(c); + ADDOP_YIELD(c, loc); break; case YieldFrom_kind: if (c->u->u_ste->ste_type != FunctionBlock) - return compiler_error(c, "'yield' outside function"); + return compiler_error(c, loc, "'yield' outside function"); if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) - return compiler_error(c, "'yield from' inside async function"); + return compiler_error(c, loc, "'yield from' inside async function"); VISIT(c, expr, e->v.YieldFrom.value); - ADDOP(c, GET_YIELD_FROM_ITER); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 0); + ADDOP(c, loc, GET_YIELD_FROM_ITER); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: if (!IS_TOP_LEVEL_AWAIT(c)){ if (c->u->u_ste->ste_type != FunctionBlock){ - return compiler_error(c, "'await' outside function"); + return compiler_error(c, loc, "'await' outside function"); } if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){ - return compiler_error(c, "'await' outside async function"); + return compiler_error(c, loc, "'await' outside async function"); } } VISIT(c, expr, e->v.Await.value); - ADDOP_I(c, GET_AWAITABLE, 0); - ADDOP_LOAD_CONST(c, Py_None); - ADD_YIELD_FROM(c, 1); + ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); break; case Compare_kind: return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); case Constant_kind: - ADDOP_LOAD_CONST(c, e->v.Constant.value); + ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; case JoinedStr_kind: return compiler_joined_str(c, e); @@ -5781,21 +5880,20 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) /* The following exprs can be assignment targets. */ case Attribute_kind: VISIT(c, expr, e->v.Attribute.value); - update_start_location_to_match_attr(c, e); + loc = LOC(e); + loc = update_start_location_to_match_attr(c, loc, e); switch (e->v.Attribute.ctx) { case Load: - { - ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names); + ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); break; - } case Store: - if (forbidden_name(c, e->v.Attribute.attr, e->v.Attribute.ctx)) { + if (forbidden_name(c, loc, e->v.Attribute.attr, e->v.Attribute.ctx)) { return 0; } - ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); + ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); break; case Del: - ADDOP_NAME(c, DELETE_ATTR, e->v.Attribute.attr, names); + ADDOP_NAME(c, loc, DELETE_ATTR, e->v.Attribute.attr, names); break; } break; @@ -5806,10 +5904,10 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) case Store: /* In all legitimate cases, the Starred node was already replaced * by compiler_list/compiler_tuple. XXX: is that okay? */ - return compiler_error(c, + return compiler_error(c, loc, "starred assignment target must be in a list or tuple"); default: - return compiler_error(c, + return compiler_error(c, loc, "can't use starred expression here"); } break; @@ -5819,11 +5917,11 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) if (n == 0) { return 0; } - ADDOP_I(c, BUILD_SLICE, n); + ADDOP_I(c, loc, BUILD_SLICE, n); break; } case Name_kind: - return compiler_nameop(c, e->v.Name.id, e->v.Name.ctx); + return compiler_nameop(c, loc, e->v.Name.id, e->v.Name.ctx); /* child nodes of List and Tuple will have expr_context set */ case List_kind: return compiler_list(c, e); @@ -5836,10 +5934,8 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) static int compiler_visit_expr(struct compiler *c, expr_ty e) { - struct location old_loc = c->u->u_loc; SET_LOC(c, e); int res = compiler_visit_expr1(c, e); - c->u->u_loc = old_loc; return res; } @@ -5856,15 +5952,15 @@ compiler_augassign(struct compiler *c, stmt_ty s) assert(s->kind == AugAssign_kind); expr_ty e = s->v.AugAssign.target; - struct location old_loc = c->u->u_loc; + location loc = LOC(e); SET_LOC(c, e); switch (e->kind) { case Attribute_kind: VISIT(c, expr, e->v.Attribute.value); - ADDOP_I(c, COPY, 1); - update_start_location_to_match_attr(c, e); - ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names); + ADDOP_I(c, loc, COPY, 1); + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); break; case Subscript_kind: VISIT(c, expr, e->v.Subscript.value); @@ -5872,20 +5968,20 @@ compiler_augassign(struct compiler *c, stmt_ty s) if (!compiler_slice(c, e->v.Subscript.slice)) { return 0; } - ADDOP_I(c, COPY, 3); - ADDOP_I(c, COPY, 3); - ADDOP_I(c, COPY, 3); - ADDOP(c, BINARY_SLICE); + ADDOP_I(c, loc, COPY, 3); + ADDOP_I(c, loc, COPY, 3); + ADDOP_I(c, loc, COPY, 3); + ADDOP(c, loc, BINARY_SLICE); } else { VISIT(c, expr, e->v.Subscript.slice); - ADDOP_I(c, COPY, 2); - ADDOP_I(c, COPY, 2); - ADDOP(c, BINARY_SUBSCR); + ADDOP_I(c, loc, COPY, 2); + ADDOP_I(c, loc, COPY, 2); + ADDOP(c, loc, BINARY_SUBSCR); } break; case Name_kind: - if (!compiler_nameop(c, e->v.Name.id, Load)) + if (!compiler_nameop(c, loc, e->v.Name.id, Load)) return 0; break; default: @@ -5895,34 +5991,35 @@ compiler_augassign(struct compiler *c, stmt_ty s) return 0; } - c->u->u_loc = old_loc; + loc = LOC(s); VISIT(c, expr, s->v.AugAssign.value); - ADDOP_INPLACE(c, s->v.AugAssign.op); + ADDOP_INPLACE(c, loc, s->v.AugAssign.op); SET_LOC(c, e); + loc = LOC(e); switch (e->kind) { case Attribute_kind: - update_start_location_to_match_attr(c, e); - ADDOP_I(c, SWAP, 2); - ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names); + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); break; case Subscript_kind: if (is_two_element_slice(e->v.Subscript.slice)) { - ADDOP_I(c, SWAP, 4); - ADDOP_I(c, SWAP, 3); - ADDOP_I(c, SWAP, 2); - ADDOP(c, STORE_SLICE); + ADDOP_I(c, loc, SWAP, 4); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, STORE_SLICE); } else { - ADDOP_I(c, SWAP, 3); - ADDOP_I(c, SWAP, 2); - ADDOP(c, STORE_SUBSCR); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, STORE_SUBSCR); } break; case Name_kind: - return compiler_nameop(c, e->v.Name.id, Store); + return compiler_nameop(c, loc, e->v.Name.id, Store); default: Py_UNREACHABLE(); } @@ -5933,7 +6030,7 @@ static int check_ann_expr(struct compiler *c, expr_ty e) { VISIT(c, expr, e); - ADDOP(c, POP_TOP); + ADDOP(c, LOC(e), POP_TOP); return 1; } @@ -5989,6 +6086,7 @@ check_ann_subscr(struct compiler *c, expr_ty e) static int compiler_annassign(struct compiler *c, stmt_ty s) { + location loc = LOC(s); expr_ty targ = s->v.AnnAssign.target; PyObject* mangled; @@ -6001,7 +6099,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) } switch (targ->kind) { case Name_kind: - if (forbidden_name(c, targ->v.Name.id, Store)) + if (forbidden_name(c, loc, targ->v.Name.id, Store)) return 0; /* If we have a simple name in a module or class, store annotation. */ if (s->v.AnnAssign.simple && @@ -6013,14 +6111,14 @@ compiler_annassign(struct compiler *c, stmt_ty s) else { VISIT(c, expr, s->v.AnnAssign.annotation); } - ADDOP_NAME(c, LOAD_NAME, &_Py_ID(__annotations__), names); + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); - ADDOP_LOAD_CONST_NEW(c, mangled); - ADDOP(c, STORE_SUBSCR); + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + ADDOP(c, loc, STORE_SUBSCR); } break; case Attribute_kind: - if (forbidden_name(c, targ->v.Attribute.attr, Store)) + if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) return 0; if (!s->v.AnnAssign.value && !check_ann_expr(c, targ->v.Attribute.value)) { @@ -6052,7 +6150,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) */ static int -compiler_error(struct compiler *c, const char *format, ...) +compiler_error(struct compiler *c, location loc, + const char *format, ...) { va_list vargs; va_start(vargs, format); @@ -6061,22 +6160,21 @@ compiler_error(struct compiler *c, const char *format, ...) if (msg == NULL) { return 0; } - PyObject *loc = PyErr_ProgramTextObject(c->c_filename, c->u->u_loc.lineno); - if (loc == NULL) { + PyObject *loc_obj = PyErr_ProgramTextObject(c->c_filename, loc.lineno); + if (loc_obj == NULL) { Py_INCREF(Py_None); - loc = Py_None; + loc_obj = Py_None; } - struct location u_loc = c->u->u_loc; PyObject *args = Py_BuildValue("O(OiiOii)", msg, c->c_filename, - u_loc.lineno, u_loc.col_offset + 1, loc, - u_loc.end_lineno, u_loc.end_col_offset + 1); + loc.lineno, loc.col_offset + 1, loc_obj, + loc.end_lineno, loc.end_col_offset + 1); Py_DECREF(msg); if (args == NULL) { goto exit; } PyErr_SetObject(PyExc_SyntaxError, args); exit: - Py_DECREF(loc); + Py_DECREF(loc_obj); Py_XDECREF(args); return 0; } @@ -6086,7 +6184,8 @@ compiler_error(struct compiler *c, const char *format, ...) and returns 0. */ static int -compiler_warn(struct compiler *c, const char *format, ...) +compiler_warn(struct compiler *c, location loc, + const char *format, ...) { va_list vargs; va_start(vargs, format); @@ -6096,14 +6195,14 @@ compiler_warn(struct compiler *c, const char *format, ...) return 0; } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, c->c_filename, - c->u->u_loc.lineno, NULL, NULL) < 0) + loc.lineno, NULL, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError to get a more accurate error report */ PyErr_Clear(); assert(PyUnicode_AsUTF8(msg) != NULL); - compiler_error(c, PyUnicode_AsUTF8(msg)); + compiler_error(c, loc, PyUnicode_AsUTF8(msg)); } Py_DECREF(msg); return 0; @@ -6115,6 +6214,7 @@ compiler_warn(struct compiler *c, const char *format, ...) static int compiler_subscript(struct compiler *c, expr_ty e) { + location loc = LOC(e); expr_context_ty ctx = e->v.Subscript.ctx; int op = 0; @@ -6133,11 +6233,11 @@ compiler_subscript(struct compiler *c, expr_ty e) return 0; } if (ctx == Load) { - ADDOP(c, BINARY_SLICE); + ADDOP(c, loc, BINARY_SLICE); } else { assert(ctx == Store); - ADDOP(c, STORE_SLICE); + ADDOP(c, loc, STORE_SLICE); } } else { @@ -6148,7 +6248,7 @@ compiler_subscript(struct compiler *c, expr_ty e) case Del: op = DELETE_SUBSCR; break; } assert(op); - ADDOP(c, op); + ADDOP(c, loc, op); } return 1; } @@ -6166,14 +6266,14 @@ compiler_slice(struct compiler *c, expr_ty s) VISIT(c, expr, s->v.Slice.lower); } else { - ADDOP_LOAD_CONST(c, Py_None); + ADDOP_LOAD_CONST(c, LOC(s), Py_None); } if (s->v.Slice.upper) { VISIT(c, expr, s->v.Slice.upper); } else { - ADDOP_LOAD_CONST(c, Py_None); + ADDOP_LOAD_CONST(c, LOC(s), Py_None); } if (s->v.Slice.step) { @@ -6230,19 +6330,21 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) // Use op to jump to the correct fail_pop block. static int -jump_to_fail_pop(struct compiler *c, pattern_context *pc, int op) +jump_to_fail_pop(struct compiler *c, location loc, + pattern_context *pc, int op) { // Pop any items on the top of the stack, plus any objects we were going to // capture on success: Py_ssize_t pops = pc->on_top + PyList_GET_SIZE(pc->stores); RETURN_IF_FALSE(ensure_fail_pop(c, pc, pops)); - ADDOP_JUMP(c, op, pc->fail_pop[pops]); + ADDOP_JUMP(c, loc, op, pc->fail_pop[pops]); return 1; } // Build all of the fail_pop blocks and reset fail_pop. static int -emit_and_reset_fail_pop(struct compiler *c, pattern_context *pc) +emit_and_reset_fail_pop(struct compiler *c, location loc, + pattern_context *pc) { if (!pc->fail_pop_size) { assert(pc->fail_pop == NULL); @@ -6250,7 +6352,7 @@ emit_and_reset_fail_pop(struct compiler *c, pattern_context *pc) } while (--pc->fail_pop_size) { USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, COMPILER_LOC(c))) { + if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc)) { pc->fail_pop_size = 0; PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; @@ -6264,29 +6366,31 @@ emit_and_reset_fail_pop(struct compiler *c, pattern_context *pc) } static int -compiler_error_duplicate_store(struct compiler *c, identifier n) +compiler_error_duplicate_store(struct compiler *c, location loc, identifier n) { - return compiler_error(c, "multiple assignments to name %R in pattern", n); + return compiler_error(c, loc, + "multiple assignments to name %R in pattern", n); } // Duplicate the effect of 3.10's ROT_* instructions using SWAPs. static int -pattern_helper_rotate(struct compiler *c, Py_ssize_t count) +pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) { while (1 < count) { - ADDOP_I(c, SWAP, count--); + ADDOP_I(c, loc, SWAP, count--); } return 1; } static int -pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) +pattern_helper_store_name(struct compiler *c, location loc, + identifier n, pattern_context *pc) { if (n == NULL) { - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); return 1; } - if (forbidden_name(c, n, Store)) { + if (forbidden_name(c, loc, n, Store)) { return 0; } // Can't assign to the same name twice: @@ -6295,17 +6399,18 @@ pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) return 0; } if (duplicate) { - return compiler_error_duplicate_store(c, n); + return compiler_error_duplicate_store(c, loc, n); } // Rotate this object underneath any items we need to preserve: Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; - RETURN_IF_FALSE(pattern_helper_rotate(c, rotations)); + RETURN_IF_FALSE(pattern_helper_rotate(c, loc, rotations)); return !PyList_Append(pc->stores, n); } static int -pattern_unpack_helper(struct compiler *c, asdl_pattern_seq *elts) +pattern_unpack_helper(struct compiler *c, location loc, + asdl_pattern_seq *elts) { Py_ssize_t n = asdl_seq_LEN(elts); int seen_star = 0; @@ -6314,28 +6419,29 @@ pattern_unpack_helper(struct compiler *c, asdl_pattern_seq *elts) if (elt->kind == MatchStar_kind && !seen_star) { if ((i >= (1 << 8)) || (n-i-1 >= (INT_MAX >> 8))) - return compiler_error(c, + return compiler_error(c, loc, "too many expressions in " "star-unpacking sequence pattern"); - ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8))); + ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); seen_star = 1; } else if (elt->kind == MatchStar_kind) { - return compiler_error(c, + return compiler_error(c, loc, "multiple starred expressions in sequence pattern"); } } if (!seen_star) { - ADDOP_I(c, UNPACK_SEQUENCE, n); + ADDOP_I(c, loc, UNPACK_SEQUENCE, n); } return 1; } static int -pattern_helper_sequence_unpack(struct compiler *c, asdl_pattern_seq *patterns, - Py_ssize_t star, pattern_context *pc) +pattern_helper_sequence_unpack(struct compiler *c, location *ploc, + asdl_pattern_seq *patterns, Py_ssize_t star, + pattern_context *pc) { - RETURN_IF_FALSE(pattern_unpack_helper(c, patterns)); + RETURN_IF_FALSE(pattern_unpack_helper(c, *ploc, patterns)); Py_ssize_t size = asdl_seq_LEN(patterns); // We've now got a bunch of new subjects on the stack. They need to remain // there after each subpattern match: @@ -6344,7 +6450,7 @@ pattern_helper_sequence_unpack(struct compiler *c, asdl_pattern_seq *patterns, // One less item to keep track of each time we loop through: pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, ploc, pattern, pc)); } return 1; } @@ -6353,8 +6459,9 @@ pattern_helper_sequence_unpack(struct compiler *c, asdl_pattern_seq *patterns, // UNPACK_SEQUENCE / UNPACK_EX. This is more efficient for patterns with a // starred wildcard like [first, *_] / [first, *_, last] / [*_, last] / etc. static int -pattern_helper_sequence_subscr(struct compiler *c, asdl_pattern_seq *patterns, - Py_ssize_t star, pattern_context *pc) +pattern_helper_sequence_subscr(struct compiler *c, location *ploc, + asdl_pattern_seq *patterns, Py_ssize_t star, + pattern_context *pc) { // We need to keep the subject around for extracting elements: pc->on_top++; @@ -6368,39 +6475,41 @@ pattern_helper_sequence_subscr(struct compiler *c, asdl_pattern_seq *patterns, assert(WILDCARD_STAR_CHECK(pattern)); continue; } - ADDOP_I(c, COPY, 1); + ADDOP_I(c, *ploc, COPY, 1); if (i < star) { - ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(i)); + ADDOP_LOAD_CONST_NEW(c, *ploc, PyLong_FromSsize_t(i)); } else { // The subject may not support negative indexing! Compute a // nonnegative index: - ADDOP(c, GET_LEN); - ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(size - i)); - ADDOP_BINARY(c, Sub); + ADDOP(c, *ploc, GET_LEN); + ADDOP_LOAD_CONST_NEW(c, *ploc, PyLong_FromSsize_t(size - i)); + ADDOP_BINARY(c, *ploc, Sub); } - ADDOP(c, BINARY_SUBSCR); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + ADDOP(c, *ploc, BINARY_SUBSCR); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, ploc, pattern, pc)); } // Pop the subject, we're done with it: pc->on_top--; - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); return 1; } // Like compiler_pattern, but turn off checks for irrefutability. static int -compiler_pattern_subpattern(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern_subpattern(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { int allow_irrefutable = pc->allow_irrefutable; pc->allow_irrefutable = 1; - RETURN_IF_FALSE(compiler_pattern(c, p, pc)); + RETURN_IF_FALSE(compiler_pattern(c, ploc, p, pc)); pc->allow_irrefutable = allow_irrefutable; return 1; } static int -compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern_as(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { assert(p->kind == MatchAs_kind); if (p->v.MatchAs.pattern == NULL) { @@ -6408,20 +6517,20 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) if (!pc->allow_irrefutable) { if (p->v.MatchAs.name) { const char *e = "name capture %R makes remaining patterns unreachable"; - return compiler_error(c, e, p->v.MatchAs.name); + return compiler_error(c, *ploc, e, p->v.MatchAs.name); } const char *e = "wildcard makes remaining patterns unreachable"; - return compiler_error(c, e); + return compiler_error(c, *ploc, e); } - return pattern_helper_store_name(c, p->v.MatchAs.name, pc); + return pattern_helper_store_name(c, *ploc, p->v.MatchAs.name, pc); } // Need to make a copy for (possibly) storing later: pc->on_top++; - ADDOP_I(c, COPY, 1); - RETURN_IF_FALSE(compiler_pattern(c, p->v.MatchAs.pattern, pc)); + ADDOP_I(c, *ploc, COPY, 1); + RETURN_IF_FALSE(compiler_pattern(c, ploc, p->v.MatchAs.pattern, pc)); // Success! Store it: pc->on_top--; - RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.MatchAs.name, pc)); + RETURN_IF_FALSE(pattern_helper_store_name(c, *ploc, p->v.MatchAs.name, pc)); return 1; } @@ -6429,7 +6538,8 @@ static int compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchStar_kind); - RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.MatchStar.name, pc)); + location loc = LOC(p); + RETURN_IF_FALSE(pattern_helper_store_name(c, loc, p->v.MatchStar.name, pc)); return 1; } @@ -6442,14 +6552,16 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ for (Py_ssize_t i = 0; i < nattrs; i++) { identifier attr = ((identifier)asdl_seq_GET(attrs, i)); SET_LOC(c, ((pattern_ty) asdl_seq_GET(patterns, i))); - if (forbidden_name(c, attr, Store)) { + location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); + if (forbidden_name(c, loc, attr, Store)) { return -1; } for (Py_ssize_t j = i + 1; j < nattrs; j++) { identifier other = ((identifier)asdl_seq_GET(attrs, j)); if (!PyUnicode_Compare(attr, other)) { + location loc = LOC((pattern_ty) asdl_seq_GET(patterns, j)); SET_LOC(c, ((pattern_ty) asdl_seq_GET(patterns, j))); - compiler_error(c, "attribute name repeated in class pattern: %U", attr); + compiler_error(c, loc, "attribute name repeated in class pattern: %U", attr); return -1; } } @@ -6458,7 +6570,8 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ } static int -compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern_class(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { assert(p->kind == MatchClass_kind); asdl_pattern_seq *patterns = p->v.MatchClass.patterns; @@ -6471,11 +6584,11 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) // AST validator shouldn't let this happen, but if it does, // just fail, don't crash out of the interpreter const char * e = "kwd_attrs (%d) / kwd_patterns (%d) length mismatch in class pattern"; - return compiler_error(c, e, nattrs, nkwd_patterns); + return compiler_error(c, *ploc, e, nattrs, nkwd_patterns); } if (INT_MAX < nargs || INT_MAX < nargs + nattrs - 1) { const char *e = "too many sub-patterns in class pattern %R"; - return compiler_error(c, e, p->v.MatchClass.cls); + return compiler_error(c, *ploc, e, p->v.MatchClass.cls); } if (nattrs) { RETURN_IF_FALSE(!validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); @@ -6490,15 +6603,15 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) Py_INCREF(name); PyTuple_SET_ITEM(attr_names, i, name); } - ADDOP_LOAD_CONST_NEW(c, attr_names); - ADDOP_I(c, MATCH_CLASS, nargs); - ADDOP_I(c, COPY, 1); - ADDOP_LOAD_CONST(c, Py_None); - ADDOP_I(c, IS_OP, 1); + ADDOP_LOAD_CONST_NEW(c, *ploc, attr_names); + ADDOP_I(c, *ploc, MATCH_CLASS, nargs); + ADDOP_I(c, *ploc, COPY, 1); + ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADDOP_I(c, *ploc, IS_OP, 1); // TOS is now a tuple of (nargs + nattrs) attributes (or None): pc->on_top++; - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); - ADDOP_I(c, UNPACK_SEQUENCE, nargs + nattrs); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); + ADDOP_I(c, *ploc, UNPACK_SEQUENCE, nargs + nattrs); pc->on_top += nargs + nattrs - 1; for (i = 0; i < nargs + nattrs; i++) { pc->on_top--; @@ -6512,17 +6625,19 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) pattern = asdl_seq_GET(kwd_patterns, i - nargs); } if (WILDCARD_CHECK(pattern)) { - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); continue; } - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + *ploc = LOC(pattern); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, ploc, pattern, pc)); } // Success! Pop the tuple of attributes: return 1; } static int -compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern_mapping(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { assert(p->kind == MatchMapping_kind); asdl_expr_seq *keys = p->v.MatchMapping.keys; @@ -6533,29 +6648,29 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) // AST validator shouldn't let this happen, but if it does, // just fail, don't crash out of the interpreter const char * e = "keys (%d) / patterns (%d) length mismatch in mapping pattern"; - return compiler_error(c, e, size, npatterns); + return compiler_error(c, *ploc, e, size, npatterns); } // We have a double-star target if "rest" is set PyObject *star_target = p->v.MatchMapping.rest; // We need to keep the subject on top during the mapping and length checks: pc->on_top++; - ADDOP(c, MATCH_MAPPING); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP(c, *ploc, MATCH_MAPPING); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); if (!size && !star_target) { // If the pattern is just "{}", we're done! Pop the subject: pc->on_top--; - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); return 1; } if (size) { // If the pattern has any keys in it, perform a length check: - ADDOP(c, GET_LEN); - ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(size)); - ADDOP_COMPARE(c, GtE); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP(c, *ploc, GET_LEN); + ADDOP_LOAD_CONST_NEW(c, *ploc, PyLong_FromSsize_t(size)); + ADDOP_COMPARE(c, *ploc, GtE); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); } if (INT_MAX < size - 1) { - return compiler_error(c, "too many sub-patterns in mapping pattern"); + return compiler_error(c, *ploc, "too many sub-patterns in mapping pattern"); } // Collect all of the keys into a tuple for MATCH_KEYS and // **rest. They can either be dotted names or literals: @@ -6573,8 +6688,9 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) if (key == NULL) { const char *e = "can't use NULL keys in MatchMapping " "(set 'rest' parameter instead)"; - SET_LOC(c, ((pattern_ty) asdl_seq_GET(patterns, i))); - compiler_error(c, e); + location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); + SET_LOC(c, (pattern_ty) asdl_seq_GET(patterns, i)); + compiler_error(c, loc, e); goto error; } @@ -6585,7 +6701,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) } if (in_seen) { const char *e = "mapping pattern checks duplicate key (%R)"; - compiler_error(c, e, key->v.Constant.value); + compiler_error(c, *ploc, e, key->v.Constant.value); goto error; } if (PySet_Add(seen, key->v.Constant.value)) { @@ -6595,7 +6711,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) else if (key->kind != Attribute_kind) { const char *e = "mapping pattern keys may only match literals and attribute lookups"; - compiler_error(c, e); + compiler_error(c, *ploc, e); goto error; } if (!compiler_visit_expr(c, key)) { @@ -6606,22 +6722,22 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) // all keys have been checked; there are no duplicates Py_DECREF(seen); - ADDOP_I(c, BUILD_TUPLE, size); - ADDOP(c, MATCH_KEYS); + ADDOP_I(c, *ploc, BUILD_TUPLE, size); + ADDOP(c, *ploc, MATCH_KEYS); // There's now a tuple of keys and a tuple of values on top of the subject: pc->on_top += 2; - ADDOP_I(c, COPY, 1); - ADDOP_LOAD_CONST(c, Py_None); - ADDOP_I(c, IS_OP, 1); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP_I(c, *ploc, COPY, 1); + ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADDOP_I(c, *ploc, IS_OP, 1); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); // So far so good. Use that tuple of values on the stack to match // sub-patterns against: - ADDOP_I(c, UNPACK_SEQUENCE, size); + ADDOP_I(c, *ploc, UNPACK_SEQUENCE, size); pc->on_top += size - 1; for (Py_ssize_t i = 0; i < size; i++) { pc->on_top--; pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, ploc, pattern, pc)); } // If we get this far, it's a match! Whatever happens next should consume // the tuple of keys and the subject: @@ -6633,20 +6749,20 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) // rest = dict(TOS1) // for key in TOS: // del rest[key] - ADDOP_I(c, BUILD_MAP, 0); // [subject, keys, empty] - ADDOP_I(c, SWAP, 3); // [empty, keys, subject] - ADDOP_I(c, DICT_UPDATE, 2); // [copy, keys] - ADDOP_I(c, UNPACK_SEQUENCE, size); // [copy, keys...] + ADDOP_I(c, *ploc, BUILD_MAP, 0); // [subject, keys, empty] + ADDOP_I(c, *ploc, SWAP, 3); // [empty, keys, subject] + ADDOP_I(c, *ploc, DICT_UPDATE, 2); // [copy, keys] + ADDOP_I(c, *ploc, UNPACK_SEQUENCE, size); // [copy, keys...] while (size) { - ADDOP_I(c, COPY, 1 + size--); // [copy, keys..., copy] - ADDOP_I(c, SWAP, 2); // [copy, keys..., copy, key] - ADDOP(c, DELETE_SUBSCR); // [copy, keys...] + ADDOP_I(c, *ploc, COPY, 1 + size--); // [copy, keys..., copy] + ADDOP_I(c, *ploc, SWAP, 2); // [copy, keys..., copy, key] + ADDOP(c, *ploc, DELETE_SUBSCR); // [copy, keys...] } - RETURN_IF_FALSE(pattern_helper_store_name(c, star_target, pc)); + RETURN_IF_FALSE(pattern_helper_store_name(c, *ploc, star_target, pc)); } else { - ADDOP(c, POP_TOP); // Tuple of keys. - ADDOP(c, POP_TOP); // Subject. + ADDOP(c, *ploc, POP_TOP); // Tuple of keys. + ADDOP(c, *ploc, POP_TOP); // Subject. } return 1; @@ -6656,7 +6772,8 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) } static int -compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern_or(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { assert(p->kind == MatchOr_kind); NEW_JUMP_TARGET_LABEL(c, end); @@ -6683,8 +6800,9 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; - if (!cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, COMPILER_LOC(c)) || - !compiler_pattern(c, alt, pc)) { + *ploc = LOC(alt); + if (!cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, *ploc) || + !compiler_pattern(c, ploc, alt, pc)) { goto error; } // Success! @@ -6739,7 +6857,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Do the same thing to the stack, using several // rotations: while (rotations--) { - if (!pattern_helper_rotate(c, icontrol + 1)){ + if (!pattern_helper_rotate(c, *ploc, icontrol + 1)){ goto error; } } @@ -6747,8 +6865,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) } } assert(control); - if (!cfg_builder_addop_j(CFG_BUILDER(c), JUMP, end, COMPILER_LOC(c)) || - !emit_and_reset_fail_pop(c, pc)) + if (!cfg_builder_addop_j(CFG_BUILDER(c), *ploc, JUMP, end) || + !emit_and_reset_fail_pop(c, *ploc, pc)) { goto error; } @@ -6759,7 +6877,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to NULL this for the PyObject_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: - if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, COMPILER_LOC(c)) || !jump_to_fail_pop(c, pc, JUMP)) { + if (!cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, *ploc) || + !jump_to_fail_pop(c, *ploc, pc, JUMP)) { goto error; } @@ -6774,7 +6893,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); for (Py_ssize_t i = 0; i < nstores; i++) { // Rotate this capture to its proper place on the stack: - if (!pattern_helper_rotate(c, nrots)) { + if (!pattern_helper_rotate(c, *ploc, nrots)) { goto error; } // Update the list of previous stores with this new name, checking for @@ -6785,7 +6904,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) goto error; } if (dupe) { - compiler_error_duplicate_store(c, name); + compiler_error_duplicate_store(c, *ploc, name); goto error; } if (PyList_Append(pc->stores, name)) { @@ -6796,10 +6915,10 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) Py_DECREF(control); // NOTE: Returning macros are safe again. // Pop the copy of the subject: - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); return 1; diff: - compiler_error(c, "alternative patterns bind different names"); + compiler_error(c, *ploc, "alternative patterns bind different names"); error: PyObject_Free(old_pc.fail_pop); Py_DECREF(old_pc.stores); @@ -6809,7 +6928,8 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) static int -compiler_pattern_sequence(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern_sequence(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { assert(p->kind == MatchSequence_kind); asdl_pattern_seq *patterns = p->v.MatchSequence.patterns; @@ -6823,7 +6943,7 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, pattern_context *pc) if (pattern->kind == MatchStar_kind) { if (star >= 0) { const char *e = "multiple starred names in sequence pattern"; - return compiler_error(c, e); + return compiler_error(c, *ploc, e); } star_wildcard = WILDCARD_STAR_CHECK(pattern); only_wildcard &= star_wildcard; @@ -6834,33 +6954,33 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, pattern_context *pc) } // We need to keep the subject on top during the sequence and length checks: pc->on_top++; - ADDOP(c, MATCH_SEQUENCE); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP(c, *ploc, MATCH_SEQUENCE); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); if (star < 0) { // No star: len(subject) == size - ADDOP(c, GET_LEN); - ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(size)); - ADDOP_COMPARE(c, Eq); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP(c, *ploc, GET_LEN); + ADDOP_LOAD_CONST_NEW(c, *ploc, PyLong_FromSsize_t(size)); + ADDOP_COMPARE(c, *ploc, Eq); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); } else if (size > 1) { // Star: len(subject) >= size - 1 - ADDOP(c, GET_LEN); - ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(size - 1)); - ADDOP_COMPARE(c, GtE); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP(c, *ploc, GET_LEN); + ADDOP_LOAD_CONST_NEW(c, *ploc, PyLong_FromSsize_t(size - 1)); + ADDOP_COMPARE(c, *ploc, GtE); + RETURN_IF_FALSE(jump_to_fail_pop(c, *ploc, pc, POP_JUMP_IF_FALSE)); } // Whatever comes next should consume the subject: pc->on_top--; if (only_wildcard) { // Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. - ADDOP(c, POP_TOP); + ADDOP(c, *ploc, POP_TOP); } else if (star_wildcard) { - RETURN_IF_FALSE(pattern_helper_sequence_subscr(c, patterns, star, pc)); + RETURN_IF_FALSE(pattern_helper_sequence_subscr(c, ploc, patterns, star, pc)); } else { - RETURN_IF_FALSE(pattern_helper_sequence_unpack(c, patterns, star, pc)); + RETURN_IF_FALSE(pattern_helper_sequence_unpack(c, ploc, patterns, star, pc)); } return 1; } @@ -6869,14 +6989,15 @@ static int compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchValue_kind); + location loc = LOC(p); expr_ty value = p->v.MatchValue.value; if (!MATCH_VALUE_EXPR(value)) { const char *e = "patterns may only match literals and attribute lookups"; - return compiler_error(c, e); + return compiler_error(c, loc, e); } VISIT(c, expr, value); - ADDOP_COMPARE(c, Eq); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + ADDOP_COMPARE(c, loc, Eq); + RETURN_IF_FALSE(jump_to_fail_pop(c, loc, pc, POP_JUMP_IF_FALSE)); return 1; } @@ -6884,38 +7005,41 @@ static int compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchSingleton_kind); - ADDOP_LOAD_CONST(c, p->v.MatchSingleton.value); - ADDOP_COMPARE(c, Is); - RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE)); + location loc = LOC(p); + ADDOP_LOAD_CONST(c, loc, p->v.MatchSingleton.value); + ADDOP_COMPARE(c, loc, Is); + RETURN_IF_FALSE(jump_to_fail_pop(c, loc, pc, POP_JUMP_IF_FALSE)); return 1; } static int -compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) +compiler_pattern(struct compiler *c, location *ploc, + pattern_ty p, pattern_context *pc) { SET_LOC(c, p); + *ploc = LOC(p); switch (p->kind) { case MatchValue_kind: return compiler_pattern_value(c, p, pc); case MatchSingleton_kind: return compiler_pattern_singleton(c, p, pc); case MatchSequence_kind: - return compiler_pattern_sequence(c, p, pc); + return compiler_pattern_sequence(c, ploc, p, pc); case MatchMapping_kind: - return compiler_pattern_mapping(c, p, pc); + return compiler_pattern_mapping(c, ploc, p, pc); case MatchClass_kind: - return compiler_pattern_class(c, p, pc); + return compiler_pattern_class(c, ploc, p, pc); case MatchStar_kind: return compiler_pattern_star(c, p, pc); case MatchAs_kind: - return compiler_pattern_as(c, p, pc); + return compiler_pattern_as(c, ploc, p, pc); case MatchOr_kind: - return compiler_pattern_or(c, p, pc); + return compiler_pattern_or(c, ploc, p, pc); } // AST validator shouldn't let this happen, but if it does, // just fail, don't crash out of the interpreter const char *e = "invalid match pattern node in AST (kind=%d)"; - return compiler_error(c, e, p->kind); + return compiler_error(c, *ploc, e, p->kind); } static int @@ -6931,8 +7055,9 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) m = asdl_seq_GET(s->v.Match.cases, i); SET_LOC(c, m->pattern); // Only copy the subject if we're *not* on the last case: + location loc = LOC(m->pattern); if (i != cases - has_default - 1) { - ADDOP_I(c, COPY, 1); + ADDOP_I(c, loc, COPY, 1); } RETURN_IF_FALSE(pc->stores = PyList_New(0)); // Irrefutable cases must be either guarded, last, or both: @@ -6941,7 +7066,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) pc->fail_pop_size = 0; pc->on_top = 0; // NOTE: Can't use returning macros here (they'll leak pc->stores)! - if (!compiler_pattern(c, m->pattern, pc)) { + if (!compiler_pattern(c, &loc, m->pattern, pc)) { Py_DECREF(pc->stores); return 0; } @@ -6950,7 +7075,7 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); for (Py_ssize_t n = 0; n < nstores; n++) { PyObject *name = PyList_GET_ITEM(pc->stores, n); - if (!compiler_nameop(c, name, Store)) { + if (!compiler_nameop(c, loc, name, Store)) { Py_DECREF(pc->stores); return 0; } @@ -6959,35 +7084,36 @@ compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) // NOTE: Returning macros are safe again. if (m->guard) { RETURN_IF_FALSE(ensure_fail_pop(c, pc, 0)); - RETURN_IF_FALSE(compiler_jump_if(c, m->guard, pc->fail_pop[0], 0)); + RETURN_IF_FALSE(compiler_jump_if(c, &loc, m->guard, pc->fail_pop[0], 0)); } // Success! Pop the subject off, we're done with it: if (i != cases - has_default - 1) { - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); } VISIT_SEQ(c, stmt, m->body); - ADDOP_JUMP(c, JUMP, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); // If the pattern fails to match, we want the line number of the // cleanup to be associated with the failed pattern, not the last line // of the body SET_LOC(c, m->pattern); - RETURN_IF_FALSE(emit_and_reset_fail_pop(c, pc)); + RETURN_IF_FALSE(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); } if (has_default) { // A trailing "case _" is common, and lets us save a bit of redundant // pushing and popping in the loop above: m = asdl_seq_GET(s->v.Match.cases, cases - 1); SET_LOC(c, m->pattern); + location loc = LOC(m->pattern); if (cases == 1) { // No matches. Done with the subject: - ADDOP(c, POP_TOP); + ADDOP(c, loc, POP_TOP); } else { // Show line coverage for default case (it doesn't create bytecode) - ADDOP(c, NOP); + ADDOP(c, loc, NOP); } if (m->guard) { - RETURN_IF_FALSE(compiler_jump_if(c, m->guard, end, 0)); + RETURN_IF_FALSE(compiler_jump_if(c, &loc, m->guard, end, 0)); } VISIT_SEQ(c, stmt, m->body); } @@ -8585,9 +8711,10 @@ assemble(struct compiler *c, int addNone) /* Make sure every block that falls off the end returns None. */ if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { UNSET_LOC(c); - if (addNone) - ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, RETURN_VALUE); + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + } + ADDOP(c, NO_LOCATION, RETURN_VALUE); } assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); @@ -9437,7 +9564,7 @@ propagate_line_numbers(basicblock *entryblock) { continue; } - struct location prev_location = NO_LOCATION; + location prev_location = NO_LOCATION; for (int i = 0; i < b->b_iused; i++) { if (b->b_instr[i].i_loc.lineno < 0) { b->b_instr[i].i_loc = prev_location; @@ -9695,7 +9822,7 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) if (PyErr_Occurred()) { return -1; } - struct location loc; + location loc; loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); if (PyErr_Occurred()) { return -1; @@ -9743,7 +9870,7 @@ cfg_to_instructions(cfg_builder *g) Py_DECREF(lbl); for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; - struct location loc = instr->i_loc; + location loc = instr->i_loc; int arg = HAS_TARGET(instr->i_opcode) ? instr->i_target->b_label : instr->i_oparg; PyObject *inst_tuple = Py_BuildValue( "(iiiiii)", instr->i_opcode, arg, diff --git a/Python/deepfreeze/README.txt b/Python/deepfreeze/README.txt index da55d4e7c7469e..276ab51143ab33 100644 --- a/Python/deepfreeze/README.txt +++ b/Python/deepfreeze/README.txt @@ -3,4 +3,4 @@ modules. Python/frozen.c depends on these files. None of these files are committed into the repo. -See Tools/scripts/freeze_modules.py for more info. +See Tools/build/freeze_modules.py for more info. diff --git a/Python/frozen.c b/Python/frozen.c index 8a2a7243537cc5..48b429519b6606 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -8,7 +8,7 @@ * These files must be regenerated any time the corresponding .pyc * file would change (including with changes to the compiler, bytecode * format, marshal format). This can be done with "make regen-frozen". - * That make target just runs Tools/scripts/freeze_modules.py. + * That make target just runs Tools/build/freeze_modules.py. * * The freeze_modules.py script also determines which modules get * frozen. Update the list at the top of the script to add, remove, diff --git a/Python/frozen_modules/README.txt b/Python/frozen_modules/README.txt index 444167cc496af3..795bb0efad34df 100644 --- a/Python/frozen_modules/README.txt +++ b/Python/frozen_modules/README.txt @@ -4,4 +4,4 @@ modules. Python/frozen.c depends on these files. Note that, other than the required frozen modules, none of these files are committed into the repo. -See Tools/scripts/freeze_modules.py for more info. +See Tools/build/freeze_modules.py for more info. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index acb1330b85fb75..a0005b32fcf146 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1107,16 +1107,9 @@ print_exception_suggestions(struct exception_print_context *ctx, PyObject *f = ctx->file; PyObject *suggestions = _Py_Offer_Suggestions(value); if (suggestions) { - // Add a trailer ". Did you mean: (...)?" - if (PyFile_WriteString(". Did you mean: '", f) < 0) { - goto error; - } if (PyFile_WriteObject(suggestions, f, Py_PRINT_RAW) < 0) { goto error; } - if (PyFile_WriteString("'?", f) < 0) { - goto error; - } Py_DECREF(suggestions); } else if (PyErr_Occurred()) { diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index b28156608d1b80..12827e775754f7 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -1,4 +1,4 @@ -// Auto-generated by Tools/scripts/generate_stdlib_module_names.py. +// Auto-generated by Tools/build/generate_stdlib_module_names.py. // List used to create sys.stdlib_module_names. static const char* _Py_stdlib_module_names[] = { diff --git a/Python/suggestions.c b/Python/suggestions.c index c336ec8ffffc9c..89b86f78bc7ac8 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -3,6 +3,7 @@ #include "pycore_pyerrors.h" #include "pycore_code.h" // _PyCode_GetVarnames() +#include "stdlib_module_names.h" // _Py_stdlib_module_names #define MAX_CANDIDATE_ITEMS 750 #define MAX_STRING_SIZE 40 @@ -175,7 +176,7 @@ calculate_suggestions(PyObject *dir, } static PyObject * -offer_suggestions_for_attribute_error(PyAttributeErrorObject *exc) +get_suggestions_for_attribute_error(PyAttributeErrorObject *exc) { PyObject *name = exc->name; // borrowed reference PyObject *obj = exc->obj; // borrowed reference @@ -195,35 +196,25 @@ offer_suggestions_for_attribute_error(PyAttributeErrorObject *exc) return suggestions; } - static PyObject * -offer_suggestions_for_name_error(PyNameErrorObject *exc) +offer_suggestions_for_attribute_error(PyAttributeErrorObject *exc) { - PyObject *name = exc->name; // borrowed reference - PyTracebackObject *traceback = (PyTracebackObject *) exc->traceback; // borrowed reference - // Abort if we don't have a variable name or we have an invalid one - // or if we don't have a traceback to work with - if (name == NULL || !PyUnicode_CheckExact(name) || - traceback == NULL || !Py_IS_TYPE(traceback, &PyTraceBack_Type) - ) { + PyObject* suggestion = get_suggestions_for_attribute_error(exc); + if (suggestion == NULL) { return NULL; } + // Add a trailer ". Did you mean: (...)?" + PyObject* result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); + Py_DECREF(suggestion); + return result; +} - // Move to the traceback of the exception - while (1) { - PyTracebackObject *next = traceback->tb_next; - if (next == NULL || !Py_IS_TYPE(next, &PyTraceBack_Type)) { - break; - } - else { - traceback = next; - } - } - - PyFrameObject *frame = traceback->tb_frame; - assert(frame != NULL); +static PyObject * +get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) +{ PyCodeObject *code = PyFrame_GetCode(frame); assert(code != NULL && code->co_localsplusnames != NULL); + PyObject *varnames = _PyCode_GetVarnames(code); if (varnames == NULL) { return NULL; @@ -261,6 +252,66 @@ offer_suggestions_for_name_error(PyNameErrorObject *exc) return suggestions; } +static bool +is_name_stdlib_module(PyObject* name) +{ + const char* the_name = PyUnicode_AsUTF8(name); + Py_ssize_t len = Py_ARRAY_LENGTH(_Py_stdlib_module_names); + for (Py_ssize_t i = 0; i < len; i++) { + if (strcmp(the_name, _Py_stdlib_module_names[i]) == 0) { + return 1; + } + } + return 0; +} + +static PyObject * +offer_suggestions_for_name_error(PyNameErrorObject *exc) +{ + PyObject *name = exc->name; // borrowed reference + PyTracebackObject *traceback = (PyTracebackObject *) exc->traceback; // borrowed reference + // Abort if we don't have a variable name or we have an invalid one + // or if we don't have a traceback to work with + if (name == NULL || !PyUnicode_CheckExact(name) || + traceback == NULL || !Py_IS_TYPE(traceback, &PyTraceBack_Type) + ) { + return NULL; + } + + // Move to the traceback of the exception + while (1) { + PyTracebackObject *next = traceback->tb_next; + if (next == NULL || !Py_IS_TYPE(next, &PyTraceBack_Type)) { + break; + } + else { + traceback = next; + } + } + + PyFrameObject *frame = traceback->tb_frame; + assert(frame != NULL); + + PyObject* suggestion = get_suggestions_for_name_error(name, frame); + bool is_stdlib_module = is_name_stdlib_module(name); + + if (suggestion == NULL && !is_stdlib_module) { + return NULL; + } + + // Add a trailer ". Did you mean: (...)?" + PyObject* result = NULL; + if (!is_stdlib_module) { + result = PyUnicode_FromFormat(". Did you mean: %R?", suggestion); + } else if (suggestion == NULL) { + result = PyUnicode_FromFormat(". Did you forget to import %R?", name); + } else { + result = PyUnicode_FromFormat(". Did you mean: %R? Or did you forget to import %R?", suggestion, name); + } + Py_XDECREF(suggestion); + return result; +} + // Offer suggestions for a given exception. Returns a python string object containing the // suggestions. This function returns NULL if no suggestion was found or if an exception happened, // users must call PyErr_Occurred() to disambiguate. diff --git a/Tools/scripts/check_extension_modules.py b/Tools/build/check_extension_modules.py similarity index 100% rename from Tools/scripts/check_extension_modules.py rename to Tools/build/check_extension_modules.py diff --git a/Tools/scripts/deepfreeze.py b/Tools/build/deepfreeze.py similarity index 100% rename from Tools/scripts/deepfreeze.py rename to Tools/build/deepfreeze.py diff --git a/Tools/scripts/freeze_modules.py b/Tools/build/freeze_modules.py similarity index 99% rename from Tools/scripts/freeze_modules.py rename to Tools/build/freeze_modules.py index aa1e4fe2ea0f44..810224b28f2faa 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -581,7 +581,7 @@ def regen_makefile(modules): frozenfiles = [] rules = [''] deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)", - "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \\"] + "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') @@ -646,7 +646,7 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] - deepfreezerules = ['\t