diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3422ef835279bc..d40519e40d3cc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,9 @@ # GitHub .github/** @ezio-melotti @hugovk +# pre-commit +.pre-commit-config.yaml @hugovk @AlexWaygood + # Build system configure* @erlend-aasland @corona10 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000000..4481ea80bfd936 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 916bbeb4352734..88aaea039f04f4 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,6 +4,10 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: + issues: read + pull-requests: read + jobs: label: name: DO-NOT-MERGE / unresolved review diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000000..808622f19a3dbf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + types_or: [c, python, rst] diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index a46ebc1c3d25a9..b2d7362a9de213 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -560,6 +560,13 @@ Eager Task Factory using the provided *custom_task_constructor* when creating a new task instead of the default :class:`Task`. + *custom_task_constructor* must be a *callable* with the signature matching + the signature of :class:`Task.__init__ `. + The callable must return a :class:`asyncio.Task`-compatible object. + + This function returns a *callable* intended to be used as a task factory of an + event loop via :meth:`loop.set_task_factory(factory) `). + .. versionadded:: 3.12 @@ -1014,7 +1021,7 @@ Introspection Task Object =========== -.. class:: Task(coro, *, loop=None, name=None, context=None) +.. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -1054,6 +1061,13 @@ Task Object If no *context* is provided, the Task copies the current context and later runs its coroutine in the copied context. + An optional keyword-only *eager_start* argument allows eagerly starting + the execution of the :class:`asyncio.Task` at task creation time. + If set to ``True`` and the event loop is running, the task will start + executing the coroutine immediately, until the first time the coroutine + blocks. If the coroutine returns or raises without blocking, the task + will be finished eagerly and will skip scheduling to the event loop. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -1067,6 +1081,9 @@ Task Object .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.12 + Added the *eager_start* parameter. + .. method:: done() Return ``True`` if the Task is *done*. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index e3c8c801904b61..8022c596f0af97 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -24,6 +24,8 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`__lt__` method and will return an insertion point between values in an array. +.. _bisect functions: + The following functions are provided: @@ -55,7 +57,7 @@ The following functions are provided: .. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None) bisect(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`bisect_left`, but returns an insertion point which comes + Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes after (to the right of) any existing entries of *x* in *a*. The returned insertion point *ip* partitions the array *a* into two slices @@ -70,7 +72,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. - This function first runs :func:`bisect_left` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -87,10 +89,10 @@ The following functions are provided: .. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None) insort(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`insort_left`, but inserting *x* in *a* after any existing + Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing entries of *x*. - This function first runs :func:`bisect_right` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -120,7 +122,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:func:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -140,7 +142,7 @@ thoughts in mind: Searching Sorted Lists ---------------------- -The above :func:`bisect` functions are useful for finding insertion points but +The above `bisect functions`_ are useful for finding insertion points but can be tricky or awkward to use for common searching tasks. The following five functions show how to transform them into the standard lookups for sorted lists:: @@ -186,8 +188,8 @@ Examples .. _bisect-example: -The :func:`bisect` function can be useful for numeric table lookups. This -example uses :func:`bisect` to look up a letter grade for an exam score (say) +The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This +example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say) based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: @@ -198,8 +200,8 @@ a 'B', and so on:: >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] -The :func:`bisect` and :func:`insort` functions also work with lists of -tuples. The *key* argument can serve to extract the field used for ordering +The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with +lists of tuples. The *key* argument can serve to extract the field used for ordering records in a table:: >>> from collections import namedtuple diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 162041fc7a846e..ebab1389f07e58 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2130,15 +2130,10 @@ Corresponding to collections in :mod:`collections.abc` .. class:: ByteString(Sequence[int]) - A generic version of :class:`collections.abc.ByteString`. - This type represents the types :class:`bytes`, :class:`bytearray`, and :class:`memoryview` of byte sequences. - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. - - .. deprecated:: 3.9 + .. deprecated-removed:: 3.9 3.14 Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) @@ -2977,6 +2972,8 @@ convenience. This is subject to change, and not all deprecations are listed. | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | | collections | | | | +----------------------------------+---------------+-------------------+----------------+ +| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | ++----------------------------------+---------------+-------------------+----------------+ | ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | +----------------------------------+---------------+-------------------+----------------+ | ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 4fe54e30b82b25..cd8d9febb0d13b 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -674,7 +674,14 @@ def process_audit_events(app, doctree, fromdocname): node.replace_self(table) -def patch_pairindextypes(app) -> None: +def patch_pairindextypes(app, _env) -> None: + """Remove all entries from ``pairindextypes`` before writing POT files. + + We want to run this just before writing output files, as the check to + circumvent is in ``I18nBuilder.write_doc()``. + As such, we link this to ``env-check-consistency``, even though it has + nothing to do with the environment consistency check. + """ if app.builder.name != 'gettext': return @@ -688,14 +695,7 @@ def patch_pairindextypes(app) -> None: # the Sphinx-translated pairindextypes values. As we intend to move # away from this, we need Sphinx to believe that these values don't # exist, by deleting them when using the gettext builder. - - pairindextypes.pop('module', None) - pairindextypes.pop('keyword', None) - pairindextypes.pop('operator', None) - pairindextypes.pop('object', None) - pairindextypes.pop('exception', None) - pairindextypes.pop('statement', None) - pairindextypes.pop('builtin', None) + pairindextypes.clear() def setup(app): @@ -719,7 +719,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) - app.connect('builder-inited', patch_pairindextypes) + app.connect('env-check-consistency', patch_pairindextypes) app.connect('doctree-resolved', process_audit_events) app.connect('env-merge-info', audit_events_merge) app.connect('env-purge-doc', audit_events_purge) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ec04178238b6b0..12d357f117b1b6 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -282,6 +282,11 @@ asyncio writing to sockets and uses :meth:`~socket.socket.sendmsg` if the platform supports it. (Contributed by Kumar Aditya in :gh:`91166`.) +* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` + functions to allow opting an event loop in to eager task execution, + making some use-cases 2x to 5x faster. + (Contributed by Jacob Bower & Itamar O in :gh:`102853`, :gh:`104140`, and :gh:`104138`) + * On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of :class:`~asyncio.ThreadedChildWatcher`. @@ -342,8 +347,9 @@ inspect (Contributed by Thomas Krennwallner in :issue:`35759`.) * The performance of :func:`inspect.getattr_static` has been considerably - improved. Most calls to the function should be around 2x faster than they - were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) + improved. Most calls to the function should be at least 2x faster than they + were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex + Waygood in :gh:`103193`.) pathlib ------- @@ -597,7 +603,7 @@ typing :func:`runtime-checkable protocols ` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with seven + faster or more. However, ``isinstance()`` checks against protocols with fourteen or more members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) @@ -643,11 +649,6 @@ Optimizations * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. (Contributed by Itamar O in :gh:`103793`.) -* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` - functions to allow opting an event loop in to eager task execution, - speeding up some use-cases by up to 50%. - (Contributed by Jacob Bower & Itamar O in :gh:`102853`) - CPython bytecode changes ======================== @@ -1168,6 +1169,14 @@ Build Changes optimization levels (0, 1, 2) at once. (Contributed by Victor Stinner in :gh:`99289`.) +* Add platform triplets for 64-bit LoongArch: + + * loongarch64-linux-gnusf + * loongarch64-linux-gnuf32 + * loongarch64-linux-gnu + + (Contributed by Zhang Na in :gh:`90656`.) + C API Changes ============= diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index deab3cc89f726e..3837fa8c6ab5aa 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -24,6 +24,7 @@ typedef struct { #define _Py_MEMORYVIEW_FORTRAN 0x004 /* Fortran contiguous layout */ #define _Py_MEMORYVIEW_SCALAR 0x008 /* scalar: ndim = 0 */ #define _Py_MEMORYVIEW_PIL 0x010 /* PIL-style layout */ +#define _Py_MEMORYVIEW_RESTRICTED 0x020 /* Disallow new references to the memoryview's buffer */ typedef struct { PyObject_VAR_HEAD diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 9fd8571cbc87f4..3c8b368bd2af4e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -21,8 +21,7 @@ struct _ceval_runtime_state; extern void _Py_FinishPendingCalls(PyThreadState *tstate); -extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *); -extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock); +extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); PyAPI_FUNC(int) _PyEval_AddPendingCall( diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 4781dd5735dcf6..b352801673c40a 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -49,9 +49,6 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; - - /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */ - struct _gil_runtime_state gil; }; #ifdef PY_HAVE_PERF_TRAMPOLINE diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 4bd4ef57238f98..d2b12c91fe7a00 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -97,7 +97,8 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, PyCompilerFlags *flags, - int optimize); + int optimize, + int compile_mode); PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9377fd8526e3a2..7e495817981f06 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -847,6 +847,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cookie)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ed9b2bb44ddffc..8ebfee85c87c23 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -335,6 +335,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(code) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) + STRUCT_FOR_ID(compile_mode) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(cookie) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7276ce35ba68f0..527b2121148f4c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -178,6 +178,9 @@ struct _is { basis. Also see _PyRuntimeState regarding the various mutex fields. */ + /* The per-interpreter GIL, which might not be used. */ + struct _gil_runtime_state _gil; + /* the initial PyInterpreterState.threads.head */ PyThreadState _initial_thread; }; diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h index acc12c9275172c..fe19e3f9611a16 100644 --- a/Include/internal/pycore_memoryobject.h +++ b/Include/internal/pycore_memoryobject.h @@ -9,7 +9,8 @@ extern "C" { #endif PyObject * -PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); +_PyMemoryView_FromBufferProc(PyObject *v, int flags, + getbufferproc bufferproc); #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d1b165d0ab9c38..6e06e874711bc2 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -32,8 +32,6 @@ struct _getargs_runtime_state { struct _PyArg_Parser *static_parsers; }; -/* ceval state */ - /* GIL state */ struct _gilstate_runtime_state { diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6ade8fb6eade03..7b9c73dd1edf3b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -841,6 +841,7 @@ extern "C" { INIT_ID(code), \ INIT_ID(command), \ INIT_ID(comment_factory), \ + INIT_ID(compile_mode), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(cookie), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0b33ea187e60ff..8e086edbdf8193 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -858,6 +858,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(comment_factory); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(compile_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(consts); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/argparse.py b/Lib/argparse.py index 68089a5c1e80b0..f5f44ff02c0d38 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -345,21 +345,22 @@ def _format_usage(self, usage, actions, groups, prefix): def get_lines(parts, indent, prefix=None): lines = [] line = [] + indent_length = len(indent) if prefix is not None: line_len = len(prefix) - 1 else: - line_len = len(indent) - 1 + line_len = indent_length - 1 for part in parts: if line_len + 1 + len(part) > text_width and line: lines.append(indent + ' '.join(line)) line = [] - line_len = len(indent) - 1 + line_len = indent_length - 1 line.append(part) line_len += len(part) + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: - lines[0] = lines[0][len(indent):] + lines[0] = lines[0][indent_length:] return lines # if prog is short, follow it with optionals or positionals diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 7eb647bd129819..8d5bde09ea9b5b 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -942,18 +942,31 @@ def callback(): def create_eager_task_factory(custom_task_constructor): + """Create a function suitable for use as a task factory on an event-loop. - if "eager_start" not in inspect.signature(custom_task_constructor).parameters: - raise TypeError( - "Provided constructor does not support eager task execution") + Example usage: + + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) + + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. + + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( coro, loop=loop, name=name, context=context, eager_start=True) - return factory + eager_task_factory = create_eager_task_factory(Task) diff --git a/Lib/gzip.py b/Lib/gzip.py index 75c6ddc3f2cffb..8796c8d9fd9a2d 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -22,6 +22,7 @@ _COMPRESS_LEVEL_BEST = 9 READ_BUFFER_SIZE = 128 * 1024 +_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, @@ -120,6 +121,21 @@ class BadGzipFile(OSError): """Exception raised in some cases for invalid gzip files.""" +class _WriteBufferStream(io.RawIOBase): + """Minimal object to pass WriteBuffer flushes into GzipFile""" + def __init__(self, gzip_file): + self.gzip_file = gzip_file + + def write(self, data): + return self.gzip_file._write_raw(data) + + def seekable(self): + return False + + def writable(self): + return True + + class GzipFile(_compression.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -184,6 +200,7 @@ def __init__(self, filename=None, mode=None, if mode is None: mode = getattr(fileobj, 'mode', 'rb') + if mode.startswith('r'): self.mode = READ raw = _GzipReader(fileobj) @@ -206,6 +223,9 @@ def __init__(self, filename=None, mode=None, zlib.DEF_MEM_LEVEL, 0) self._write_mtime = mtime + self._buffer_size = _WRITE_BUFFER_SIZE + self._buffer = io.BufferedWriter(_WriteBufferStream(self), + buffer_size=self._buffer_size) else: raise ValueError("Invalid mode: {!r}".format(mode)) @@ -231,6 +251,11 @@ def _init_write(self, filename): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc + def tell(self): + self._check_not_closed() + self._buffer.flush() + return super().tell() + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method @@ -272,6 +297,10 @@ def write(self,data): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") + return self._buffer.write(data) + + def _write_raw(self, data): + # Called by our self._buffer underlying WriteBufferStream. if isinstance(data, (bytes, bytearray)): length = len(data) else: @@ -322,9 +351,9 @@ def close(self): fileobj = self.fileobj if fileobj is None: return - self.fileobj = None try: if self.mode == WRITE: + self._buffer.flush() fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB @@ -332,6 +361,7 @@ def close(self): elif self.mode == READ: self._buffer.close() finally: + self.fileobj = None myfileobj = self.myfileobj if myfileobj: self.myfileobj = None @@ -341,7 +371,7 @@ def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: # Ensure the compressor's buffer is flushed - self.fileobj.write(self.compress.flush(zlib_mode)) + self._buffer.flush() self.fileobj.flush() def fileno(self): @@ -378,10 +408,10 @@ def seek(self, offset, whence=io.SEEK_SET): if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset - chunk = b'\0' * 1024 - for i in range(count // 1024): + chunk = b'\0' * self._buffer_size + for i in range(count // self._buffer_size): self.write(chunk) - self.write(b'\0' * (count % 1024)) + self.write(b'\0' * (count % self._buffer_size)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 5294578cc26cf3..b1d9271f8e47ca 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -112,6 +112,43 @@ def find_spec(name, package=None): return spec +# Normally we would use contextlib.contextmanager. However, this module +# is imported by runpy, which means we want to avoid any unnecessary +# dependencies. Thus we use a class. + +class allowing_all_extensions: + """A context manager that lets users skip the compatibility check. + + Normally, extensions that do not support multiple interpreters + may not be imported in a subinterpreter. That implies modules + that do not implement multi-phase init. + + Likewise for modules import in a subinterpeter with its own GIL + when the extension does not support a per-interpreter GIL. This + implies the module does not have a Py_mod_multiple_interpreters slot + set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. + + In both cases, this context manager may be used to temporarily + disable the check for compatible extension modules. + """ + + def __init__(self, disable_check=True): + self.disable_check = disable_check + + def __enter__(self): + self.old = _imp._override_multi_interp_extensions_check(self.override) + return self + + def __exit__(self, *args): + old = self.old + del self.old + _imp._override_multi_interp_extensions_check(old) + + @property + def override(self): + return -1 if self.disable_check else 1 + + class _LazyModule(types.ModuleType): """A subclass of the module type which triggers loading upon attribute access.""" diff --git a/Lib/inspect.py b/Lib/inspect.py index 95da7fb71a3997..a64e85e4fd67a4 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1794,8 +1794,9 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel -def _shadowed_dict(klass): - for entry in _static_getmro(klass): +@functools.lru_cache() +def _shadowed_dict_from_mro_tuple(mro): + for entry in mro: dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1805,6 +1806,9 @@ def _shadowed_dict(klass): return class_dict return _sentinel +def _shadowed_dict(klass): + return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 480c354ce8b656..20ec1ce9d80374 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -64,17 +64,25 @@ def _is_case_sensitive(flavour): @functools.lru_cache() def _make_selector(pattern_parts, flavour, case_sensitive): pat = pattern_parts[0] - child_parts = pattern_parts[1:] if not pat: return _TerminatingSelector() if pat == '**': - cls = _RecursiveWildcardSelector - elif pat == '..': - cls = _ParentSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") + child_parts_idx = 1 + while child_parts_idx < len(pattern_parts) and pattern_parts[child_parts_idx] == '**': + child_parts_idx += 1 + child_parts = pattern_parts[child_parts_idx:] + if '**' in child_parts: + cls = _DoubleRecursiveWildcardSelector + else: + cls = _RecursiveWildcardSelector else: - cls = _WildcardSelector + child_parts = pattern_parts[1:] + if pat == '..': + cls = _ParentSelector + elif '**' in pat: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + else: + cls = _WildcardSelector return cls(pat, child_parts, flavour, case_sensitive) @@ -183,20 +191,32 @@ def _iterate_directories(self, parent_path, scandir): def _select_from(self, parent_path, scandir): try: - yielded = set() - try: - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, scandir): - for p in successor_select(starting_point, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() + successor_select = self.successor._select_from + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): + yield p except PermissionError: return +class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): + """ + Like _RecursiveWildcardSelector, but also de-duplicates results from + successive selectors. This is necessary if the pattern contains + multiple non-adjacent '**' segments. + """ + + def _select_from(self, parent_path, scandir): + yielded = set() + try: + for p in super()._select_from(parent_path, scandir): + if p not in yielded: + yield p + yielded.add(p) + finally: + yielded.clear() + + # # Public API # @@ -740,214 +760,185 @@ class Path(PurePath): """ __slots__ = () - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self._flavour.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). + def stat(self, *, follow_symlinks=True): """ - return cls("~").expanduser() - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). + Return the result of the stat() system call on this path, like + os.stat() does. """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return self._flavour.samestat(st, other_st) - - def iterdir(self): - """Yield path objects of the directory contents. + return os.stat(self, follow_symlinks=follow_symlinks) - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. + def lstat(self): """ - for name in os.listdir(self): - yield self._make_child_relpath(name) - - def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) - - def glob(self, pattern, *, case_sensitive=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p + return self.stat(follow_symlinks=False) - def rglob(self, pattern, *, case_sensitive=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. + # Convenience functions for querying the stat results - Use resolve() to get the canonical path to a file. + def exists(self, *, follow_symlinks=True): """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = self._flavour.abspath(self.drive) - else: - cwd = os.getcwd() - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - if not self.root and not self._tail: - result = self.with_segments(cwd) - result._str = cwd - return result - return self.with_segments(cwd, self) + Whether this path exists. - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. """ - - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - try: - s = self._flavour.realpath(self, strict=strict) + self.stat(follow_symlinks=follow_symlinks) except OSError as e: - check_eloop(e) - raise - p = self.with_segments(s) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p + if not _ignore_error(e): + raise + return False + except ValueError: + # Non-encodable path + return False + return True - def stat(self, *, follow_symlinks=True): + def is_dir(self): """ - Return the result of the stat() system call on this path, like - os.stat() does. + Whether this path is a directory. """ - return os.stat(self, follow_symlinks=follow_symlinks) + try: + return S_ISDIR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - def owner(self): + def is_file(self): """ - Return the login name of the file owner. + Whether this path is a regular file (also True for symlinks pointing + to regular files). """ try: - import pwd - return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise NotImplementedError("Path.owner() is unsupported on this system") + return S_ISREG(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - def group(self): + def is_mount(self): """ - Return the group name of the file gid. + Check if this path is a mount point """ + return self._flavour.ismount(self) + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ try: - import grp - return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise NotImplementedError("Path.group() is unsupported on this system") + return S_ISLNK(self.lstat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist + return False + except ValueError: + # Non-encodable path + return False - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): + def is_junction(self): """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. + Whether this path is a junction. """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) + return self._flavour.isjunction(self) + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return self._flavour.samestat(st, other_st) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) def read_bytes(self): """ @@ -984,6 +975,232 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + for name in os.listdir(self): + yield self._make_child_relpath(name) + + def _scandir(self): + # bpo-24132: a future version of pathlib will support subclassing of + # pathlib.Path to customize how the filesystem is accessed. This + # includes scandir(), which is used to implement glob(). + return os.scandir(self) + + def _make_child_relpath(self, name): + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self._flavour.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path + + def glob(self, pattern, *, case_sensitive=None): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not pattern: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + if pattern[-1] in (self._flavour.sep, self._flavour.altsep): + pattern_parts.append('') + selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) + for p in selector.select_from(self): + yield p + + def rglob(self, pattern, *, case_sensitive=None): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): + pattern_parts.append('') + selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) + for p in selector.select_from(self): + yield p + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + paths = [self] + + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + + # We may not have read permission for self, in which case we can't + # get a list of the files the directory contains. os.walk() + # always suppressed the exception in that instance, rather than + # blow up for a minor reason when (say) a thousand readable + # directories are still left to visit. That logic is copied here. + try: + scandir_it = path._scandir() + except OSError as error: + if on_error is not None: + on_error(error) + continue + + with scandir_it: + dirnames = [] + filenames = [] + for entry in scandir_it: + try: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + # Carried over from os.path.isdir(). + is_dir = False + + if is_dir: + dirnames.append(entry.name) + else: + filenames.append(entry.name) + + if top_down: + yield path, dirnames, filenames + else: + paths.append((path, dirnames, filenames)) + + paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def __enter__(self): + # In previous versions of pathlib, __exit__() marked this path as + # closed; subsequent attempts to perform I/O would raise an IOError. + # This functionality was never documented, and had the effect of + # making Path objects mutable, contrary to PEP 428. + # In Python 3.9 __exit__() was made a no-op. + # In Python 3.11 __enter__() began emitting DeprecationWarning. + # In Python 3.13 __enter__() and __exit__() should be removed. + warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " + "for removal in Python 3.13; Path objects as a context " + "manager is a no-op", + DeprecationWarning, stacklevel=2) + return self + + def __exit__(self, t, v, tb): + pass + + # Public API + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls().absolute() + + @classmethod + def home(cls): + """Return a new path pointing to the user's home directory (as + returned by os.path.expanduser('~')). + """ + return cls("~").expanduser() + + def absolute(self): + """Return an absolute version of this path by prepending the current + working directory. No normalization or symlink resolution is performed. + + Use resolve() to get the canonical path to a file. + """ + if self.is_absolute(): + return self + elif self.drive: + # There is a CWD on each drive-letter drive. + cwd = self._flavour.abspath(self.drive) + else: + cwd = os.getcwd() + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + if not self.root and not self._tail: + result = self.with_segments(cwd) + result._str = cwd + return result + return self.with_segments(cwd, self) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + def check_eloop(e): + winerror = getattr(e, 'winerror', 0) + if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: + raise RuntimeError("Symlink loop from %r" % e.filename) + + try: + s = self._flavour.realpath(self, strict=strict) + except OSError as e: + check_eloop(e) + raise + p = self.with_segments(s) + + # In non-strict mode, realpath() doesn't raise on symlink loops. + # Ensure we get an exception by calling stat() + if not strict: + try: + p.stat() + except OSError as e: + check_eloop(e) + return p + + def owner(self): + """ + Return the login name of the file owner. + """ + try: + import pwd + return pwd.getpwuid(self.stat().st_uid).pw_name + except ImportError: + raise NotImplementedError("Path.owner() is unsupported on this system") + + def group(self): + """ + Return the group name of the file gid. + """ + + try: + import grp + return grp.getgrgid(self.stat().st_gid).gr_name + except ImportError: + raise NotImplementedError("Path.group() is unsupported on this system") + def readlink(self): """ Return the path to which the symbolic link points. @@ -1061,13 +1278,6 @@ def rmdir(self): """ os.rmdir(self) - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - def rename(self, target): """ Rename this path to the target path. @@ -1113,151 +1323,6 @@ def hardlink_to(self, target): raise NotImplementedError("os.link() not available on this system") os.link(target, self) - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - return self._flavour.ismount(self) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - return self._flavour.isjunction(self) - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) @@ -1272,51 +1337,6 @@ def expanduser(self): return self - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_it = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - class PosixPath(Path, PurePosixPath): """Path subclass for non-Windows systems. diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fb13fa0e243ba7..fd46819fd903fe 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -210,6 +210,13 @@ def clear_caches(): else: fractions._hash_algorithm.cache_clear() + try: + inspect = sys.modules['inspect'] + except KeyError: + pass + else: + inspect._shadowed_dict_from_mro_tuple.cache_clear() + def get_build_info(): # Get most important configure and build options as a list of strings. diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 772c0987c2ebef..67f18e530edc4b 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -115,6 +115,8 @@ def multi_interp_extensions_check(enabled=True): It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and _xxsubinterpreters.create()). + + Also see importlib.utils.allowing_all_extensions(). """ old = _imp._override_multi_interp_extensions_check(1 if enabled else -1) try: diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index b6e82ad4db266a..2c65ae8114818f 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4579,6 +4579,176 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 022753e0c99483..ea57df9cd2400b 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -25,6 +25,8 @@ def test_if_expression(self): ('LOAD_CONST', 2, 1), exit_lbl, ('POP_TOP', None), + ('LOAD_CONST', 3), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -46,5 +48,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8fb97bc0c1a1a7..de7ac97d72cb8e 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -10,7 +10,7 @@ import gc import pickle from test import support -from test.support import warnings_helper +from test.support import warnings_helper, import_helper, check_disallow_instantiation from itertools import permutations from textwrap import dedent from collections import OrderedDict @@ -1430,5 +1430,12 @@ def test_subclassable(self): # issue 44089 class Foo(csv.Error): ... + @support.cpython_only + def test_disallow_instantiation(self): + _csv = import_helper.import_module("_csv") + for tp in _csv.Reader, _csv.Writer: + with self.subTest(tp=tp): + check_disallow_instantiation(self, tp) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 08a615ecf5288b..0be504925ecc6a 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -8,14 +8,29 @@ import importlib.util import os import pathlib +import re import string import sys from test import support +import textwrap import types import unittest import unittest.mock import warnings +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _xxsubinterpreters as _interpreters +except ModuleNotFoundError: + _interpreters = None + class DecodeSourceBytesTests: @@ -637,5 +652,111 @@ def test_magic_number(self): self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) +@unittest.skipIf(_interpreters is None, 'subinterpreters required') +class AllowingAllExtensionsTests(unittest.TestCase): + + ERROR = re.compile("^: module (.*) does not support loading in subinterpreters") + + def run_with_own_gil(self, script): + interpid = _interpreters.create(isolated=True) + try: + _interpreters.run_string(interpid, script) + except _interpreters.RunFailedError as exc: + if m := self.ERROR.match(str(exc)): + modname, = m.groups() + raise ImportError(modname) + + def run_with_shared_gil(self, script): + interpid = _interpreters.create(isolated=False) + try: + _interpreters.run_string(interpid, script) + except _interpreters.RunFailedError as exc: + if m := self.ERROR.match(str(exc)): + modname, = m.groups() + raise ImportError(modname) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_single_phase_init_module(self): + script = textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + import _testsinglephase + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = textwrap.dedent(f''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + import _testsinglephase + ''') + with self.subTest('check enabled, shared GIL'): + with self.assertRaises(ImportError): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + with self.assertRaises(ImportError): + self.run_with_own_gil(script) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_incomplete_multi_phase_init_module(self): + prescript = textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import ExtensionFileLoader + + name = '_test_shared_gil_only' + filename = {_testmultiphase.__file__!r} + loader = ExtensionFileLoader(name, filename) + spec = spec_from_loader(name, loader) + + ''') + + script = prescript + textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + module = module_from_spec(spec) + loader.exec_module(module) + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = prescript + textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + module = module_from_spec(spec) + loader.exec_module(module) + ''') + with self.subTest('check enabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + with self.assertRaises(ImportError): + self.run_with_own_gil(script) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_complete_multi_phase_init_module(self): + script = textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + import _testmultiphase + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = textwrap.dedent(f''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + import _testmultiphase + ''') + with self.subTest('check enabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 42e3d709bd683f..d2b2f3171e785d 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2111,6 +2111,28 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + def test_mutated_mro(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + class Bar(dict): + a = 4 + + class Baz(Bar): pass + + baz = Baz() + self.assertEqual(inspect.getattr_static(baz, 'a'), 4) + Baz.__bases__ = (Foo,) + self.assertEqual(inspect.getattr_static(baz, 'a'), 3) + self.assertFalse(test.called) + def test_custom_object_dict(self): test = self test.called = False @@ -2165,6 +2187,35 @@ class Thing(metaclass=Meta): inspect.getattr_static(Thing, "spam") self.assertFalse(Thing.executed) + def test_custom___getattr__(self): + test = self + test.called = False + + class Foo: + def __getattr__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'whatever') + + self.assertFalse(test.called) + + def test_custom___getattribute__(self): + test = self + test.called = False + + class Foo: + def __getattribute__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'really_could_be_anything') + + self.assertFalse(test.called) + + class TestGetGeneratorState(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 7586610833b063..ee0ef9a34c385c 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1655,7 +1655,8 @@ class P(_BasePurePathSubclass, self.cls): p = P(BASE, session_id=42) self.assertEqual(42, p.absolute().session_id) self.assertEqual(42, p.resolve().session_id) - self.assertEqual(42, p.with_segments('~').expanduser().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) if os_helper.can_symlink(): @@ -1852,13 +1853,14 @@ def _check(path, pattern, case_sensitive, expected): def test_rglob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) P = self.cls p = P(BASE) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) _check(p.rglob("fileB"), ["dirB/fileB"]) + _check(p.rglob("**/fileB"), ["dirB/fileB"]) _check(p.rglob("*/fileA"), []) if not os_helper.can_symlink(): _check(p.rglob("*/fileB"), ["dirB/fileB"]) @@ -1882,9 +1884,12 @@ def _check(glob, expected): _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) + _check(p.rglob("**"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 2b977d78d39324..5418f9f35485f8 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -1,4 +1,5 @@ import unittest +import string import subprocess import sys import sysconfig @@ -70,9 +71,14 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - self.assertIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + perf_lines = perf_file_contents.splitlines(); + expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + for expected_symbol in expected_symbols: + perf_line = next((line for line in perf_lines if expected_symbol in line), None) + self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_addr = perf_line.split(" ")[0] + self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") + self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") def test_trampoline_works_with_forks(self): code = """if 1: diff --git a/Makefile.pre.in b/Makefile.pre.in index 736a520d0e8fb6..329466580b9cb0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2698,7 +2698,7 @@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) -MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst b/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst new file mode 100644 index 00000000000000..dfe71a5552070d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst @@ -0,0 +1,7 @@ +Add platform triplets for 64-bit LoongArch: + +* loongarch64-linux-gnusf +* loongarch64-linux-gnuf32 +* loongarch64-linux-gnu + +Patch by Zhang Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst new file mode 100644 index 00000000000000..5434660e9d6ffb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst @@ -0,0 +1 @@ +Change the perf map format to remove the '0x' prefix from the addresses diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst new file mode 100644 index 00000000000000..42e26cb27b6e01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst @@ -0,0 +1,6 @@ +The GIL is now (optionally) per-interpreter. This is the fundamental change +for PEP 684. This is all made possible by virtue of the isolated state of +each interpreter in the process. The behavior of the main interpreter +remains unchanged. Likewise, interpreters created using +``Py_NewInterpreter()`` are not affected. To get an interpreter with its +own GIL, call ``Py_NewInterpreterFromConfig()``. diff --git a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst new file mode 100644 index 00000000000000..556db0eae00c0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst @@ -0,0 +1,2 @@ +Decrease execution time of some :mod:`gzip` file writes by 15% by +adding more appropriate buffering. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst index 59870de3e02edd..ced3b7cea04954 100644 --- a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst @@ -1 +1,2 @@ -Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. Skip scheduling done callbacks when all tasks finish without blocking. +Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. +Skip scheduling a done callback if a TaskGroup task completes eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst new file mode 100644 index 00000000000000..01f8b948d2cb65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst @@ -0,0 +1,3 @@ +Improve performance of :meth:`pathlib.Path.glob` when expanding recursive +wildcards ("``**``") by merging adjacent wildcards and de-duplicating +results only when necessary. diff --git a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst new file mode 100644 index 00000000000000..9c582844bf909b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst @@ -0,0 +1,4 @@ +Prevent possible crash by disallowing instantiation of the +:class:`!_csv.Reader` and :class:`!_csv.Writer` types. +The regression was introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). +Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst new file mode 100644 index 00000000000000..3743d569995f2e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst @@ -0,0 +1,3 @@ +Users may now use ``importlib.util.allowing_all_extensions()`` (a context +manager) to temporarily disable the strict compatibility checks for +importing extension modules in subinterpreters. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst new file mode 100644 index 00000000000000..03775845450caa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst @@ -0,0 +1 @@ +:func:`socket.getnameinfo` now releases the GIL while contacting the DNS server diff --git a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst new file mode 100644 index 00000000000000..f259acf753831c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst @@ -0,0 +1,2 @@ +Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified implementation +from the ``HACL*`` project. Used when OpenSSL is not present or lacks SHA3. diff --git a/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst b/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst new file mode 100644 index 00000000000000..b6b18dcfd81394 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst @@ -0,0 +1,2 @@ +Support reading SOCKS proxy configuration from macOS System Configuration. +Patch by Sam Schott. diff --git a/Modules/Setup b/Modules/Setup index 1c6f2f7ea5182d..312e99fea530dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -#_sha3 _sha3/sha3module.c +#_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE # text encodings and unicode #_codecs_cn cjkcodecs/_codecs_cn.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a7803cf7c00e68..8e66576b5c5f00 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -80,7 +80,7 @@ @MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -@MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c +@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c ############################################################################ diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h index aa8f281178eadc..c8144ec9d48d29 100644 --- a/Modules/_blake2/blake2module.h +++ b/Modules/_blake2/blake2module.h @@ -38,6 +38,6 @@ #endif // HAVE_LIBB2 // for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" +#include "impl/blake2-impl.h" #endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h index cebc22550da4cd..5b452c4d63babe 100644 --- a/Modules/_blake2/impl/blake2b-round.h +++ b/Modules/_blake2/impl/blake2b-round.h @@ -62,7 +62,7 @@ \ row2l = _mm_roti_epi64(row2l, -24); \ row2h = _mm_roti_epi64(row2h, -24); \ - + #define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ @@ -81,7 +81,7 @@ \ row2l = _mm_roti_epi64(row2l, -63); \ row2h = _mm_roti_epi64(row2h, -63); \ - + #if defined(HAVE_SSSE3) #define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ t0 = _mm_alignr_epi8(row2h, row2l, 8); \ diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h index ac591a77d191a7..14d9e7f7640672 100644 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ b/Modules/_blake2/impl/blake2s-load-xop.h @@ -166,7 +166,7 @@ buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); #define LOAD_MSG_8_3(buf) \ t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - + #define LOAD_MSG_8_4(buf) \ buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h index 1e2f2b7f59bd6c..3af4be35bee5d4 100644 --- a/Modules/_blake2/impl/blake2s-round.h +++ b/Modules/_blake2/impl/blake2s-round.h @@ -86,6 +86,6 @@ LOAD_MSG_ ##r ##_4(buf4); \ G2(row1,row2,row3,row4,buf4); \ UNDIAGONALIZE(row1,row2,row3,row4); \ - + #endif diff --git a/Modules/_csv.c b/Modules/_csv.c index 0cde5c5a8bdc68..9ab2ad266c2739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1000,7 +1000,7 @@ PyType_Spec Reader_Type_spec = { .name = "_csv.reader", .basicsize = sizeof(ReaderObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Reader_Type_slots }; @@ -1431,7 +1431,7 @@ PyType_Spec Writer_Type_spec = { .name = "_csv.writer", .basicsize = sizeof(WriterObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Writer_Type_slots, }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ce652b362d5bb3..ddfb2c8a332a9e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1036,7 +1036,7 @@ EXPORT (HRESULT) KeepObject(IUnknown *punk) #ifdef MS_WIN32 -// i38748: c stub for testing stack corruption +// i38748: c stub for testing stack corruption // When executing a Python callback with a long and a long long typedef long(__stdcall *_test_i38748_funcType)(long, long long); diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c new file mode 100644 index 00000000000000..100afe7c2c6d1f --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -0,0 +1,826 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_SHA3.h" + +static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)144U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)136U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)104U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)72U; + } + case Spec_Hash_Definitions_Shake128: + { + return (uint32_t)168U; + } + case Spec_Hash_Definitions_Shake256: + { + return (uint32_t)136U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)28U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)32U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)48U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)64U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = blocks + i * block_len(a); + Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + } +} + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +) +{ + uint8_t suffix; + if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) + { + suffix = (uint8_t)0x1fU; + } + else + { + suffix = (uint8_t)0x06U; + } + uint32_t len = block_len(a); + if (input_len == len) + { + Hacl_Impl_SHA3_absorb_inner(len, input, s); + uint8_t *uu____0 = input + input_len; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); + lastBlock[0U] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); + return; + } + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, input, input_len * sizeof (uint8_t)); + lastBlock[input_len] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +typedef struct hash_buf2_s +{ + Hacl_Streaming_Keccak_hash_buf fst; + Hacl_Streaming_Keccak_hash_buf snd; +} +hash_buf2; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + return block_state.fst; +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +{ + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); + uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + return p; +} + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + KRML_HOST_FREE(s1); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +{ + Hacl_Streaming_Keccak_state scrut0 = *s0; + Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + uint8_t *buf0 = scrut0.buf; + uint64_t total_len0 = scrut0.total_len; + Spec_Hash_Definitions_hash_alg i = block_state0.fst; + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); + uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); + memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + Hacl_Streaming_Keccak_state + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_Keccak_state s = *p; + Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + uint64_t total_len = s.total_len; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + if ((uint64_t)len > (uint64_t)0xffffffffU - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = block_len(i); + } + else + { + sz = (uint32_t)(total_len % (uint64_t)block_len(i)); + } + if (len <= block_len(i) - sz) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = block_len(i) - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = block_len(i); + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_Keccak_state s10 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)block_len(i) + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +static void +finish_( + Spec_Hash_Definitions_hash_alg a, + Hacl_Streaming_Keccak_state *p, + uint8_t *dst, + uint32_t l +) +{ + Hacl_Streaming_Keccak_state scrut0 = *p; + Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + uint8_t *buf_ = scrut0.buf; + uint64_t total_len = scrut0.total_len; + uint32_t r; + if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = block_len(a); + } + else + { + r = (uint32_t)(total_len % (uint64_t)block_len(a)); + } + uint8_t *buf_1 = buf_; + uint64_t buf[25U] = { 0U }; + Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + uint32_t ite0; + if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + { + ite0 = block_len(a); + } + else + { + ite0 = r % block_len(a); + } + uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_multi = buf_1; + Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; + uint64_t *s0 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; + uint64_t *s1 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); + Spec_Hash_Definitions_hash_alg a11 = tmp_block_state.fst; + uint64_t *s = tmp_block_state.snd; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + uint32_t ite; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + ite = l; + } + else + { + ite = hash_len(a11); + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + return; + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + finish_(a1, s, dst, hash_len(a1)); + return Hacl_Streaming_Keccak_Success; +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + if (l == (uint32_t)0U) + { + return Hacl_Streaming_Keccak_InvalidLength; + } + finish_(a1, s, dst, l); + return Hacl_Streaming_Keccak_Success; +} + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return block_len(a1); +} + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return hash_len(a1); +} + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; +} + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1344U, + (uint32_t)256U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1152U, + (uint32_t)448U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)28U, + output); +} + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)32U, + output); +} + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)832U, + (uint32_t)768U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)48U, + output); +} + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)576U, + (uint32_t)1024U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)64U, + output); +} + +static const +uint32_t +keccak_rotc[24U] = + { + (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, + (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, + (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, + (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U + }; + +static const +uint32_t +keccak_piln[24U] = + { + (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, + (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, + (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, + (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U + }; + +static const +uint64_t +keccak_rndc[24U] = + { + (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, + (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, + (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, + (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, + (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, + (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, + (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, + (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U + }; + +void Hacl_Impl_SHA3_state_permute(uint64_t *s) +{ + for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + _C[i] = + s[i + + (uint32_t)0U] + ^ + (s[i + + (uint32_t)5U] + ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); + KRML_MAYBE_FOR5(i1, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; + uint64_t + _D = + _C[(i1 + (uint32_t)4U) + % (uint32_t)5U] + ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + { + uint32_t _Y = keccak_piln[i]; + uint32_t r = keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t + v0 = + s[(uint32_t)0U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); + uint64_t + v1 = + s[(uint32_t)1U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); + uint64_t + v2 = + s[(uint32_t)2U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); + uint64_t + v3 = + s[(uint32_t)3U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); + uint64_t + v4 = + s[(uint32_t)4U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); + s[(uint32_t)0U + (uint32_t)5U * i] = v0; + s[(uint32_t)1U + (uint32_t)5U * i] = v1; + s[(uint32_t)2U + (uint32_t)5U * i] = v2; + s[(uint32_t)3U + (uint32_t)5U * i] = v3; + s[(uint32_t)4U + (uint32_t)5U * i] = v4;); + uint64_t c = keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } +} + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +{ + uint8_t block[200U] = { 0U }; + memcpy(block, input, rateInBytes * sizeof (uint8_t)); + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t u = load64_le(block + i * (uint32_t)8U); + uint64_t x = u; + s[i] = s[i] ^ x; + } +} + +static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) +{ + uint8_t block[200U] = { 0U }; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t sj = s[i]; + store64_le(block + i * (uint32_t)8U, sj); + } + memcpy(res, block, rateInBytes * sizeof (uint8_t)); +} + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +{ + Hacl_Impl_SHA3_loadState(rateInBytes, block, s); + Hacl_Impl_SHA3_state_permute(s); +} + +static void +absorb( + uint64_t *s, + uint32_t rateInBytes, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix +) +{ + uint32_t n_blocks = inputByteLen / rateInBytes; + uint32_t rem = inputByteLen % rateInBytes; + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = input + i * rateInBytes; + Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); + } + uint8_t *last = input + n_blocks * rateInBytes; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, last, rem * sizeof (uint8_t)); + lastBlock[rem] = delimitedSuffix; + Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); + if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t outBlocks = outputByteLen / rateInBytes; + uint32_t remOut = outputByteLen % rateInBytes; + uint8_t *last = output + outputByteLen - remOut; + uint8_t *blocks = output; + for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + { + storeState(rateInBytes, s, blocks + i * rateInBytes); + Hacl_Impl_SHA3_state_permute(s); + } + storeState(remOut, s, last); +} + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t rateInBytes = rate / (uint32_t)8U; + uint64_t s[25U] = { 0U }; + absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); + Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); +} + diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..2a5cf4b1844b9d --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -0,0 +1,136 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_SHA3_H +#define __Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef struct Hacl_Streaming_Keccak_hash_buf_s +{ + Spec_Hash_Definitions_hash_alg fst; + uint64_t *snd; +} +Hacl_Streaming_Keccak_hash_buf; + +typedef struct Hacl_Streaming_Keccak_state_s +{ + Hacl_Streaming_Keccak_hash_buf block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_Keccak_state; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); + +#define Hacl_Streaming_Keccak_Success 0 +#define Hacl_Streaming_Keccak_InvalidAlgorithm 1 +#define Hacl_Streaming_Keccak_InvalidLength 2 + +typedef uint8_t Hacl_Streaming_Keccak_error_code; + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h index 51057611ca978d..8a60b707bc4958 100644 --- a/Modules/_hacl/Hacl_Streaming_Types.h +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -35,6 +35,23 @@ extern "C" { #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" +#define Spec_Hash_Definitions_SHA2_224 0 +#define Spec_Hash_Definitions_SHA2_256 1 +#define Spec_Hash_Definitions_SHA2_384 2 +#define Spec_Hash_Definitions_SHA2_512 3 +#define Spec_Hash_Definitions_SHA1 4 +#define Spec_Hash_Definitions_MD5 5 +#define Spec_Hash_Definitions_Blake2S 6 +#define Spec_Hash_Definitions_Blake2B 7 +#define Spec_Hash_Definitions_SHA3_256 8 +#define Spec_Hash_Definitions_SHA3_224 9 +#define Spec_Hash_Definitions_SHA3_384 10 +#define Spec_Hash_Definitions_SHA3_512 11 +#define Spec_Hash_Definitions_Shake128 12 +#define Spec_Hash_Definitions_Shake256 13 + +typedef uint8_t Spec_Hash_Definitions_hash_alg; + typedef struct Hacl_Streaming_MD_state_32_s { uint32_t *block_state; diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index dcbe7007b17be8..5a2f94eb2ec8da 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -19,6 +19,28 @@ # define inline __inline__ #endif +/******************************************************************************/ +/* Macros that KaRaMeL will generate. */ +/******************************************************************************/ + +/* For "bare" targets that do not have a C stdlib, the user might want to use + * [-add-early-include '"mydefinitions.h"'] and override these. */ +#ifndef KRML_HOST_PRINTF +# define KRML_HOST_PRINTF printf +#endif + +#if ( \ + (defined __STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + (!(defined KRML_HOST_EPRINTF))) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#elif !(defined KRML_HOST_EPRINTF) && defined(_MSC_VER) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#endif + +#ifndef KRML_HOST_EXIT +# define KRML_HOST_EXIT exit +#endif + #ifndef KRML_HOST_MALLOC # define KRML_HOST_MALLOC malloc #endif @@ -35,6 +57,28 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +/* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of + * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). + */ +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4)) +# define _KRML_CHECK_SIZE_PRAGMA \ + _Pragma("GCC diagnostic ignored \"-Wtype-limits\"") +#else +# define _KRML_CHECK_SIZE_PRAGMA +#endif + +#define KRML_CHECK_SIZE(size_elt, sz) \ + do { \ + _KRML_CHECK_SIZE_PRAGMA \ + if (((size_t)(sz)) > ((size_t)(SIZE_MAX / (size_elt)))) { \ + KRML_HOST_PRINTF( \ + "Maximum allocatable size exceeded, aborting before overflow at " \ + "%s:%d\n", \ + __FILE__, __LINE__); \ + KRML_HOST_EXIT(253); \ + } \ + } while (0) + /* Macros for prettier unrolling of loops */ #define KRML_LOOP1(i, n, x) { \ x \ diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index 32a7391e817ebb..1aa2ccd644c06f 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -77,7 +77,7 @@ # define le64toh(x) (x) /* ... for Windows (GCC-like, e.g. mingw or clang) */ -#elif (defined(_WIN32) || defined(_WIN64)) && \ +#elif (defined(_WIN32) || defined(_WIN64) || defined(__EMSCRIPTEN__)) && \ (defined(__GNUC__) || defined(__clang__)) # define htobe16(x) __builtin_bswap16(x) @@ -96,7 +96,8 @@ # define le64toh(x) (x) /* ... generic big-endian fallback code */ -#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +/* ... AIX doesn't have __BYTE_ORDER__ (with XLC compiler) & is always big-endian */ +#elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || defined(_AIX) /* byte swapping code inspired by: * https://github.com/rweather/arduinolibs/blob/master/libraries/Crypto/utility/EndianUtil.h diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..1c9808b8dd497c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_SHA3_H +#define __internal_Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_SHA3.h" + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +); + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Impl_SHA3_state_permute(uint64_t *s); + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index ee28f244266b85..0df236282ac509 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -59,5 +59,28 @@ #define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy #define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 +#define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 +#define Hacl_Impl_SHA3_absorb_inner python_hashlib_Hacl_Impl_SHA3_absorb_inner +#define Hacl_Impl_SHA3_keccak python_hashlib_Hacl_Impl_SHA3_keccak +#define Hacl_Impl_SHA3_loadState python_hashlib_Hacl_Impl_SHA3_loadState +#define Hacl_Impl_SHA3_squeeze python_hashlib_Hacl_Impl_SHA3_squeeze +#define Hacl_Impl_SHA3_state_permute python_hashlib_Hacl_Impl_SHA3_state_permute +#define Hacl_SHA3_sha3_224 python_hashlib_Hacl_SHA3_sha3_224 +#define Hacl_SHA3_sha3_256 python_hashlib_Hacl_SHA3_sha3_256 +#define Hacl_SHA3_sha3_384 python_hashlib_Hacl_SHA3_sha3_384 +#define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 +#define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl +#define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl +#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len +#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy +#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish +#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free +#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg +#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len +#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake +#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc +#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset +#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 76b92ec4599102..220ebbe5561341 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=13e0c6721ac9206c4249ecc1dc04ed617ad1e262 +expected_hacl_star_rev=363eae2c2eb60e46f182ddd4bd1cd3f1d00b35c9 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -45,11 +45,14 @@ dist_files=( Hacl_Hash_SHA1.h internal/Hacl_Hash_SHA1.h Hacl_Hash_MD5.h + Hacl_Hash_SHA3.h internal/Hacl_Hash_MD5.h + internal/Hacl_Hash_SHA3.h internal/Hacl_SHA2_Generic.h Hacl_Streaming_SHA2.c Hacl_Hash_SHA1.c Hacl_Hash_MD5.c + Hacl_Hash_SHA3.c ) declare -a include_files @@ -134,9 +137,9 @@ $sed -i -z 's!#include \n!#include \n#include "python_hacl_n # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. -$sed -z -i 's!#ifndef KRML_\(HOST_PRINTF\|HOST_EXIT\|PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h +$sed -z -i 's!#ifndef KRML_\(PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|CHECK_SIZE_PRAGMA\|HOST_EPRINTF\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index e66918016b8da6..0df0324df55f7d 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -206,6 +206,11 @@ get_proxies(PyObject* Py_UNUSED(mod), PyObject *Py_UNUSED(ignored)) kSCPropNetProxiesGopherProxy, kSCPropNetProxiesGopherPort); if (r == -1) goto error; + r = set_proxy(result, "socks", proxyDict, + kSCPropNetProxiesSOCKSEnable, + kSCPropNetProxiesSOCKSProxy, + kSCPropNetProxiesSOCKSPort); + if (r == -1) goto error; CFRelease(proxyDict); return result; diff --git a/Modules/_sha3/LICENSE b/Modules/_sha3/LICENSE deleted file mode 100644 index d2d484d8820dcf..00000000000000 --- a/Modules/_sha3/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Markku-Juhani O. Saarinen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/Modules/_sha3/README.txt b/Modules/_sha3/README.txt deleted file mode 100644 index b35919b01677d3..00000000000000 --- a/Modules/_sha3/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -tiny_sha3 -========= - -https://github.com/mjosaarinen/tiny_sha3 -commit dcbb3192047c2a721f5f851db591871d428036a9 - -- All functions have been converted to static functions. -- sha3() function is commented out. diff --git a/Modules/_sha3/sha3.c b/Modules/_sha3/sha3.c deleted file mode 100644 index e2d3fd7b8ad855..00000000000000 --- a/Modules/_sha3/sha3.c +++ /dev/null @@ -1,193 +0,0 @@ -// sha3.c -// 19-Nov-11 Markku-Juhani O. Saarinen - -// Revised 07-Aug-15 to match with official release of FIPS PUB 202 "SHA3" -// Revised 03-Sep-15 for portability + OpenSSL - style API - -#include "sha3.h" - -// update the state with given number of rounds - -static void sha3_keccakf(uint64_t st[25]) -{ - // constants - const uint64_t keccakf_rndc[24] = { - 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, - 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, - 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, - 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, - 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, - 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, - 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, - 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 - }; - const int keccakf_rotc[24] = { - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, - 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 - }; - const int keccakf_piln[24] = { - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, - 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 - }; - - // variables - int i, j, r; - uint64_t t, bc[5]; - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - uint8_t *v; - - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - st[i] = ((uint64_t) v[0]) | (((uint64_t) v[1]) << 8) | - (((uint64_t) v[2]) << 16) | (((uint64_t) v[3]) << 24) | - (((uint64_t) v[4]) << 32) | (((uint64_t) v[5]) << 40) | - (((uint64_t) v[6]) << 48) | (((uint64_t) v[7]) << 56); - } -#endif - - // actual iteration - for (r = 0; r < KECCAKF_ROUNDS; r++) { - - // Theta - for (i = 0; i < 5; i++) - bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; - - for (i = 0; i < 5; i++) { - t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1); - for (j = 0; j < 25; j += 5) - st[j + i] ^= t; - } - - // Rho Pi - t = st[1]; - for (i = 0; i < 24; i++) { - j = keccakf_piln[i]; - bc[0] = st[j]; - st[j] = ROTL64(t, keccakf_rotc[i]); - t = bc[0]; - } - - // Chi - for (j = 0; j < 25; j += 5) { - for (i = 0; i < 5; i++) - bc[i] = st[j + i]; - for (i = 0; i < 5; i++) - st[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; - } - - // Iota - st[0] ^= keccakf_rndc[r]; - } - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - t = st[i]; - v[0] = t & 0xFF; - v[1] = (t >> 8) & 0xFF; - v[2] = (t >> 16) & 0xFF; - v[3] = (t >> 24) & 0xFF; - v[4] = (t >> 32) & 0xFF; - v[5] = (t >> 40) & 0xFF; - v[6] = (t >> 48) & 0xFF; - v[7] = (t >> 56) & 0xFF; - } -#endif -} - -// Initialize the context for SHA3 - -static int sha3_init(sha3_ctx_t *c, int mdlen) -{ - int i; - - for (i = 0; i < 25; i++) - c->st.q[i] = 0; - c->mdlen = mdlen; - c->rsiz = 200 - 2 * mdlen; - c->pt = 0; - - return 1; -} - -// update state with more data - -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - c->st.b[j++] ^= ((const uint8_t *) data)[i]; - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - } - c->pt = j; - - return 1; -} - -// finalize and output a hash - -static int sha3_final(void *md, sha3_ctx_t *c) -{ - int i; - - c->st.b[c->pt] ^= 0x06; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - - for (i = 0; i < c->mdlen; i++) { - ((uint8_t *) md)[i] = c->st.b[i]; - } - - return 1; -} - -#if 0 -// compute a SHA-3 hash (md) of given byte length from "in" - -void *sha3(const void *in, size_t inlen, void *md, int mdlen) -{ - sha3_ctx_t sha3; - - sha3_init(&sha3, mdlen); - sha3_update(&sha3, in, inlen); - sha3_final(md, &sha3); - - return md; -} -#endif - -// SHAKE128 and SHAKE256 extensible-output functionality - -static void shake_xof(sha3_ctx_t *c) -{ - c->st.b[c->pt] ^= 0x1F; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - c->pt = 0; -} - -static void shake_out(sha3_ctx_t *c, void *out, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - ((uint8_t *) out)[i] = c->st.b[j++]; - } - c->pt = j; -} - diff --git a/Modules/_sha3/sha3.h b/Modules/_sha3/sha3.h deleted file mode 100644 index f973d6733ec2cc..00000000000000 --- a/Modules/_sha3/sha3.h +++ /dev/null @@ -1,49 +0,0 @@ -// sha3.h -// 19-Nov-11 Markku-Juhani O. Saarinen - -#ifndef SHA3_H -#define SHA3_H - -#include -#include - -#ifndef KECCAKF_ROUNDS -#define KECCAKF_ROUNDS 24 -#endif - -#ifndef ROTL64 -#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) -#endif - -// state context -typedef struct { - union { // state: - uint8_t b[200]; // 8-bit bytes - uint64_t q[25]; // 64-bit words - } st; - int pt, rsiz, mdlen; // these don't overflow -} sha3_ctx_t; - -// Compression function. -static void sha3_keccakf(uint64_t st[25]); - -// OpenSSL - like interfece -static int sha3_init(sha3_ctx_t *c, int mdlen); // mdlen = hash output in bytes -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len); -static int sha3_final(void *md, sha3_ctx_t *c); // digest goes to md - -// compute a sha3 hash (md) of given byte length from "in" -#if 0 -static void *sha3(const void *in, size_t inlen, void *md, int mdlen); -#endif - -// SHAKE128 and SHAKE256 extensible-output functions -#define shake128_init(c) sha3_init(c, 16) -#define shake256_init(c) sha3_init(c, 32) -#define shake_update sha3_update - -static void shake_xof(sha3_ctx_t *c); -static void shake_out(sha3_ctx_t *c, void *out, size_t len); - -#endif - diff --git a/Modules/_struct.c b/Modules/_struct.c index 26434f714de5cc..4f9478bd98095d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1832,11 +1832,6 @@ unpackiter_iternext(unpackiterobject *self) return result; } -PyObject *unpackiter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyErr_Format(PyExc_TypeError, "Cannot create '%.200s objects", _PyType_Name(type)); - return NULL; -} - static PyType_Slot unpackiter_type_slots[] = { {Py_tp_dealloc, unpackiter_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1844,7 +1839,6 @@ static PyType_Slot unpackiter_type_slots[] = { {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, unpackiter_iternext}, {Py_tp_methods, unpackiter_methods}, - {Py_tp_new, unpackiter_new}, {0, 0}, }; @@ -1853,7 +1847,7 @@ static PyType_Spec unpackiter_type_spec = { sizeof(unpackiterobject), 0, (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), unpackiter_type_slots }; diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 10e1733d08a9ea..9f81389811c645 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -1,6 +1,6 @@ #include "parts.h" -int verify_immortality(PyObject *object) +int verify_immortality(PyObject *object) { assert(_Py_IsImmortal(object)); Py_ssize_t old_count = Py_REFCNT(object); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f35e3b48df9321..40ad6f88868daf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -593,17 +593,19 @@ _testinternalcapi.compiler_codegen -> object ast: object filename: object optimize: int + compile_mode: int = 0 Apply compiler code generation to an AST. [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize) -/*[clinic end generated code: output=fbbbbfb34700c804 input=e9fbe6562f7f75e4]*/ + PyObject *filename, int optimize, + int compile_mode) +/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ { PyCompilerFlags *flags = NULL; - return _PyCompile_CodeGen(ast, filename, flags, optimize); + return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); } diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 89573222572594..41dd50437956c4 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -9,7 +9,7 @@ preserve PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, -"compiler_codegen($module, /, ast, filename, optimize)\n" +"compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" "Apply compiler code generation to an AST."); @@ -19,7 +19,8 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize); + PyObject *filename, int optimize, + int compile_mode); static PyObject * _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -27,14 +28,14 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), }, + .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), &_Py_ID(compile_mode), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -43,19 +44,21 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"ast", "filename", "optimize", NULL}; + static const char * const _keywords[] = {"ast", "filename", "optimize", "compile_mode", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compiler_codegen", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *ast; PyObject *filename; int optimize; + int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); if (!args) { goto exit; } @@ -65,7 +68,15 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s if (optimize == -1 && PyErr_Occurred()) { goto exit; } - return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize); + if (!noptargs) { + goto skip_optional_pos; + } + compile_mode = _PyLong_AsInt(args[3]); + if (compile_mode == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize, compile_mode); exit: return return_value; @@ -190,4 +201,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab661d56a14b1a1c input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h similarity index 98% rename from Modules/_sha3/clinic/sha3module.c.h rename to Modules/clinic/sha3module.c.h index a0c7c1c043e515..299803a3420bf6 100644 --- a/Modules/_sha3/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" "--\n" "\n" -"Return a new BLAKE2b hash object."); +"Return a new SHA3 hash object."); static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); @@ -193,4 +193,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=747c3f34ddd14063 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=907cb475f3dc9ee0 input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/sha3module.c b/Modules/sha3module.c similarity index 75% rename from Modules/_sha3/sha3module.c rename to Modules/sha3module.c index 93abc3b2710e55..f05187498a19b3 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/sha3module.c @@ -22,23 +22,9 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "../hashlib.h" - -#include "sha3.c" +#include "hashlib.h" #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ -#define SHA3_LANESIZE 0 -#define SHA3_state sha3_ctx_t -#define SHA3_init sha3_init -#define SHA3_process sha3_update -#define SHA3_done(state, digest) sha3_final(digest, state) -#define SHA3_squeeze(state, out, len) shake_xof(state), shake_out(state, out, len) -#define SHA3_copystate(dest, src) memcpy(&(dest), &(src), sizeof(SHA3_state)) - -// no optimization -#define KeccakOpt 0 - -typedef enum { SUCCESS = 1, FAIL = 0, BAD_HASHLEN = 2 } HashReturn; typedef struct { PyTypeObject *sha3_224_type; @@ -70,10 +56,11 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" /* The structure for storing SHA3 info */ +#include "_hacl/Hacl_Hash_SHA3.h" + typedef struct { PyObject_HEAD - SHA3_state hash_state; - PyThread_type_lock lock; + Hacl_Streaming_Keccak_state *hash_state; } SHA3object; #include "clinic/sha3module.c.h" @@ -86,10 +73,23 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } - newobj->lock = NULL; return newobj; } +static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to hash more than 2^64 bytes. */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); +} + /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new @@ -98,14 +98,13 @@ _sha3.sha3_224.__new__ as py_sha3_new * usedforsecurity: bool = True -Return a new BLAKE2b hash object. +Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=bcfcdf2e4368347a]*/ +/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ { - HashReturn res; Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -116,49 +115,29 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - res = sha3_init(&self->hash_state, 28); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - res = sha3_init(&self->hash_state, 48); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - res = sha3_init(&self->hash_state, 64); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - res = sha3_init(&self->hash_state, 16); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 initialize()"); - goto error; - } - if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* invariant: New objects can't be accessed by other code yet, - * thus it's safe to release the GIL without locking the object. - */ - Py_BEGIN_ALLOW_THREADS - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - goto error; - } - PyBuffer_Release(&buf); + sha3_update(self->hash_state, buf.buf, buf.len); } + PyBuffer_Release(&buf); + return (PyObject *)self; error: @@ -177,10 +156,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) static void SHA3_dealloc(SHA3object *self) { - if (self->lock) { - PyThread_free_lock(self->lock); - } - + Hacl_Streaming_Keccak_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -205,9 +181,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { return NULL; } - ENTER_HASHLIB(self); - SHA3_copystate(newobj->hash_state, self->hash_state); - LEAVE_HASHLIB(self); + newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); return (PyObject *)newobj; } @@ -222,20 +196,12 @@ static PyObject * _sha3_sha3_224_digest_impl(SHA3object *self) /*[clinic end generated code: output=fd531842e20b2d5b input=5b2a659536bbd248]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + // This function errors out if the algorithm is Shake. Here, we know this + // not to be the case, and therefore do not perform error checking. + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -249,21 +215,10 @@ static PyObject * _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return _Py_strhex((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -281,36 +236,8 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; - HashReturn res; - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - /* add new data, the function takes the length in bits not bytes */ - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - } - /* Once a lock exists all code paths must be synchronized. We have to - * release the GIL even for small buffers as acquiring the lock may take - * an unlimited amount of time when another thread updates this object - * with lots of data. */ - if (self->lock) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - - if (res != SUCCESS) { - PyBuffer_Release(&buf); - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - return NULL; - } - + sha3_update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -328,7 +255,7 @@ static PyMethodDef SHA3_methods[] = { static PyObject * SHA3_get_block_size(SHA3object *self, void *closure) { - int rate = self->hash_state.rsiz; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); return PyLong_FromLong(rate); } @@ -363,14 +290,19 @@ SHA3_get_name(SHA3object *self, void *closure) static PyObject * SHA3_get_digest_size(SHA3object *self, void *closure) { - return PyLong_FromLong(self->hash_state.mdlen); + // Preserving previous behavior: variable-length algorithms return 0 + if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + return PyLong_FromLong(0); + else + return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); } static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { - int capacity = 1600 - self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -378,7 +310,7 @@ SHA3_get_capacity_bits(SHA3object *self, void *closure) static PyObject * SHA3_get_rate_bits(SHA3object *self, void *closure) { - unsigned int rate = self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } @@ -455,28 +387,26 @@ static PyObject * _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) { unsigned char *digest = NULL; - SHA3_state temp; PyObject *result = NULL; if (digestlen >= (1 << 29)) { PyErr_SetString(PyExc_ValueError, "length is too large"); return NULL; } - /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and - * SHA3_LANESIZE extra space. - */ - digest = (unsigned char*)PyMem_Malloc(digestlen + SHA3_LANESIZE); + digest = (unsigned char*)PyMem_Malloc(digestlen); if (digest == NULL) { return PyErr_NoMemory(); } - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - SHA3_squeeze(&temp, digest, digestlen); + /* Get the raw (binary) digest value. The HACL functions errors out if: + * - the algorith is not shake -- not the case here + * - the output length is zero -- we follow the existing behavior and return + * an empty digest, without raising an error */ + if (digestlen > 0) { + Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + } if (hex) { - result = _Py_strhex((const char *)digest, digestlen); + result = _Py_strhex((const char *)digest, digestlen); } else { result = PyBytes_FromStringAndSize((const char *)digest, digestlen); @@ -628,11 +558,8 @@ _sha3_exec(PyObject *m) init_sha3type(shake_256_type, SHAKE256_spec); #undef init_sha3type - if (PyModule_AddIntConstant(m, "keccakopt", KeccakOpt) < 0) { - return -1; - } if (PyModule_AddStringConstant(m, "implementation", - "tiny_sha3") < 0) { + "HACL") < 0) { return -1; } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 60219593be61e2..c11fb4400eab2f 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -6883,8 +6883,10 @@ socket_getnameinfo(PyObject *self, PyObject *args) } #endif } + Py_BEGIN_ALLOW_THREADS error = getnameinfo(res->ai_addr, (socklen_t) res->ai_addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf), flags); + Py_END_ALLOW_THREADS if (error) { socket_state *state = get_module_state(self); set_gaierror(state, error); diff --git a/Modules/termios.c b/Modules/termios.c index 169a36fc6477d8..6dc8200572bc0c 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -85,7 +85,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) int r; Py_BEGIN_ALLOW_THREADS - r = tcgetattr(fd, &mode); + r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); @@ -372,7 +372,7 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #if defined(TIOCGWINSZ) termiosmodulestate *state = PyModule_GetState(module); struct winsize w; - int r; + int r; Py_BEGIN_ALLOW_THREADS r = ioctl(fd, TIOCGWINSZ, &w); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 49d4dd524696a5..c36db59baaa10d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -61,6 +61,7 @@ static void bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) { obj->ob_exports--; + assert(obj->ob_exports >= 0); } static int diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f008a8cc3e0474..b0168044d9f85a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -193,6 +193,20 @@ PyTypeObject _PyManagedBuffer_Type = { return -1; \ } +#define CHECK_RESTRICTED(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return NULL; \ + } + +#define CHECK_RESTRICTED_INT(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return -1; \ + } + /* See gh-92888. These macros signal that we need to check the memoryview again due to possible read after frees. */ #define CHECK_RELEASED_AGAIN(mv) CHECK_RELEASED(mv) @@ -781,7 +795,7 @@ PyMemoryView_FromBuffer(const Py_buffer *info) using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ -PyObject * +static PyObject * PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -789,6 +803,7 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) if (PyMemoryView_Check(v)) { PyMemoryViewObject *mv = (PyMemoryViewObject *)v; CHECK_RELEASED(mv); + CHECK_RESTRICTED(mv); return mbuf_add_view(mv->mbuf, &mv->view); } else if (PyObject_CheckBuffer(v)) { @@ -806,6 +821,30 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) Py_TYPE(v)->tp_name); return NULL; } + +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +_PyMemoryView_FromBufferProc(PyObject *v, int flags, getbufferproc bufferproc) +{ + _PyManagedBufferObject *mbuf = mbuf_alloc(); + if (mbuf == NULL) + return NULL; + + int res = bufferproc(v, &mbuf->master, flags); + if (res < 0) { + mbuf->master.obj = NULL; + Py_DECREF(mbuf); + return NULL; + } + + PyObject *ret = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + return ret; +} + /* Create a memoryview from an object that implements the buffer protocol. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ @@ -1397,6 +1436,7 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, Py_ssize_t ndim = 1; CHECK_RELEASED(self); + CHECK_RESTRICTED(self); if (!MV_C_CONTIGUOUS(self->flags)) { PyErr_SetString(PyExc_TypeError, @@ -1452,6 +1492,7 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=2c7e056f04c99e62 input=dc06d20f19ba236f]*/ { CHECK_RELEASED(self); + CHECK_RESTRICTED(self); /* Even if self is already readonly, we still need to create a new * object for .release() to work correctly. */ @@ -1474,6 +1515,7 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) int baseflags = self->flags; CHECK_RELEASED_INT(self); + CHECK_RESTRICTED_INT(self); /* start with complete information */ *view = *base; @@ -2535,6 +2577,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) return memory_item(self, index); } else if (PySlice_Check(key)) { + CHECK_RESTRICTED(self); PyMemoryViewObject *sliced; sliced = (PyMemoryViewObject *)mbuf_add_view(self->mbuf, view); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 456b10ee01d6bc..98fac276a873e1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,7 +6,7 @@ #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() +#include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_long.h" // _PyLong_IsNegative() @@ -56,6 +56,11 @@ typedef struct PySlot_Offset { short slot_offset; } PySlot_Offset; +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer); + +static void +releasebuffer_call_python(PyObject *self, Py_buffer *buffer); static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -8078,7 +8083,8 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } - return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); + return _PyMemoryView_FromBufferProc(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int), + (getbufferproc)wrapped); } static PyObject * @@ -8094,6 +8100,10 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; + if (mview->view.obj == NULL) { + // Already released, ignore + Py_RETURN_NONE; + } if (mview->view.obj != self) { PyErr_SetString(PyExc_ValueError, "memoryview's buffer is not this object"); @@ -8978,12 +8988,26 @@ bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) { PyBufferWrapper *bw = (PyBufferWrapper *)self; - assert(PyMemoryView_Check(bw->mv)); - Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); - if (Py_TYPE(bw->obj)->tp_as_buffer != NULL - && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { - Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + if (bw->mv == NULL || bw->obj == NULL) { + // Already released + return; + } + + PyObject *mv = bw->mv; + PyObject *obj = bw->obj; + + assert(PyMemoryView_Check(mv)); + Py_TYPE(mv)->tp_as_buffer->bf_releasebuffer(mv, view); + // We only need to call bf_releasebuffer if it's a Python function. If it's a C + // bf_releasebuf, it will be called when the memoryview is released. + if (((PyMemoryViewObject *)mv)->view.obj != obj + && Py_TYPE(obj)->tp_as_buffer != NULL + && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer == slot_bf_releasebuffer) { + releasebuffer_call_python(obj, view); } + + Py_CLEAR(bw->mv); + Py_CLEAR(bw->obj); } static PyBufferProcs bufferwrapper_as_buffer = { @@ -9047,31 +9071,112 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } +static int +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + PyTypeObject *self_type = Py_TYPE(self); + PyObject *mro = lookup_tp_mro(self_type); + if (mro == NULL) { + return -1; + } + + assert(PyTuple_Check(mro)); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + Py_ssize_t i; + + /* No need to check the last one: it's gonna be skipped anyway. */ + for (i = 0; i < n -1; i++) { + if ((PyObject *)(self_type) == PyTuple_GET_ITEM(mro, i)) + break; + } + i++; /* skip self_type */ + if (i >= n) + return -1; + + releasebufferproc base_releasebuffer = NULL; + for (; i < n; i++) { + PyObject *obj = PyTuple_GET_ITEM(mro, i); + if (!PyType_Check(obj)) { + continue; + } + PyTypeObject *base_type = (PyTypeObject *)obj; + if (base_type->tp_as_buffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { + base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; + break; + } + } + + if (base_releasebuffer != NULL) { + base_releasebuffer(self, buffer); + } + return 0; +} + static void -slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +releasebuffer_call_python(PyObject *self, Py_buffer *buffer) { PyObject *mv; - if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + bool is_buffer_wrapper = Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type; + if (is_buffer_wrapper) { // Make sure we pass the same memoryview to // __release_buffer__() that __buffer__() returned. - mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + PyBufferWrapper *bw = (PyBufferWrapper *)buffer->obj; + if (bw->mv == NULL) { + return; + } + mv = Py_NewRef(bw->mv); } else { + // This means we are not dealing with a memoryview returned + // from a Python __buffer__ function. mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { PyErr_WriteUnraisable(self); return; } + // Set the memoryview to restricted mode, which forbids + // users from saving any reference to the underlying buffer + // (e.g., by doing .cast()). This is necessary to ensure + // no Python code retains a reference to the to-be-released + // buffer. + ((PyMemoryViewObject *)mv)->flags |= _Py_MEMORYVIEW_RESTRICTED; } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); - Py_DECREF(mv); if (ret == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(ret); } + if (!is_buffer_wrapper) { + PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + } + Py_DECREF(mv); +} + +/* + * bf_releasebuffer is very delicate, because we need to ensure that + * C bf_releasebuffer slots are called correctly (or we'll leak memory), + * but we cannot trust any __release_buffer__ implemented in Python to + * do so correctly. Therefore, if a base class has a C bf_releasebuffer + * slot, we call it directly here. That is safe because this function + * only gets called from C callers of the bf_releasebuffer slot. Python + * code that calls __release_buffer__ directly instead goes through + * wrap_releasebuffer(), which doesn't call the bf_releasebuffer slot + * directly but instead simply releases the associated memoryview. + */ +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + releasebuffer_call_python(self, buffer); + if (releasebuffer_maybe_call_super(self, buffer) < 0) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } } static PyObject * diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8aafcb786a6064..28b1517c6f6b3a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -375,13 +375,16 @@ + + + + - @@ -404,17 +407,15 @@ - - - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 07476f30833372..75e6fbb13f98ba 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -776,6 +776,15 @@ Modules + + Modules + + + Modules + + + Modules + Modules @@ -794,9 +803,6 @@ Modules - - Modules - Modules @@ -875,16 +881,13 @@ Modules - - Modules - Modules - + Modules - + Modules diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 52d0d9a534cb6a..91ffabac56c7b3 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -48,7 +48,7 @@ static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { } static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); } #else diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 9958856bae8019..42e1436bc9130d 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -464,8 +464,7 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - /* XXX per-interpreter GIL */ - PyInterpreterState *interp = _PyInterpreterState_Main(); + PyInterpreterState *interp = _PyInterpreterState_Get(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); gil->interval = microseconds; @@ -473,8 +472,7 @@ void _PyEval_SetSwitchInterval(unsigned long microseconds) unsigned long _PyEval_GetSwitchInterval(void) { - /* XXX per-interpreter GIL */ - PyInterpreterState *interp = _PyInterpreterState_Main(); + PyInterpreterState *interp = _PyInterpreterState_Get(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); return gil->interval; @@ -484,7 +482,9 @@ unsigned long _PyEval_GetSwitchInterval(void) int _PyEval_ThreadsInitialized(void) { - /* XXX per-interpreter GIL */ + /* XXX This is only needed for an assert in PyGILState_Ensure(), + * which currently does not work with subinterpreters. + * Thus we only use the main interpreter. */ PyInterpreterState *interp = _PyInterpreterState_Main(); if (interp == NULL) { return 0; @@ -532,27 +532,16 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) assert(tstate->interp->ceval.gil == NULL); int locked; if (!own_gil) { + /* The interpreter will share the main interpreter's instead. */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(tstate->interp != main_interp); struct _gil_runtime_state *gil = main_interp->ceval.gil; init_shared_gil(tstate->interp, gil); locked = current_thread_holds_gil(gil, tstate); } - /* XXX per-interpreter GIL */ - else if (!_Py_IsMainInterpreter(tstate->interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - struct _gil_runtime_state *main_gil = _PyInterpreterState_Main()->ceval.gil; - init_shared_gil(tstate->interp, main_gil); - // XXX For now we lie. - tstate->interp->ceval.own_gil = 1; - locked = current_thread_holds_gil(main_gil, tstate); - } else { PyThread_init_thread(); - // XXX per-interpreter GIL: switch to interp->_gil. - init_own_gil(tstate->interp, &tstate->interp->runtime->ceval.gil); + init_own_gil(tstate->interp, &tstate->interp->_gil); locked = 0; } if (!locked) { @@ -565,7 +554,8 @@ _PyEval_InitGIL(PyThreadState *tstate, int own_gil) void _PyEval_FiniGIL(PyInterpreterState *interp) { - if (interp->ceval.gil == NULL) { + struct _gil_runtime_state *gil = interp->ceval.gil; + if (gil == NULL) { /* It was already finalized (or hasn't been initialized yet). */ assert(!interp->ceval.own_gil); return; @@ -573,24 +563,13 @@ _PyEval_FiniGIL(PyInterpreterState *interp) else if (!interp->ceval.own_gil) { #ifdef Py_DEBUG PyInterpreterState *main_interp = _PyInterpreterState_Main(); - assert(interp != main_interp); + assert(main_interp != NULL && interp != main_interp); assert(interp->ceval.gil == main_interp->ceval.gil); #endif interp->ceval.gil = NULL; return; } - /* XXX per-interpreter GIL */ - struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; - if (!_Py_IsMainInterpreter(interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - assert(interp->ceval.gil == gil); - interp->ceval.gil = NULL; - return; - } - if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -974,21 +953,13 @@ Py_MakePendingCalls(void) return 0; } -/* The interpreter's recursion limit */ - void -_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) +_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock) { - /* XXX per-interpreter GIL */ - _gil_initialize(&ceval->gil); -} + _gil_initialize(&interp->_gil); -void -_PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock) -{ - struct _pending_calls *pending = &ceval->pending; + struct _pending_calls *pending = &interp->ceval.pending; assert(pending->lock == NULL); - pending->lock = pending_lock; } diff --git a/Python/compile.c b/Python/compile.c index cbe5403aafbc87..f875e4e17e0a9f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7258,7 +7258,7 @@ cfg_to_instructions(cfg_builder *g) PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, - int optimize) + int optimize, int compile_mode) { PyObject *res = NULL; @@ -7272,7 +7272,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); + mod_ty mod = PyAST_obj2mod(ast, arena, compile_mode); if (mod == NULL || !_PyAST_Validate(mod)) { _PyArena_Free(arena); return NULL; @@ -7287,6 +7287,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (compiler_codegen(c, mod) < 0) { goto finally; } + int addNone = mod->kind != Expression_kind; + if (add_return_at_end(c, addNone) < 0) { + return NULL; + } res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 1957ab82c33951..3b183280e1f24c 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -253,7 +253,7 @@ perf_map_write_entry(void *state, const void *code_addr, NULL); return; } - fprintf(method_file, "%p %x py::%s:%s\n", code_addr, code_size, entry, + fprintf(method_file, "%" PRIxPTR " %x py::%s:%s\n", (uintptr_t) code_addr, code_size, entry, filename); fflush(method_file); } diff --git a/Python/pystate.c b/Python/pystate.c index f14934361dab78..26debf1f88b94a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -425,8 +425,6 @@ init_runtime(_PyRuntimeState *runtime, runtime->open_code_userdata = open_code_userdata; runtime->audit_hook_head = audit_hook_head; - _PyEval_InitRuntimeState(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); PyThread_type_lock *lockptrs[NUMLOCKS] = { @@ -682,7 +680,7 @@ init_interpreter(PyInterpreterState *interp, memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } - _PyEval_InitState(&interp->ceval, pending_lock); + _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 5924ab7860d8d5..9bd54db0f59c51 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -114,6 +114,7 @@ def clean_lines(text): Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include +Modules/sha3module.c Modules/_hacl/include Objects/stringlib/*.h Objects # possible system-installed headers, just in case @@ -271,13 +272,6 @@ def clean_lines(text): Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 -# from Modules/_sha3/sha3module.c -Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian -Modules/_sha3/kcp/*.c KeccakOpt 64 -Modules/_sha3/kcp/*.c KeccakP200_excluded 1 -Modules/_sha3/kcp/*.c KeccakP400_excluded 1 -Modules/_sha3/kcp/*.c KeccakP800_excluded 1 - # others Modules/_sre/sre_lib.h LOCAL(type) static inline type Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 704325670bc994..a6f330d1502dad 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -356,6 +356,7 @@ def __init__(self): # you should check the _return_value for errors, and # "goto exit" if there are any. self.return_conversion = [] + self.converter_retval = "_return_value" # The C statements required to do some operations # after the end of parsing but before cleaning up. @@ -3893,15 +3894,15 @@ def __init__(self, *, py_default=None, **kwargs): def return_converter_init(self): pass - def declare(self, data, name="_return_value"): + def declare(self, data): line = [] add = line.append add(self.type) if not self.type.endswith('*'): add(' ') - add(name + ';') + add(data.converter_retval + ';') data.declarations.append(''.join(line)) - data.return_value = name + data.return_value = data.converter_retval def err_occurred_if(self, expr, data): data.return_conversion.append('if (({}) && PyErr_Occurred()) {{\n goto exit;\n}}\n'.format(expr)) @@ -3923,8 +3924,10 @@ class bool_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1", data) - data.return_conversion.append('return_value = PyBool_FromLong((long)_return_value);\n') + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) class long_return_converter(CReturnConverter): type = 'long' @@ -3934,9 +3937,10 @@ class long_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == {}-1".format(self.unsigned_cast), data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( - ''.join(('return_value = ', self.conversion_fn, '(', self.cast, '_return_value);\n'))) + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) class int_return_converter(long_return_converter): type = 'int' @@ -3978,9 +3982,10 @@ class double_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1.0", data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( - 'return_value = PyFloat_FromDouble(' + self.cast + '_return_value);\n') + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) class float_return_converter(double_return_converter): type = 'float' diff --git a/Tools/msi/bundle/bootstrap/pch.h b/Tools/msi/bundle/bootstrap/pch.h index b0aa5111dabd0d..6d0974b34c61e7 100644 --- a/Tools/msi/bundle/bootstrap/pch.h +++ b/Tools/msi/bundle/bootstrap/pch.h @@ -5,7 +5,7 @@ // The license and further copyright text can be found in the file // LICENSE.TXT at the root directory of the distribution. // -// +// // // Precompiled header for standard bootstrapper application. // diff --git a/Tools/msi/bundle/bootstrap/resource.h b/Tools/msi/bundle/bootstrap/resource.h index 53c03c319f091f..d951e651f6d20d 100644 --- a/Tools/msi/bundle/bootstrap/resource.h +++ b/Tools/msi/bundle/bootstrap/resource.h @@ -14,7 +14,7 @@ // Next default values for new objects -// +// #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 102 diff --git a/configure b/configure index b8fa9d66e735dd..c9ea72cf6efacf 100755 --- a/configure +++ b/configure @@ -6143,6 +6143,20 @@ cat > conftest.c <=6) && defined(_MIPSEL) diff --git a/configure.ac b/configure.ac index 0940b93c25f7bf..10672bd3761da8 100644 --- a/configure.ac +++ b/configure.ac @@ -959,6 +959,20 @@ cat > conftest.c <=6) && defined(_MIPSEL)