Merge branch 'main' into extract_from_wsgi

.github/workflows/tests.yaml

@@ -35,7 +35,7 @@ jobs:
           - {name: Typing, python: '3.10', os: ubuntu-latest, tox: typing}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
           cache: 'pip'
@@ -46,7 +46,7 @@ jobs:
           pip install -U setuptools
           python -m pip install -U pip
       - name: cache mypy
-        uses: actions/cache@v3.0.2
+        uses: actions/cache@v3.0.4
         with:
           path: ./.mypy_cache
           key: mypy|${{ matrix.python }}|${{ hashFiles('setup.cfg') }}

.pre-commit-config.yaml

@@ -3,12 +3,12 @@ ci:
   autoupdate_schedule: monthly
 repos:
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.0
+    rev: v2.37.1
     hooks:
       - id: pyupgrade
         args: ["--py37-plus"]
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.1.0
+    rev: v3.8.1
     hooks:
       - id: reorder-python-imports
         name: Reorder Python imports (src, tests)
@@ -21,7 +21,7 @@ repos:
         args: ["--application-directories", "examples"]
         additional_dependencies: ["setuptools>60.9"]
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.6.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
@@ -36,7 +36,7 @@ repos:
     hooks:
       - id: pip-compile-multi-verify
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.3.0
     hooks:
       - id: fix-byte-order-marker
       - id: trailing-whitespace

CHANGES.rst

@@ -3,9 +3,25 @@
 Version 2.2.0
 -------------
 
-Unreleased
-
--   Extracted utility functions from wsgi.py
+-   Add MarkupSafe as a dependency and use it to escape values when
+    rendering HTML. :issue:`2419`
+-   Added the ``werkzeug.debug.preserve_context`` mechanism for
+    restoring context-local data for a request when running code in the
+    debug console. :pr:`2439`
+-   Fix compatibility with Python 3.11 by ensuring that ``end_lineno``
+    and ``end_col_offset`` are present on AST nodes. :issue:`2425`
+-   Add a new faster matching router based on a state
+    machine. :pr:`2433`
+-   Names within options headers are always converted to lowercase. This
+    matches :rfc:`6266` that the case is not relevant. :issue:`2442`
+-   ``AnyConverter`` validates the value passed for it when building
+    URLs. :issue:`2388`
+-   The debugger shows enhanced error locations in tracebacks in Python
+    3.11. :issue:`2407`
+-   Extracted get_content_length, get_query_string, get_path_info
+    utility functions from wsgi.py. :pr:`2415`
+-   Extracted is_resource_modified and parse_cookie from http.py
+    to sansio/http.py. :issue:`2408`
 
 
 Version 2.1.2

docs/levels.rst

@@ -19,7 +19,7 @@ user with the name entered.
 
 .. code-block:: python
 
-    from html import escape
+    from markupsafe import escape
     from werkzeug.wrappers import Request, Response
 
     @Request.application
@@ -38,7 +38,7 @@ user with the name entered.
 Alternatively the same application could be used without request and response
 objects but by taking advantage of the parsing functions werkzeug provides::
 
-    from html import escape
+    from markupsafe import escape
     from werkzeug.formparser import parse_form_data
 
     def hello_world(environ, start_response):

docs/local.rst

@@ -1,77 +1,110 @@
-==============
 Context Locals
 ==============
 
 .. module:: werkzeug.local
 
-Sooner or later you have some things you want to have in every single view
-or helper function or whatever.  In PHP the way to go are global
-variables.  However, that isn't possible in WSGI applications without a
-major drawback:  As soon as you operate on the global namespace your
-application isn't thread-safe any longer.
+You may find that you have some data during each request that you want
+to use across functions. Instead of passing these as arguments between
+every function, you may want to access them as global data. However,
+using global variables in Python web applications is not thread safe;
+different workers might interfere with each others' data.
+
+Instead of storing common data during a request using global variables,
+you must use context-local variables instead. A context local is
+defined/imported globally, but the data it contains is specific to the
+current thread, asyncio task, or greenlet. You won't accidentally get
+or overwrite another worker's data.
+
+The current approach for storing per-context data in Python is the
+:class:`contextvars` module. Context vars store data per thread, async
+task, or greenlet. This replaces the older :class:`threading.local`
+which only handled threads.
 
-The Python standard library has a concept called "thread locals" (or thread-local
-data). A thread local is a global object in which you can put stuff in and get back
-later in a thread-safe and thread-specific way.  That means that whenever you set
-or get a value on a thread local object, the thread local object checks in which
-thread you are and retrieves the value corresponding to your thread (if one exists).
-So, you won't accidentally get another thread's data.
+Werkzeug provides wrappers around :class:`~contextvars.ContextVar` to
+make it easier to work with.
 
-This approach, however, has a few disadvantages.  For example, besides threads,
-there are other types of concurrency in Python.  A very popular one
-is greenlets.  Also, whether every request gets its own thread is not
-guaranteed in WSGI.  It could be that a request is reusing a thread from
-a previous request, and hence data is left over in the thread local object.
 
-Werkzeug provides its own implementation of local data storage called `werkzeug.local`.
-This approach provides a similar functionality to thread locals but also works with
-greenlets.
+Proxy Objects
+=============
 
-Here's a simple example of how one could use werkzeug.local::
+:class:`LocalProxy` allows treating a context var as an object directly
+instead of needing to use and check
+:meth:`ContextVar.get() <contextvars.ContextVar.get>`. If the context
+var is set, the local proxy will look and behave like the object the var
+is set to. If it's not set, a ``RuntimeError`` is raised for most
+operations.
 
-    from werkzeug.local import Local, LocalManager
+.. code-block:: python
 
-    local = Local()
-    local_manager = LocalManager([local])
+    from contextvars import ContextVar
+    from werkzeug.local import LocalProxy
 
-    def application(environ, start_response):
-        local.request = request = Request(environ)
+    _request_var = ContextVar("request")
+    request = LocalProxy(_request_var)
+
+    from werkzeug.wrappers import Request
+
+    @Request.application
+    def app(r):
+        _request_var.set(r)
+        check_auth()
         ...
 
-    application = local_manager.make_middleware(application)
+    from werkzeug.exceptions import Unauthorized
 
-This binds the request to `local.request`.  Every other piece of code executed
-after this assignment in the same context can safely access local.request and
-will get the same request object.  The `make_middleware` method on the local
-manager ensures that all references to the local objects are cleared up after
-the request.
+    def check_auth():
+        if request.form["username"] != "admin":
+            raise Unauthorized()
 
-The same context means the same greenlet (if you're using greenlets) in
-the same thread and same process.
+Accessing ``request`` will point to the specific request that each
+server worker is handling. You can treat ``request`` just like an actual
+``Request`` object.
 
-If a request object is not yet set on the local object and you try to
-access it, you will get an `AttributeError`.  You can use `getattr` to avoid
-that::
+``bool(proxy)`` will always return ``False`` if the var is not set. If
+you need access to the object directly instead of the proxy, you can get
+it with the :meth:`~LocalProxy._get_current_object` method.
 
-    def get_request():
-        return getattr(local, 'request', None)
+.. autoclass:: LocalProxy
+    :members: _get_current_object
 
-This will try to get the request or return `None` if the request is not
-(yet?) available.
 
-Note that local objects cannot manage themselves, for that you need a local
-manager.  You can pass a local manager multiple locals or add additionals
-later by appending them to `manager.locals` and every time the manager
-cleans up it will clean up all the data left in the locals for this
-context.
+Stacks and Namespaces
+=====================
 
-.. autofunction:: release_local
+:class:`~contextvars.ContextVar` stores one value at a time. You may
+find that you need to store a stack of items, or a namespace with
+multiple attributes. A list or dict can be used for these, but using
+them as context var values requires some extra care. Werkzeug provides
+:class:`LocalStack` which wraps a list, and :class:`Local` which wraps a
+dict.
 
-.. autoclass:: LocalManager
-   :members: cleanup, make_middleware, middleware
+There is some amount of performance penalty associated with these
+objects. Because lists and dicts are mutable, :class:`LocalStack` and
+:class:`Local` need to do extra work to ensure data isn't shared between
+nested contexts. If possible, design your application to use
+:class:`LocalProxy` around a context var directly.
 
 .. autoclass:: LocalStack
-   :members: push, pop, top
+    :members: push, pop, top, __call__
 
-.. autoclass:: LocalProxy
-    :members: _get_current_object
+.. autoclass:: Local
+    :members: __call__
+
+
+Releasing Data
+==============
+
+A previous implementation of ``Local`` used internal data structures
+which could not be cleaned up automatically when each context ended.
+Instead, the following utilities could be used to release the data.
+
+.. warning::
+
+    This should not be needed with the modern implementation, as the
+    data in context vars is automatically managed by Python. It is kept
+    for compatibility for now, but may be removed in the future.
+
+.. autoclass:: LocalManager
+   :members: cleanup, make_middleware, middleware
+
+.. autofunction:: release_local

docs/routing.rst

@@ -105,6 +105,10 @@ converters can be overridden or extended through :attr:`Map.converters`.
 
 .. autoclass:: UUIDConverter
 
+If a custom converter can match a forward slash, ``/``, it should have
+the attribute ``part_isolating`` set to ``False``. This will ensure
+that rules using the custom converter are correctly matched.
+
 
 Maps, Rules and Adapters
 ========================
@@ -127,6 +131,13 @@ Maps, Rules and Adapters
    :members: empty
 
 
+Matchers
+========
+
+.. autoclass:: StateMachineMatcher
+   :members:
+
+
 Rule Factories
 ==============
 
@@ -261,3 +272,34 @@ scheme and host, ``force_external=True`` is implied.
 
     url = adapter.build("comm")
     assert url == "ws://example.org/ws"
+
+
+State Machine Matching
+======================
+
+The default matching algorithm uses a state machine that transitions
+between parts of the request path to find a match. To understand how
+this works consider this rule::
+
+    /resource/<id>
+
+Firstly this rule is decomposed into two ``RulePart``. The first is a
+static part with a content equal to ``resource``, the second is
+dynamic and requires a regex match to ``[^/]+``.
+
+A state machine is then created with an initial state that represents
+the rule's first ``/``. This initial state has a single, static
+transition to the next state which represents the rule's second
+``/``. This second state has a single dynamic transition to the final
+state which includes the rule.
+
+To match a path the matcher starts and the initial state and follows
+transitions that work. Clearly a trial path of ``/resource/2`` has the
+parts ``""``, ``resource``, and ``2`` which match the transitions and
+hence a rule will match. Whereas ``/other/2`` will not match as there
+is no transition for the ``other`` part from the initial state.
+
+The only diversion from this rule is if a ``RulePart`` is not
+part-isolating i.e. it will match ``/``. In this case the ``RulePart``
+is considered final and represents a transition that must include all
+the subsequent parts of the trial path.

examples/plnt/sync.py

@@ -1,8 +1,8 @@
 """Does the synchronization. Called by "manage-plnt.py sync"."""
 from datetime import datetime
-from html import escape
 
 import feedparser
+from markupsafe import escape
 
 from .database import Blog
 from .database import Entry

requirements/dev.txt

@@ -8,33 +8,35 @@
 -r docs.txt
 -r tests.txt
 -r typing.txt
+build==0.8.0
+    # via pip-tools
 cfgv==3.3.1
     # via pre-commit
-click==8.1.2
+click==8.1.3
     # via
     #   pip-compile-multi
     #   pip-tools
 distlib==0.3.4
     # via virtualenv
-filelock==3.6.0
+filelock==3.7.1
     # via
     #   tox
     #   virtualenv
 greenlet==1.1.2 ; python_version < "3.11"
     # via -r requirements/tests.in
-identify==2.5.0
+identify==2.5.1
     # via pre-commit
-nodeenv==1.6.0
+nodeenv==1.7.0
     # via pre-commit
 pep517==0.12.0
-    # via pip-tools
+    # via build
 pip-compile-multi==2.4.5
     # via -r requirements/dev.in
-pip-tools==6.6.0
+pip-tools==6.8.0
     # via pip-compile-multi
 platformdirs==2.5.2
     # via virtualenv
-pre-commit==2.18.1
+pre-commit==2.20.0
     # via -r requirements/dev.in
 pyyaml==6.0
     # via pre-commit
@@ -48,9 +50,9 @@ toml==0.10.2
     #   tox
 toposort==1.7
     # via pip-compile-multi
-tox==3.25.0
+tox==3.25.1
     # via -r requirements/dev.in
-virtualenv==20.14.1
+virtualenv==20.15.1
     # via
     #   pre-commit
     #   tox