gh-135801: Improve filtering by module in warn_explicit() without module argument

* Try to match the module name pattern with module names constructed
  starting from different parent directories of the filename.
  E.g., for "/path/to/package/module" try to match with
  "path.to.package.module", "to.package.module", "package.module" and
  "module".
* Ignore trailing "/__init__.py".
* Ignore trailing ".py" on Windows.
* Keep matching with the full filename (without optional ".py" extension)
  for compatibility.
* Only ignore the case of the ".py" extension on Windows.

Author: serhiy-storchaka

Doc/library/warnings.rst

@@ -480,14 +480,27 @@ Available Functions
 .. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None)
 
    This is a low-level interface to the functionality of :func:`warn`, passing in
-   explicitly the message, category, filename and line number, and optionally the
-   module name and the registry (which should be the ``__warningregistry__``
-   dictionary of the module).  The module name defaults to the filename with
-   ``.py`` stripped; if no registry is passed, the warning is never suppressed.
+   explicitly the message, category, filename and line number, and optionally
+   other arguments.
    *message* must be a string and *category* a subclass of :exc:`Warning` or
    *message* may be a :exc:`Warning` instance, in which case *category* will be
    ignored.
 
+   *module*, if supplied, should be the module name.
+   If no module is passed, the module regular expression in
+   :ref:`warnings filter <warning-filter>` will be tested against the filename
+   with ``/__init__.py`` and ``.py`` (and ``.pyw`` on Windows) stripped and
+   against the module names constructed from the path components starting
+   from all parent directories.
+   For example, when filename is ``'/path/to/package/module.py'``, it will
+   be tested against ``'/path/to/package/module'``,
+   ``'path.to.package.module'``, ``'to.package.module'``
+   ``'package.module'`` and ``'module'``.
+
+   *registry*, if supplied, should be the ``__warningregistry__`` dictionary
+   of the module.
+   If no registry is passed, each warning is treated as the first occurrence.
+
    *module_globals*, if supplied, should be the global namespace in use by the code
    for which the warning is issued.  (This argument is used to support displaying
    source for modules found in zipfiles or other non-filesystem import
@@ -499,6 +512,11 @@ Available Functions
    .. versionchanged:: 3.6
       Add the *source* parameter.
 
+   .. versionchanged:: next
+      If no module is passed, test the filter regular expression against
+      module names created from the path, not only the path itself;
+      strip also ``.pyw`` (on Windows) and ``/__init__.py``.
+
 
 .. function:: showwarning(message, category, filename, lineno, file=None, line=None)
 

Doc/whatsnew/3.15.rst

@@ -601,6 +601,18 @@ unittest
   (Contributed by Garry Cairns in :gh:`134567`.)
 
 
+warnings
+--------
+
+* Improve filtering by module in :func:`warnings.warn_explicit` if no *module*
+  argument is passed.
+  It now tests the module regular expression in the warnings filter not only
+  against the filename with ``.py`` stripped, but also against module names
+  constructed starting from different parent directories of the filename.
+  Strip also ``.pyw`` (on Windows) and ``/__init__.py``.
+  (Contributed by Serhiy Storchaka in :gh:`135801`.)
+
+
 xml.parsers.expat
 -----------------
 

Lib/_py_warnings.py

@@ -520,20 +520,54 @@ def warn(message, category=None, stacklevel=1, source=None,
     )
 
 
+def _match_filename(pattern, filename, *, MS_WINDOWS=(sys.platform == 'win32')):
+    if not filename:
+        return pattern.match('<unknown>') is not None
+    if filename[0] == '<' and filename[-1] == '>':
+        return pattern.match(filename) is not None
+
+    if MS_WINDOWS:
+        if filename[-12:].lower() in (r'\__init__.py', '/__init__.py'):
+            if pattern.match(filename[:-3]): # without '.py'
+                return True
+            filename = filename[:-12]
+        elif filename[-3:].lower() == '.py':
+            filename = filename[:-3]
+        elif filename[-4:].lower() == '.pyw':
+            filename = filename[:-4]
+        if pattern.match(filename):
+            return True
+        filename = filename.replace('\\', '/')
+    else:
+        if filename.endswith('/__init__.py'):
+            if pattern.match(filename[:-3]): # without '.py'
+                return True
+            filename = filename[:-12]
+        elif filename.endswith('.py'):
+            filename = filename[:-3]
+        if pattern.match(filename):
+            return True
+    filename = filename.replace('/', '.')
+    i = 0
+    while True:
+        if pattern.match(filename, i):
+            return True
+        i = filename.find('.', i) + 1
+        if not i:
+            return False
+
+
 def warn_explicit(message, category, filename, lineno,
                   module=None, registry=None, module_globals=None,
                   source=None):
     lineno = int(lineno)
-    if module is None:
-        module = filename or "<unknown>"
-        if module[-3:].lower() == ".py":
-            module = module[:-3] # XXX What about leading pathname?
     if isinstance(message, Warning):
         text = str(message)
         category = message.__class__
     else:
         text = message
         message = category(message)
+    modules = None
     key = (text, category, lineno)
     with _wm._lock:
         if registry is None:
@@ -549,9 +583,11 @@ def warn_explicit(message, category, filename, lineno,
             action, msg, cat, mod, ln = item
             if ((msg is None or msg.match(text)) and
                 issubclass(category, cat) and
-                (mod is None or mod.match(module)) and
-                (ln == 0 or lineno == ln)):
-                break
+                (ln == 0 or lineno == ln) and
+                (mod is None or (_match_filename(mod, filename)
+                                 if module is None else
+                                 mod.match(module)))):
+                    break
         else:
             action = _wm.defaultaction
         # Early exit actions

Lib/test/test_ast/test_ast.py

@@ -13,6 +13,7 @@
 import textwrap
 import types
 import unittest
+import warnings
 import weakref
 from io import StringIO
 from pathlib import Path
@@ -1124,6 +1125,19 @@ def test_tstring(self):
         self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
         self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
 
+    def test_filter_syntax_warnings_by_module(self):
+        filename = support.findfile('test_import/data/syntax_warnings.py')
+        with open(filename, 'rb') as f:
+            source = f.read()
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=r'<unknown>\z')
+            ast.parse(source)
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 21])
+        for wm in wlog:
+            self.assertEqual(wm.filename, '<unknown>')
+            self.assertIs(wm.category, SyntaxWarning)
+
 
 class CopyTests(unittest.TestCase):
     """Test copying and pickling AST nodes."""

Lib/test/test_builtin.py

@@ -1088,6 +1088,28 @@ def four_freevars():
             three_freevars.__globals__,
             closure=my_closure)
 
+    def test_exec_filter_syntax_warnings_by_module(self):
+        filename = support.findfile('test_import/data/syntax_warnings.py')
+        with open(filename, 'rb') as f:
+            source = f.read()
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=r'<string>\z')
+            exec(source, {})
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21])
+        for wm in wlog:
+            self.assertEqual(wm.filename, '<string>')
+            self.assertIs(wm.category, SyntaxWarning)
+
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=r'<string>\z')
+            exec(source, {'__name__': 'package.module', '__file__': filename})
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21])
+        for wm in wlog:
+            self.assertEqual(wm.filename, '<string>')
+            self.assertIs(wm.category, SyntaxWarning)
+
 
     def test_filter(self):
         self.assertEqual(list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')), list('elloorld'))

Lib/test/test_cmd_line_script.py

@@ -810,6 +810,12 @@ def test_script_as_dev_fd(self):
                 out, err = p.communicate()
                 self.assertEqual(out, b"12345678912345678912345\n")
 
+    def test_filter_syntax_warnings_by_module(self):
+        filename = support.findfile('test_import/data/syntax_warnings.py')
+        rc, out, err = assert_python_ok('-Werror', '-Walways:::test.test_import.data.syntax_warnings', filename)
+        self.assertEqual(err.count(b': SyntaxWarning: '), 6)
+        rc, out, err = assert_python_ok('-Werror', '-Walways:::syntax_warnings', filename)
+        self.assertEqual(err.count(b': SyntaxWarning: '), 6)
 
 
 def tearDownModule():

Lib/test/test_compile.py

@@ -1745,6 +1745,20 @@ def test_compile_warning_in_finally(self):
             self.assertEqual(wm.category, SyntaxWarning)
             self.assertIn("\"is\" with 'int' literal", str(wm.message))
 
+    def test_filter_syntax_warnings_by_module(self):
+        filename = support.findfile('test_import/data/syntax_warnings.py')
+        with open(filename, 'rb') as f:
+            source = f.read()
+        module_re = r'test\.test_import\.data\.syntax_warnings\z'
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=module_re)
+            compile(source, filename, 'exec')
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21])
+        for wm in wlog:
+            self.assertEqual(wm.filename, filename)
+            self.assertIs(wm.category, SyntaxWarning)
+
 
 class TestBooleanExpression(unittest.TestCase):
     class Value:

Lib/test/test_import/__init__.py

@@ -15,6 +15,7 @@
 import os
 import py_compile
 import random
+import re
 import shutil
 import stat
 import subprocess
@@ -23,6 +24,7 @@
 import threading
 import time
 import types
+import warnings
 import unittest
 from unittest import mock
 import _imp
@@ -51,7 +53,7 @@
     TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE)
 from test.support import script_helper
 from test.support import threading_helper
-from test.test_importlib.util import uncache
+from test.test_importlib.util import uncache, temporary_pycache_prefix
 from types import ModuleType
 try:
     import _testsinglephase
@@ -412,7 +414,6 @@ def test_from_import_missing_attr_path_is_canonical(self):
         self.assertIsNotNone(cm.exception)
 
     def test_from_import_star_invalid_type(self):
-        import re
         with ready_to_import() as (name, path):
             with open(path, 'w', encoding='utf-8') as f:
                 f.write("__all__ = [b'invalid_type']")
@@ -1250,6 +1251,35 @@ class Spec2:
                 origin = "a\x00b"
             _imp.create_dynamic(Spec2())
 
+    def test_filter_syntax_warnings_by_module(self):
+        module_re = r'test\.test_import\.data\.syntax_warnings\z'
+        unload('test.test_import.data.syntax_warnings')
+        with (os_helper.temp_dir() as tmpdir,
+              temporary_pycache_prefix(tmpdir),
+              warnings.catch_warnings(record=True) as wlog):
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=module_re)
+            import test.test_import.data.syntax_warnings
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21])
+        filename = test.test_import.data.syntax_warnings.__file__
+        for wm in wlog:
+            self.assertEqual(wm.filename, filename)
+            self.assertIs(wm.category, SyntaxWarning)
+
+        module_re = r'syntax_warnings\z'
+        unload('test.test_import.data.syntax_warnings')
+        with (os_helper.temp_dir() as tmpdir,
+              temporary_pycache_prefix(tmpdir),
+              warnings.catch_warnings(record=True) as wlog):
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=module_re)
+            import test.test_import.data.syntax_warnings
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21])
+        filename = test.test_import.data.syntax_warnings.__file__
+        for wm in wlog:
+            self.assertEqual(wm.filename, filename)
+            self.assertIs(wm.category, SyntaxWarning)
+
 
 @skip_if_dont_write_bytecode
 class FilePermissionTests(unittest.TestCase):

Lib/test/test_import/data/syntax_warnings.py

@@ -0,0 +1,21 @@
+# Syntax warnings emitted in different parts of the Python compiler.
+
+# Parser/lexer/lexer.c
+x = 1or 0  # line 4
+
+# Parser/tokenizer/helpers.c
+'\z'  # line 7
+
+# Parser/string_parser.c
+'\400'  # line 10
+
+# _PyCompile_Warn() in Python/codegen.c
+assert(x, 'message')  # line 13
+x is 1  # line 14
+
+# _PyErr_EmitSyntaxWarning() in Python/ast_preprocess.c
+def f():
+    try:
+        pass
+    finally:
+        return 42  # line 21

Lib/test/test_symtable.py

@@ -5,6 +5,7 @@
 import re
 import textwrap
 import symtable
+import warnings
 import unittest
 
 from test import support
@@ -586,6 +587,20 @@ def test__symtable_refleak(self):
         # check error path when 'compile_type' AC conversion failed
         self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1)
 
+    def test_filter_syntax_warnings_by_module(self):
+        filename = support.findfile('test_import/data/syntax_warnings.py')
+        with open(filename, 'rb') as f:
+            source = f.read()
+        module_re = r'test\.test_import\.data\.syntax_warnings\z'
+        with warnings.catch_warnings(record=True) as wlog:
+            warnings.simplefilter('error')
+            warnings.filterwarnings('always', module=module_re)
+            symtable.symtable(source, filename, 'exec')
+        self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10])
+        for wm in wlog:
+            self.assertEqual(wm.filename, filename)
+            self.assertIs(wm.category, SyntaxWarning)
+
 
 class ComprehensionTests(unittest.TestCase):
     def get_identifiers_recursive(self, st, res):