Merge pull request #20 from plone/maurits-env-allow-module-not-found

Raise an exception when a module cannot be imported.

README.rst

@@ -105,6 +105,40 @@ You must specify at least one option, otherwise the entry point does not exist.
 
 ``module``
     Use this when your package name is different from what you import in Python.
+    See also the next section.
+
+
+Different project and module name
+---------------------------------
+
+Usually the project name of an add-on (what is in ``setup.py`` or ``setup.cfg``) is the same as how you would import it in Python code.
+It could be different though.
+In that case, you may get a ``ModuleNotFoundError`` on startup: ``plone.autoinclude`` tries to import the project name and this fails.
+
+The easiest way to solve this, is to switch from ``z3c.autoinclude.plugin`` to ``plone.autoinclude.plugin``, if you have not done so already,
+and specify the module.
+In ``setup.py``::
+
+    setup(
+        name="example.different2",
+        entry_points="""
+        [plone.autoinclude.plugin]
+        module = example.somethingelse2
+        """,
+    )
+
+If you must still support Plone 5.2 and are tied to ``z3c.autoinclude.plugin``, or if you cannot edit the problematic package, you can work around it.
+You set an environment variable ``AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR``.
+To accept ``ModuleNotFoundError`` in all packages::
+
+    export AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR=1
+
+To accept ``ModuleNotFoundError`` only in specific packages, use a comma-separated list of project names, with or without spaces::
+
+    export AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR=example.different,example.different2
+
+In the logs you will see a traceback so you can investigate, but startup continues.
+You should make sure the zcml of this package is loaded in some other way.
 
 
 Comparison with ``z3c.autoinclude``

news/19.feature

@@ -0,0 +1,5 @@
+Raise an exception when a module is not found.
+When environment variable ``AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR=1`` is set, we log an error and continue.
+To accept ``ModuleNotFoundError`` only in specific packages, use a comma-separated list of project names, with or without spaces.
+See `issue 19 <https://github.com/plone/plone.autoinclude/issues/19>`_.
+[maurits]

src/plone/autoinclude/loader.py

@@ -15,6 +15,21 @@
 # Dictionary of project names and packages that we have already imported.
 _known_module_names = {}
 
+# Maybe allow ModuleNotFoundError.
+# This can be a boolean (0/1) or a list of project names (from setup.py),
+# separated by comma.
+AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR = os.getenv(
+    "AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR", ""
+)
+ALLOW_MODULE_NOT_FOUND_SET = set()
+ALLOW_MODULE_NOT_FOUND_ALL = False
+try:
+    ALLOW_MODULE_NOT_FOUND_ALL = bool(int(AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR))
+except (ValueError, TypeError):
+    _allowed = AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR.replace(" ", "").split(",")
+    if _allowed:
+        ALLOW_MODULE_NOT_FOUND_SET = set(_allowed)
+
 
 def load_z3c_packages(target=""):
     """Load packages from the z3c.autoinclude.plugin entry points.
@@ -37,7 +52,25 @@ def load_z3c_packages(target=""):
             except ModuleNotFoundError:
                 # Note: this may happen a lot, at least for z3c.autoinclude,
                 # because the project name may not be the same as the package/module.
-                logger.exception(f"Could not import {module_name}.")
+                # If we accept it, we may hide real errors though:
+                # the module may be there but have an ImportError.
+                if (
+                    not ALLOW_MODULE_NOT_FOUND_ALL
+                    and module_name not in ALLOW_MODULE_NOT_FOUND_SET
+                ):
+                    logger.error(
+                        f"Could not import {module_name}. Set environment variable "
+                        "AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR=1 if you want to "
+                        f"allow this. Or set it to '{module_name}' to only allow for "
+                        "this project. Can be a comma-separated list of project "
+                        "names. Or replace the z3c.autoinclude.plugin entry point of "
+                        "this project with plone.autoinclude.plugin and a module name."
+                    )
+                    raise
+                logger.exception(
+                    f"Could not import {module_name}. Accepted due to "
+                    "AUTOINCLUDE_ALLOW_MODULE_NOT_FOUND_ERROR environment variable."
+                )
                 _known_module_names[module_name] = None
                 continue
             _known_module_names[module_name] = dist

src/plone/autoinclude/tests/test_loader.py

@@ -23,6 +23,12 @@
 
 class TestLoader(unittest.TestCase):
     def setUp(self):
+        from plone.autoinclude import loader
+
+        # Allow module not found errors in these tests.
+        self._orig_ALLOW_MODULE_NOT_FOUND_ALL = loader.ALLOW_MODULE_NOT_FOUND_ALL
+        loader.ALLOW_MODULE_NOT_FOUND_ALL = True
+
         workingset = pkg_resources.working_set
         self.workingdir = os.getcwd()
         self.stored_syspath = copy(sys.path)
@@ -46,6 +52,11 @@ def setUp(self):
         os.chdir(self.workingdir)
 
     def tearDown(self):
+        from plone.autoinclude import loader
+
+        # Restore original setting.
+        loader.ALLOW_MODULE_NOT_FOUND_ALL = self._orig_ALLOW_MODULE_NOT_FOUND_ALL
+
         os.chdir(self.workingdir)
         sys.path = self.stored_syspath
         workingset = pkg_resources.working_set

test-packages/example.basetestpackage/src/example/basetestpackage/package_base.py

@@ -1,3 +1,4 @@
+from .utils import allow_module_not_found_error
 from .utils import get_configuration_context
 from importlib import import_module
 
@@ -36,6 +37,10 @@ class PackageTestCase:
     project_name = ""
     # If module name differs from project name, fill this in:
     module_name = ""
+    # Accept ModuleNotFound errors for some projects with different module names.
+    # This should be same list for all test packages, and should contain all
+    # test packages with the old z3c.autoinclude.plugin that have this problem.
+    allow_module_not_found = {"example.different"}
     # Does the package use plone.autoinclude (True) or the old z3c.autoinclude (False)?
     # Attribute is only used when we have a different module_name.
     uses_plone_autoinclude = True
@@ -61,10 +66,23 @@ def import_me(self):
     @unittest.skipIf(not HAS_PLONE_AUTOINCLUDE, "plone.autoinclude missing")
     def test_load_packages(self):
         from plone.autoinclude.loader import load_packages
+        from plone.autoinclude import loader
 
-        packages = load_packages()
-        if self.module_name:
+        # Empty the known module names, so projects are loaded again.
+        loader._known_module_names = {}
+        if self.module_name and not self.uses_plone_autoinclude:
             # Module name differs from project name.
+            # Allowing ModuleNotFound in all known ones except our own,
+            # should fail, so the user knows something is wrong.
+            allowed = self.allow_module_not_found - {self.project_name}
+            with allow_module_not_found_error(allowed):
+                with self.assertRaises(ModuleNotFoundError):
+                    packages = load_packages()
+
+        # User can allow some modules to have ModuleNotFoundErrors.
+        with allow_module_not_found_error(self.allow_module_not_found):
+            packages = load_packages()
+        if self.module_name:
             # Only module names get in the packages list.
             self.assertNotIn(self.project_name, packages.keys())
             if not self.uses_plone_autoinclude:

test-packages/example.basetestpackage/src/example/basetestpackage/utils.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 from zope.configuration.config import ConfigurationMachine
 from zope.configuration.xmlconfig import registerCommonDirectives
 
@@ -17,3 +18,23 @@ def get_configuration_context(package=None):
         # When you set context.package, context.path(filename) works nicely.
         context.package = package
     return context
+
+
+@contextmanager
+def allow_module_not_found_error(allowed):
+    from plone.autoinclude import loader
+
+    # save original settings
+    orig_all = loader.ALLOW_MODULE_NOT_FOUND_ALL
+    orig_set = loader.ALLOW_MODULE_NOT_FOUND_SET
+    # Temporarily allow module not found error for only the
+    # packages in the allowed set.
+    loader.ALLOW_MODULE_NOT_FOUND_ALL = False
+    loader.ALLOW_MODULE_NOT_FOUND_SET = allowed
+    # breakpoint()
+    try:
+        yield
+    finally:
+        # restore
+        loader.ALLOW_MODULE_NOT_FOUND_ALL = orig_all
+        loader.ALLOW_MODULE_NOT_FOUND_SET = orig_set