util.py: include non prod versions in constraints.txt (#1234)

requirements file still contains only prod versions (from
versions-prod.cfg) but constraints is extended to also contain versions
from versions.cfg, so that during tests we can install extras with pip,
like we do for universal2 on OSX.

This fixes a problem of test failing on OSX because the version pin for
MarkupSafe was missing and when installing it with pip, the latest
version was installed, which might not be the version defined in
versions.cfg, causing buildout to install again MarkupSafe at the version
defined in versions.cfg and failing.

Co-authored-by: Jens Vagelpohl <jens@plyp.com>

CHANGES.rst

@@ -14,14 +14,18 @@ https://github.com/zopefoundation/Zope/blob/4.x/CHANGES.rst
 5.11.1 (2024-11-03)
 -------------------
 
+- Include versions constraints for production and non-production dependencies
+  in ``constraints.txt``.
+  (`#1234 <https://github.com/zopefoundation/Zope/pull/1234>`_)
+
 - Update to newest compatible versions of dependencies.
 
 - Define ``request["PARENTS"]`` before request parameter processing
   such that request failure subscribers (such as that of
   ``Products.SiteErrorLog``) can determine the error context
   for exceptions raised during request parameter processing
   (see `#1235 <https://github.com/zopefoundation/Zope/issues/1235>`_).
-   
+
 
 5.11 (2024-10-11)
 -----------------

constraints.txt

@@ -2,39 +2,91 @@ AccessControl==7.2
 Acquisition==6.1
 AuthEncoding==5.0
 BTrees==6.1
+Babel==2.16.0
 Chameleon==4.4.4; python_version == '3.8'
 Chameleon==4.4.4; python_version > '3.8'
 DateTime==5.5
 DocumentTemplate==4.6
 ExtensionClass==6.0
+Jinja2==3.1.4
+MarkupSafe==2.1.5; python_version == '3.8'
+MarkupSafe==3.0.2; python_version > '3.8'
+Missing==5.0
 MultiMapping==5.0
 Paste==3.10.1
 PasteDeploy==3.1.0
 Persistence==5.1
+Products.BTreeFolder2==5.1
+Products.ZCatalog==7.1
+Pygments==2.18.0
+Record==4.1
 RestrictedPython==7.4
+Sphinx==7.1.2; python_version == '3.8'
+Sphinx==7.4.7; python_version == '3.9'
+Sphinx==8.1.3; python_version > '3.9'
 WSGIProxy2==0.5.1
 WebOb==1.8.9
 WebTest==3.0.1
 ZConfig==4.1
 ZODB==6.0
 Zope2==4.0
+alabaster==0.7.13; python_version == '3.8'
+alabaster==0.7.14; python_version == '3.9'
+alabaster==1.0.0; python_version > '3.9'
 beautifulsoup4==4.12.3
+certifi==2024.8.30
 cffi==1.17.1
+charset-normalizer==3.4.0
+collective.recipe.template==2.2
+colorama==0.4.6
+docutils==0.20.1; python_version == '3.8'
+docutils==0.20.1; python_version == '3.9'
+docutils==0.21.2; python_version > '3.9'
+five.localsitemanager==4.0
+furo==2024.8.6
+idna==3.10
+imagesize==1.4.1
+importlib-metadata==8.5.0
+importlib-resources==6.4.0; python_version == '3.8'
+legacy-cgi==2.6.1
+mr.developer==2.0.2
 multipart==0.2.5
+packaging==24.1
 persistent==6.1
+plone.recipe.command==1.1
 pycparser==2.22
 python-gettext==5.0
 pytz==2024.2
+requests==2.32.3
 roman==4.2
 six==1.16.0
+snowballstemmer==2.2.0
 soupsieve==2.6
+sphinx-basic-ng==1.0.0b2
+sphinxcontrib-applehelp==1.0.4; python_version == '3.8'
+sphinxcontrib-applehelp==2.0.0; python_version > '3.8'
+sphinxcontrib-devhelp==1.0.2; python_version == '3.8'
+sphinxcontrib-devhelp==2.0.0; python_version > '3.8'
+sphinxcontrib-htmlhelp==2.0.1; python_version == '3.8'
+sphinxcontrib-htmlhelp==2.1.0; python_version > '3.8'
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3; python_version == '3.8'
+sphinxcontrib-qthelp==2.0.0; python_version > '3.8'
+sphinxcontrib-serializinghtml==1.1.5; python_version == '3.8'
+sphinxcontrib-serializinghtml==2.0.0; python_version > '3.8'
+tempstorage==6.0
+tomli==2.0.2
 transaction==5.0
+urllib3==2.2.3
 waitress==3.0.0; python_version == '3.8'
 waitress==3.0.1; python_version > '3.8'
+z3c.checkversions==2.1
 z3c.pt==4.4
 zExceptions==5.0
 zc.lockfile==3.0.post1
 zc.recipe.egg==2.0.7
+zc.recipe.testrunner==3.1
+zipp==3.20.2
 zodbpickle==4.1.1
 zope.annotation==5.0
 zope.browser==3.0
@@ -76,5 +128,6 @@ zope.tal==5.0.1
 zope.tales==6.0
 zope.testbrowser==7.0
 zope.testing==5.0.1
+zope.testrunner==6.6
 zope.traversing==5.0
 zope.viewlet==5.0

util.py

@@ -16,51 +16,48 @@ def optionxform(self, value):
         return value
 
 
-def generate(in_, requirements_file, constraints_file):
-    in_file = os.path.join(HERE, in_)
+def generate(
+        buildout_file: str,
+        requirements_file: str,
+        zope_requirement: str):
     out_file_requirements = os.path.join(HERE, requirements_file)
-    out_file_constraints = os.path.join(HERE, constraints_file)
     parser = CaseSensitiveParser()
-    parser.read(in_file)
+    parser.read(os.path.join(HERE, buildout_file))
+    for extend in parser.get('buildout', 'extends', fallback='').splitlines():
+        extend = extend.strip()
+        if extend:
+            parser.read(os.path.join(HERE, extend))
 
     requirements = []
-    constraints = []
-    zope_requirement = (
-        '-e git+https://github.com/zopefoundation/Zope.git@master#egg=Zope\n')
 
     # Try to include sections for all currently supported Python versions
-    for py_version in ('3.8', '3.9', '3.10', '3.11', '3.12'):
+    for py_version in ('3.8', '3.9', '3.10', '3.11', '3.12', '3.13'):
         short_version = py_version.replace('.', '')
         try:
             zope_requirement = _generate(
                 parser.items(f'versions:python{short_version}'), py_version,
-                requirements, constraints, zope_requirement)
+                requirements, zope_requirement)
         except NoSectionError:
             continue
 
     # "Unversioned" pins must come last, how they are handled depends on
     # Python version qualifiers for dependencies of the same name.
     zope_requirement = _generate(
-        parser.items('versions'), None, requirements, constraints,
-        zope_requirement)
+        parser.items('versions'), None, requirements, zope_requirement)
 
     with open(out_file_requirements, 'w') as fd:
         fd.write(zope_requirement)
         for req in sorted(requirements):
             fd.write(req)
-    with open(out_file_constraints, 'w') as fcon:
-        for con in sorted(constraints):
-            fcon.write(con)
 
 
 def _generate(
     versions: List[Tuple[str, str]],
     python_version: Optional[str],
     requirements: List[str],
-    constraints: List[str],
     zope_requirement: str
 ) -> str:
-    """Generate requirements and constraints for a specific Python version.
+    """Generate requirements or constraints file for a specific Python version.
 
     If ``python_version`` is false, generate for all python versions.
     Returns a probably changed ``zope_requirement``.
@@ -90,12 +87,16 @@ def _generate(
                 spec = f"{spec}; python_version > '{versions[-1]}'"
 
         requirements.append(spec + '\n')
-        constraints.append(spec + '\n')
     return zope_requirement
 
 
 def main():
-    generate('versions-prod.cfg', 'requirements-full.txt', 'constraints.txt')
+    generate(
+        'versions-prod.cfg',
+        'requirements-full.txt',
+        '-e git+https://github.com/zopefoundation/Zope.git@master#egg=Zope\n',
+    )
+    generate('versions.cfg', 'constraints.txt', '')
 
 
 if __name__ == '__main__':