From 1bf2cc512f033ff99ef9f03da9d4ae16c8ebbf2e Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 27 Oct 2020 09:52:05 +0100 Subject: [PATCH 1/8] add Python 3.10 beta/RC builds to the matrix --- .ci/.jenkins_exclude.yml | 66 +++++++++++++++++++++++++++------------- .ci/.jenkins_python.yml | 3 +- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.ci/.jenkins_exclude.yml b/.ci/.jenkins_exclude.yml index d231158ab..3192077ed 100644 --- a/.ci/.jenkins_exclude.yml +++ b/.ci/.jenkins_exclude.yml @@ -51,11 +51,17 @@ exclude: FRAMEWORK: celery-4-django-1.11 - PYTHON_VERSION: python-3.8 FRAMEWORK: celery-4-flask-1.0 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 FRAMEWORK: celery-4-django-2.0 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 FRAMEWORK: celery-4-django-1.11 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: celery-4-flask-1.0 + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: celery-4-django-2.0 + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: celery-4-django-1.11 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: celery-4-flask-1.0 # requests - PYTHON_VERSION: python-3.5 @@ -66,7 +72,9 @@ exclude: FRAMEWORK: requests-1.2 - PYTHON_VERSION: python-3.8 FRAMEWORK: requests-1.2 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: requests-1.2 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: requests-1.2 - PYTHON_VERSION: pypy-3 FRAMEWORK: requests-1.2 @@ -79,7 +87,9 @@ exclude: FRAMEWORK: pymongo-3.0 - PYTHON_VERSION: python-3.8 FRAMEWORK: pymongo-3.0 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: pymongo-3.0 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: pymongo-3.0 - PYTHON_VERSION: python-3.6 FRAMEWORK: pymongo-3.1 @@ -87,7 +97,9 @@ exclude: FRAMEWORK: pymongo-3.1 - PYTHON_VERSION: python-3.8 FRAMEWORK: pymongo-3.1 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: pymongo-3.1 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: pymongo-3.1 - PYTHON_VERSION: python-3.6 FRAMEWORK: pymongo-3.2 @@ -95,7 +107,9 @@ exclude: FRAMEWORK: pymongo-3.2 - PYTHON_VERSION: python-3.8 FRAMEWORK: pymongo-3.2 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: pymongo-3.2 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: pymongo-3.2 - PYTHON_VERSION: python-3.6 FRAMEWORK: pymongo-3.3 @@ -103,7 +117,9 @@ exclude: FRAMEWORK: pymongo-3.3 - PYTHON_VERSION: python-3.8 FRAMEWORK: pymongo-3.3 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: pymongo-3.3 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: pymongo-3.3 - PYTHON_VERSION: python-3.6 FRAMEWORK: pymongo-3.4 @@ -111,7 +127,9 @@ exclude: FRAMEWORK: pymongo-3.4 - PYTHON_VERSION: python-3.8 FRAMEWORK: pymongo-3.4 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: pymongo-3.4 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: pymongo-3.4 - PYTHON_VERSION: pypy-3 FRAMEWORK: pymongo-3.0 @@ -124,7 +142,9 @@ exclude: FRAMEWORK: memcached-1.51 - PYTHON_VERSION: python-3.8 FRAMEWORK: memcached-1.51 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: memcached-1.51 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: memcached-1.51 - PYTHON_VERSION: pypy-3 FRAMEWORK: memcached-1.51 @@ -139,12 +159,16 @@ exclude: FRAMEWORK: pymssql-newest - PYTHON_VERSION: python-3.8 # currently fails with error on python 3.8 due to cython issues FRAMEWORK: pymssql-newest - - PYTHON_VERSION: python-3.9-rc # currently fails with error on python 3.8 due to cython issues + - PYTHON_VERSION: python-3.9 # currently fails with error on python 3.8 due to cython issues + FRAMEWORK: pymssql-newest + - PYTHON_VERSION: python-3.10-rc # currently fails with error on python 3.8 due to cython issues FRAMEWORK: pymssql-newest # psycopg2 - PYTHON_VERSION: python-3.8 # see https://github.com/psycopg/psycopg2/issues/858 FRAMEWORK: psycopg2-2.7 - - PYTHON_VERSION: python-3.9-rc # see https://github.com/psycopg/psycopg2/issues/858 + - PYTHON_VERSION: python-3.9 # see https://github.com/psycopg/psycopg2/issues/858 + FRAMEWORK: psycopg2-2.7 + - PYTHON_VERSION: python-3.10-rc # see https://github.com/psycopg/psycopg2/issues/858 FRAMEWORK: psycopg2-2.7 # pyodbc - PYTHON_VERSION: pypy-2 @@ -158,19 +182,17 @@ exclude: FRAMEWORK: boto3-1.0 - PYTHON_VERSION: python-3.8 FRAMEWORK: boto3-1.0 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: boto3-1.0 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: boto3-1.0 - PYTHON_VERSION: pypy-3 FRAMEWORK: boto3-1.0 # boto3 is currently not importable on Python 3.9, see https://github.com/boto/botocore/issues/2002 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: boto3-1.0 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: boto3-1.0 - - PYTHON_VERSION: python-3.9-rc - FRAMEWORK: boto3-1.5 - - PYTHON_VERSION: python-3.9-rc - FRAMEWORK: boto3-1.6 - - PYTHON_VERSION: python-3.9-rc - FRAMEWORK: boto3-newest # opentracing - PYTHON_VERSION: python-3.3 FRAMEWORK: opentracing-newest @@ -188,7 +210,9 @@ exclude: FRAMEWORK: zerorpc-0.4 - PYTHON_VERSION: python-3.8 FRAMEWORK: zerorpc-0.4 - - PYTHON_VERSION: python-3.9-rc + - PYTHON_VERSION: python-3.9 + FRAMEWORK: zerorpc-0.4 + - PYTHON_VERSION: python-3.10-rc FRAMEWORK: zerorpc-0.4 # gevent - PYTHON_VERSION: pypy-2 # see https://github.com/gevent/gevent/issues/1380, fix will be in gevent 1.5 diff --git a/.ci/.jenkins_python.yml b/.ci/.jenkins_python.yml index 919810465..09db0de23 100644 --- a/.ci/.jenkins_python.yml +++ b/.ci/.jenkins_python.yml @@ -4,6 +4,7 @@ PYTHON_VERSION: - python-3.6 - python-3.7 - python-3.8 - - python-3.9-rc + - python-3.9 + - python-3.10-rc - pypy-2 - pypy-3 From 052796d7b2a2df17a3a222eddb8388ca0d8ba814 Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Wed, 6 Jan 2021 13:47:54 +0100 Subject: [PATCH 2/8] exclude gevent/python3.10 a weird exception is raised during test setup, in the vendored certifi from pip: Traceback (most recent call last): File "/usr/local/lib/python3.10/runpy.py", line 197, in _run_module_as_main return _run_code(code, main_globals, None, File "/usr/local/lib/python3.10/runpy.py", line 87, in _run_code exec(code, run_globals) File "/usr/local/lib/python3.10/site-packages/pip/__main__.py", line 26, in sys.exit(_main()) File "/usr/local/lib/python3.10/site-packages/pip/_internal/cli/main.py", line 73, in main command = create_command(cmd_name, isolated=("--isolated" in cmd_args)) File "/usr/local/lib/python3.10/site-packages/pip/_internal/commands/__init__.py", line 105, in create_command module = importlib.import_module(module_path) File "/usr/local/lib/python3.10/importlib/__init__.py", line 126, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "", line 1049, in _gcd_import File "", line 1026, in _find_and_load File "", line 1005, in _find_and_load_unlocked File "", line 698, in _load_unlocked File "", line 833, in exec_module File "", line 241, in _call_with_frames_removed File "/usr/local/lib/python3.10/site-packages/pip/_internal/commands/install.py", line 17, in from pip._internal.cli.req_command import RequirementCommand, with_cleanup File "/usr/local/lib/python3.10/site-packages/pip/_internal/cli/req_command.py", line 18, in from pip._internal.index.collector import LinkCollector File "/usr/local/lib/python3.10/site-packages/pip/_internal/index/collector.py", line 14, in from pip._vendor import html5lib, requests File "/usr/local/lib/python3.10/site-packages/pip/_vendor/requests/__init__.py", line 125, in from . import utils File "/usr/local/lib/python3.10/site-packages/pip/_vendor/requests/utils.py", line 40, in DEFAULT_CA_BUNDLE_PATH = certs.where() File "/usr/local/lib/python3.10/site-packages/pip/_vendor/certifi/core.py", line 37, in where _CACERT_PATH = str(_CACERT_CTX.__enter__()) AttributeError: 'PosixPath' object has no attribute '__enter__' --- .ci/.jenkins_exclude.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/.jenkins_exclude.yml b/.ci/.jenkins_exclude.yml index 3192077ed..57062397b 100644 --- a/.ci/.jenkins_exclude.yml +++ b/.ci/.jenkins_exclude.yml @@ -217,6 +217,8 @@ exclude: # gevent - PYTHON_VERSION: pypy-2 # see https://github.com/gevent/gevent/issues/1380, fix will be in gevent 1.5 FRAMEWORK: gevent-newest + - PYTHON_VERSION: python-3.10-rc # causes issues with PosixPath.__enter__, unsure why + FRAMEWORK: gevent-newest # aiohttp client, only supported in Python 3.7+ - PYTHON_VERSION: python-2.7 FRAMEWORK: aiohttp-3.0 From a054968d3e549c61299efb108717ecb7ca7c51cf Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 16 Mar 2021 12:12:59 +0100 Subject: [PATCH 3/8] delete any __pycache__ folders to avoid bytecode cache issues This is especially problematic when mounting the working directory in docker for testing, as the local Python version may differ from the one in docker. --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 39b1b2685..43017970b 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ flake8: flake8 test: + # delete any __pycache__ folders to avoid hard-to-debug caching issues + find . -name __pycache__ -type d -exec rm -r {} + # pypy3 should be added to the first `if` once it supports py3.7 if [[ "$$PYTHON_VERSION" =~ ^(3.7|3.8|3.9|nightly)$$ ]] ; then \ py.test -v $(PYTEST_ARGS) $(PYTEST_MARKER) $(PYTEST_JUNIT); \ From 20321b60e23998d7ec30afe810a6e9ff9e87855f Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 16 Mar 2021 12:18:13 +0100 Subject: [PATCH 4/8] remove pathlib dependency, it's part of the standard library since Python 3.4 --- tests/requirements/reqs-base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/requirements/reqs-base.txt b/tests/requirements/reqs-base.txt index 3d7f5fd16..444711cfb 100644 --- a/tests/requirements/reqs-base.txt +++ b/tests/requirements/reqs-base.txt @@ -26,7 +26,6 @@ parse-type==0.5.2 toml==0.10.2 iniconfig==1.1.1 docutils==0.16 -pathlib==1.0.1 py-cpuinfo==7.0.0 statistics==1.0.3.5 From aa32bcff539b8996d0007af2777cf02845aa1b40 Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 16 Mar 2021 12:29:11 +0100 Subject: [PATCH 5/8] Makefile requires tabs... --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43017970b..bf142bba6 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ flake8: flake8 test: - # delete any __pycache__ folders to avoid hard-to-debug caching issues - find . -name __pycache__ -type d -exec rm -r {} + + # delete any __pycache__ folders to avoid hard-to-debug caching issues + find . -name __pycache__ -type d -exec rm -r {} + # pypy3 should be added to the first `if` once it supports py3.7 if [[ "$$PYTHON_VERSION" =~ ^(3.7|3.8|3.9|nightly)$$ ]] ; then \ py.test -v $(PYTEST_ARGS) $(PYTEST_MARKER) $(PYTEST_JUNIT); \ From f17de21e604c577be53632e3f04dd1ad335e8934 Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 16 Mar 2021 13:45:41 +0100 Subject: [PATCH 6/8] added excludes for Django < 3.0 and Python 3.10 also, removed a bunch of outdated excludes --- .ci/.jenkins_exclude.yml | 29 +++++------------------------ Makefile | 2 +- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/.ci/.jenkins_exclude.yml b/.ci/.jenkins_exclude.yml index aecefb1fc..6a1e66210 100644 --- a/.ci/.jenkins_exclude.yml +++ b/.ci/.jenkins_exclude.yml @@ -2,34 +2,15 @@ exclude: # Flask - PYTHON_VERSION: pypy-3 FRAMEWORK: flask-0.11 # see https://github.com/pallets/flask/commit/6e46d0cd, 0.11.2 was never released - # celery - - PYTHON_VERSION: python-3.7 - FRAMEWORK: celery-3-django-1.11 # see https://code.djangoproject.com/ticket/28814 - # celery 4 doesn't currently work on Python 3.7, see https://github.com/celery/celery/issues/4500 - - PYTHON_VERSION: python-3.7 - FRAMEWORK: celery-4-django-2.0 - - PYTHON_VERSION: python-3.7 - FRAMEWORK: celery-4-django-1.11 - - PYTHON_VERSION: python-3.7 - FRAMEWORK: celery-4-flask-1.0 - - PYTHON_VERSION: python-3.8 - FRAMEWORK: celery-4-django-2.0 - - PYTHON_VERSION: python-3.8 - FRAMEWORK: celery-4-django-1.11 - - PYTHON_VERSION: python-3.8 - FRAMEWORK: celery-4-flask-1.0 - - PYTHON_VERSION: python-3.9 - FRAMEWORK: celery-4-django-2.0 - - PYTHON_VERSION: python-3.9 - FRAMEWORK: celery-4-django-1.11 - - PYTHON_VERSION: python-3.9 - FRAMEWORK: celery-4-flask-1.0 + # Python 3.10 removed a bunch of classes from collections, now in collections.abc + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: django-1.11 + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: django-2.0 - PYTHON_VERSION: python-3.10-rc FRAMEWORK: celery-4-django-2.0 - PYTHON_VERSION: python-3.10-rc FRAMEWORK: celery-4-django-1.11 - - PYTHON_VERSION: python-3.10-rc - FRAMEWORK: celery-4-flask-1.0 # requests - PYTHON_VERSION: python-3.6 FRAMEWORK: requests-1.2 diff --git a/Makefile b/Makefile index bf142bba6..db6d65f4b 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ test: # delete any __pycache__ folders to avoid hard-to-debug caching issues find . -name __pycache__ -type d -exec rm -r {} + # pypy3 should be added to the first `if` once it supports py3.7 - if [[ "$$PYTHON_VERSION" =~ ^(3.7|3.8|3.9|nightly)$$ ]] ; then \ + if [[ "$$PYTHON_VERSION" =~ ^(3.7|3.8|3.9|3.10|nightly)$$ ]] ; then \ py.test -v $(PYTEST_ARGS) $(PYTEST_MARKER) $(PYTEST_JUNIT); \ elif [[ "$$PYTHON_VERSION" =~ ^(3.5|3.6|pypy3)$$ ]] ; then \ py.test -v $(PYTEST_ARGS) $(PYTEST_MARKER) $(PYTEST_JUNIT) --ignore-glob='*/asyncio*/*'; \ From 0cf669328c633a177ad4a3f846b62a903d3e8743 Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 16 Mar 2021 14:14:31 +0100 Subject: [PATCH 7/8] more collections.abc-related excludes --- .ci/.jenkins_exclude.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.ci/.jenkins_exclude.yml b/.ci/.jenkins_exclude.yml index 6a1e66210..d417fc9dd 100644 --- a/.ci/.jenkins_exclude.yml +++ b/.ci/.jenkins_exclude.yml @@ -11,6 +11,12 @@ exclude: FRAMEWORK: celery-4-django-2.0 - PYTHON_VERSION: python-3.10-rc FRAMEWORK: celery-4-django-1.11 + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: celery-3-flask-1.0 + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: celery-3-django-2.0 + - PYTHON_VERSION: python-3.10-rc + FRAMEWORK: graphene-2 # requests - PYTHON_VERSION: python-3.6 FRAMEWORK: requests-1.2 From 3e033123cd08cc691d94bc862b8d61a5857fb049 Mon Sep 17 00:00:00 2001 From: Benjamin Wohlwend Date: Tue, 16 Mar 2021 14:44:54 +0100 Subject: [PATCH 8/8] asyncpg currently doesn't build on 3.10 --- .ci/.jenkins_exclude.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/.jenkins_exclude.yml b/.ci/.jenkins_exclude.yml index d417fc9dd..1c214ba38 100644 --- a/.ci/.jenkins_exclude.yml +++ b/.ci/.jenkins_exclude.yml @@ -202,3 +202,5 @@ exclude: FRAMEWORK: asyncpg-newest - PYTHON_VERSION: python-3.6 FRAMEWORK: asyncpg-newest + - PYTHON_VERSION: python-3.10-rc # https://github.com/MagicStack/asyncpg/issues/699 + FRAMEWORK: asyncpg-newest