fix(ci): pin importlib_metadata<5.0.0 (#4257)

CI jobs `profile-windows-*`, `kombu` and `celery` are failing due to the release of `importlib_metadata==5.0.0` which removed deprecated shims, resulting in the following exception: ``` AttributeError: 'EntryPoints' object has no attribute 'get' ``` We fix this by pinning the dependency for kombu and celery. In the case of the Windows tests, we address it by upgrading the Windows image to a version that includes Python 3.10 rather than Python 3.7 which removes the need to install the dependency when installing tox. - [Example log of failing job](https://app.circleci.com/pipelines/github/DataDog/dd-trace-py/21679/workflows/2a7606b5-9ca8-4212-9b02-053665de0198/jobs/1477147) - python/importlib_metadata#409 (cherry picked from commit c2b416e) # Conflicts: # .circleci/config.yml
DataDog · Oct 6, 2022 · 88686ba · 88686ba
1 parent ed81229
commit 88686ba
Show file tree

Hide file tree

Showing 3 changed files with 119 additions and 2 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -17,7 +17,11 @@ vertica_image: &vertica_image sumitchawla/vertica:latest
 rabbitmq_image: &rabbitmq_image rabbitmq:3.7-alpine
 
 orbs:
+<<<<<<< HEAD
   win: circleci/windows@2.2.0
+=======
+  win: circleci/windows@5.0
+>>>>>>> c2b416e7 (fix(ci): pin importlib_metadata<5.0.0 (#4257))
 
 machine_executor: &machine_executor
   machine:
@@ -376,6 +380,83 @@ jobs:
       - run_tox_scenario:
           pattern: '^py..-opentracer'
 
+<<<<<<< HEAD
+=======
+  # Building gevent (for which we never want wheels because they crash)
+  # on Python 2.7 requires Microsoft Visual C++ 9.0 which is not installed. :(
+  # which is not installed.
+  # profile-windows-27:
+  #   executor:
+  #     name: win/default
+  #     shell: bash.exe
+  #   steps:
+  #     - run: choco install python2
+  #     - run_tox_scenario:
+  #         store_coverage: false
+  #         pattern: '^py27-profile'
+
+  profile-windows-35:
+    executor:
+      name: win/default
+      shell: bash.exe
+    steps:
+      - run: choco install -y python --version=3.5.4 --side-by-side
+      - run_tox_scenario:
+          store_coverage: false
+          pattern: '^py35-profile'
+
+  profile-windows-36:
+    executor:
+      name: win/default
+      shell: bash.exe
+    steps:
+      - run: choco install -y python --version=3.6.8 --side-by-side
+      - run_tox_scenario:
+          store_coverage: false
+          pattern: '^py36-profile'
+
+  # For whatever reason, choco does not install Python 3.7 correctly on Windows
+  # profile-windows-37:
+  #   executor:
+  #     name: win/default
+  #     shell: bash.exe
+  #   steps:
+  #     - run: choco install python --version=3.7.9
+  #     - run_tox_scenario:
+  #         store_coverage: false
+  #         pattern: '^py37-profile'
+
+  profile-windows-38:
+    executor:
+      name: win/default
+      shell: bash.exe
+    steps:
+      - run: choco install -y python --version=3.8.10 --side-by-side
+      - run_tox_scenario:
+          store_coverage: false
+          pattern: '^py38-profile'
+
+  profile-windows-39:
+    executor:
+      name: win/default
+      shell: bash.exe
+    steps:
+      - run: choco install -y python --version=3.9.12 --side-by-side
+      - run_tox_scenario:
+          store_coverage: false
+          pattern: '^py39-profile'
+
+  profile-windows-310:
+    executor:
+      name: win/default
+      shell: bash.exe
+    steps:
+      # circleci/windows@5.0 orb includes python 3.10.6
+      - run_tox_scenario:
+          store_coverage: false
+          pattern: '^py310-profile'
+
+>>>>>>> c2b416e7 (fix(ci): pin importlib_metadata<5.0.0 (#4257))
   profile:
     <<: *contrib_job
     parallelism: 6

diff --git a/riotfile.py b/riotfile.py
@@ -354,6 +354,7 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                         ],
                         "redis": "~=2.10.6",
                         "kombu": "~=4.3.0",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib-metadata>=5.0
                     },
                 ),
                 Venv(
@@ -377,11 +378,27 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                         "celery": "~=4.2.2",
                         "redis": "~=2.10.6",
                         "kombu": "~=4.3.0",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib_metadata 5.0
                     },
                 ),
                 # Celery 4.3 wants Kombu >= 4.4 and Redis >= 3.2
+                # Split into <3.8 and >=3.8 to pin importlib_metadata dependency for kombu
                 Venv(
-                    pys=select_pys(max_version="3.9"),
+                    pys=select_pys(max_version="3.7"),
+                    pkgs={
+                        "pytest": "~=3.10",
+                        "celery": [
+                            "~=4.3.1",
+                            "~=4.4.7",
+                            "~=4.4",  # most recent 4.x
+                        ],
+                        "redis": "~=3.5",
+                        "kombu": "~=4.4",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib_metadata 5.0
+                    },
+                ),
+                Venv(
+                    pys=select_pys(min_version="3.8", max_version="3.9"),
                     pkgs={
                         "pytest": "~=3.10",
                         "celery": [
@@ -394,8 +411,25 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                     },
                 ),
                 # Celery 5.x wants Python 3.6+
+                # Split into <3.8 and >=3.8 to pin importlib_metadata dependency for kombu
                 Venv(
-                    pys=select_pys(min_version="3.6"),
+                    pys=select_pys(min_version="3.6", max_version="3.7"),
+                    env={
+                        # https://docs.celeryproject.org/en/v5.0.5/userguide/testing.html#enabling
+                        "PYTEST_PLUGINS": "celery.contrib.pytest",
+                    },
+                    pkgs={
+                        "celery": [
+                            "~=5.0.5",
+                            "~=5.0",  # most recent 5.x
+                            latest,
+                        ],
+                        "redis": "~=3.5",
+                        "importlib_metadata": "<5.0",  # kombu using deprecated shims removed in importlib_metadata 5.0
+                    },
+                ),
+                Venv(
+                    pys=select_pys(min_version="3.8"),
                     env={
                         # https://docs.celeryproject.org/en/v5.0.5/userguide/testing.html#enabling
                         "PYTEST_PLUGINS": "celery.contrib.pytest",

diff --git a/tox.ini b/tox.ini
@@ -159,6 +159,8 @@ deps =
     gevent2012: gevent>=20.12,<20.13
     gevent211: gevent>=21.1,<21.2
     gevent218: gevent>=21.8,<21.9
+    # kombu using deprecated shims removed in importlib-metadata 5.0
+    kombu{40,41,42,43,44,45,46,}: importlib_metadata<5.0; python_version<'3.8'
     kombu: kombu
     kombu40: kombu>=4.0,<4.1
     kombu41: kombu>=4.1,<4.2