From 5c1b0df5197cad8bf55f5cfd144e47b55008d093 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?=
 =?UTF-8?q?=E0=A4=A4=29?= <suagatchhetri@outlook.com>
Date: Sun, 28 Aug 2022 23:16:15 +0545
Subject: [PATCH] test on 3.11

We don't have aiohttp support yet, due to which we have to install
without cythonized extension.

Similarly, there has been some changes in how dataclasses disallows some
mutable arguments to be set as a class property, due to which
`hydra.conf` fails to import in 3.11.

See https://github.com/python/cpython/pull/29867. For now, I have
patched the module and avoid importing as a pytest-plugin.

dvc-hdfs/pyarrow is not there yet, we'll usually see a release 2-3
months after official reelase.

dvc-hdfs is skipped in 3.11 in extras_requires. `--set-params` will
return a nice error message if hydra cannot be imported in 3.11. I
decided to only patch that in tests.
---
 .github/workflows/tests.yaml       |  5 ++++
 dvc/repo/experiments/queue/base.py | 10 ++++++-
 pyproject.toml                     |  6 ++---
 setup.cfg                          |  4 +--
 tests/conftest.py                  | 12 +++++++++
 tests/func/utils/test_hydra.py     |  5 +++-
 tests/unit/fs/test_fs.py           | 43 +++++++++++++++++-------------
 7 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 4d09676e55..d20e615cac 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -43,6 +43,9 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.8", "3.9", "3.10"]
+        include:
+        - {os: ubuntu-latest, pyv: "3.11-dev"}
+
     steps:
     - uses: actions/checkout@v3
       with:
@@ -68,6 +71,8 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install -e ".[dev]"
+      env:
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.pyv == '3.11-dev' && '1' }}
     - name: run tests
       timeout-minutes: 40
       run: >-
diff --git a/dvc/repo/experiments/queue/base.py b/dvc/repo/experiments/queue/base.py
index 4ba87c98fd..d3ff14e853 100644
--- a/dvc/repo/experiments/queue/base.py
+++ b/dvc/repo/experiments/queue/base.py
@@ -1,5 +1,6 @@
 import logging
 import os
+import sys
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 from typing import (
@@ -524,7 +525,14 @@ def _update_params(self, params: Dict[str, List[str]]):
         """
         logger.debug("Using experiment params '%s'", params)
 
-        from dvc.utils.hydra import apply_overrides
+        try:
+            from dvc.utils.hydra import apply_overrides
+        except ValueError:
+            if sys.version_info >= (3, 11):
+                raise DvcException(
+                    "--set-param is not supported in Python >= 3.11"
+                )
+            raise
 
         for path, overrides in params.items():
             apply_overrides(path, overrides)
diff --git a/pyproject.toml b/pyproject.toml
index ce18034d51..f2ddf12379 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,11 +30,11 @@ line_length = 79
 
 [tool.pytest.ini_options]
 log_level = "debug"
-addopts = "-ra"
+addopts = "-ra -p no:hydra_pytest"
 markers = [
     "needs_internet: Might need network access for the tests",
-"vscode: Tests verifying contract between DVC and VSCode plugin",
-"studio: Tests verifying contract between DVC and Studio"
+    "vscode: Tests verifying contract between DVC and VSCode plugin",
+    "studio: Tests verifying contract between DVC and Studio",
 ]
 xfail_strict = true
 
diff --git a/setup.cfg b/setup.cfg
index e1f6bc8c26..0997885e06 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -62,8 +62,6 @@ install_requires =
     rich>=10.13.0
     pyparsing>=2.4.7
     typing-extensions>=3.7.4
-    fsspec[http]>=2021.10.1
-    aiohttp-retry>=2.4.5
     scmrepo==0.0.25
     dvc-render==0.0.9
     dvc-task==0.1.2
@@ -91,7 +89,7 @@ dev =
 azure = dvc-azure==2.19.0
 gdrive = dvc-gdrive==2.19.0
 gs = dvc-gs==2.19.1
-hdfs = dvc-hdfs==2.19.0
+hdfs = dvc-hdfs==2.19.0; python_version < '3.11'
 oss = dvc-oss==2.19.0
 s3 = dvc-s3==2.19.0
 ssh = dvc-ssh==2.19.0
diff --git a/tests/conftest.py b/tests/conftest.py
index 0dffd7eeef..4c46ba448c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -250,3 +250,15 @@ def run(
         return stage
 
     return run
+
+
+@pytest.fixture(autouse=True)
+def mock_hydra_conf(mocker):
+    if sys.version_info < (3, 11):
+        return
+
+    # `hydra.conf` fails to import in 3.11, it raises ValueError due to changes
+    # in dataclasses. See https://github.com/python/cpython/pull/29867.
+    # NOTE: using sentinel here so that any imports from `hydra.conf`
+    # return a mock.
+    sys.modules["hydra.conf"] = mocker.sentinel
diff --git a/tests/func/utils/test_hydra.py b/tests/func/utils/test_hydra.py
index 462405ab77..06bf159a6c 100644
--- a/tests/func/utils/test_hydra.py
+++ b/tests/func/utils/test_hydra.py
@@ -1,7 +1,6 @@
 import pytest
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.utils.hydra import apply_overrides
 
 
 @pytest.mark.parametrize("suffix", ["yaml", "toml", "json"])
@@ -90,6 +89,8 @@
     ],
 )
 def test_apply_overrides(tmp_dir, suffix, overrides, expected):
+    from dvc.utils.hydra import apply_overrides
+
     if suffix == "toml":
         if overrides in [
             ["foo=baz"],
@@ -114,6 +115,8 @@ def test_apply_overrides(tmp_dir, suffix, overrides, expected):
     [["foobar=2"], ["lorem=3,2"], ["+lorem=3"], ["foo[0]=bar"]],
 )
 def test_invalid_overrides(tmp_dir, overrides):
+    from dvc.utils.hydra import apply_overrides
+
     params_file = tmp_dir / "params.yaml"
     params_file.dump(
         {"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}
diff --git a/tests/unit/fs/test_fs.py b/tests/unit/fs/test_fs.py
index c96d2f69dc..456e792689 100644
--- a/tests/unit/fs/test_fs.py
+++ b/tests/unit/fs/test_fs.py
@@ -1,5 +1,4 @@
 import pytest
-from dvc_hdfs import HDFSFileSystem
 from dvc_http import HTTPFileSystem, HTTPSFileSystem
 from dvc_s3 import S3FileSystem
 from dvc_ssh import SSHFileSystem
@@ -7,25 +6,31 @@
 from dvc.config import RemoteNotFoundError
 from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
 
+url_cls_pairs = [
+    ("s3://bucket/path", S3FileSystem),
+    ("ssh://example.com:/dir/path", SSHFileSystem),
+    ("http://example.com/path/to/file", HTTPFileSystem),
+    ("https://example.com/path/to/file", HTTPSFileSystem),
+    ("path/to/file", LocalFileSystem),
+    ("path\\to\\file", LocalFileSystem),
+    ("file", LocalFileSystem),
+    ("./file", LocalFileSystem),
+    (".\\file", LocalFileSystem),
+    ("../file", LocalFileSystem),
+    ("..\\file", LocalFileSystem),
+    ("unknown://path", LocalFileSystem),
+]
 
-@pytest.mark.parametrize(
-    "url, cls",
-    [
-        ("s3://bucket/path", S3FileSystem),
-        ("ssh://example.com:/dir/path", SSHFileSystem),
-        ("hdfs://example.com/dir/path", HDFSFileSystem),
-        ("http://example.com/path/to/file", HTTPFileSystem),
-        ("https://example.com/path/to/file", HTTPSFileSystem),
-        ("path/to/file", LocalFileSystem),
-        ("path\\to\\file", LocalFileSystem),
-        ("file", LocalFileSystem),
-        ("./file", LocalFileSystem),
-        (".\\file", LocalFileSystem),
-        ("../file", LocalFileSystem),
-        ("..\\file", LocalFileSystem),
-        ("unknown://path", LocalFileSystem),
-    ],
-)
+
+try:
+    from dvc_hdfs import HDFSFileSystem
+
+    url_cls_pairs += [("hdfs://example.com/dir/path", HDFSFileSystem)]
+except ImportError:
+    pass
+
+
+@pytest.mark.parametrize("url, cls", url_cls_pairs)
 def test_get_fs_cls(url, cls):
     assert get_fs_cls({"url": url}) == cls