test on 3.11

We don't have aiohttp support yet, due to which we have to install
without cythonized extension. Similarly, hydra does not yet support
3.11 (given it's track record, it'll be almost a year we'll see the wheels
for it). dvc-hdfs/pyarrow is not there yet, we'll usually see a release 2-3
months after official reelase.

dvc-hdfs is skipped in 3.11 in extras_requires. `--set-params` fails with
hydra being not available.

Author: skshetry

.github/workflows/tests.yaml

@@ -43,6 +43,9 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.8", "3.9", "3.10"]
+        include:
+        - {os: ubuntu-latest, pyv: "3.11-dev"}
+
     steps:
     - uses: actions/checkout@v3
       with:
@@ -68,6 +71,8 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install -e ".[dev]"
+      env:
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.pyv == '3.11-dev' && '1' }}
     - name: run tests
       timeout-minutes: 40
       run: >-

dvc/repo/experiments/queue/base.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import sys
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 from typing import (
@@ -524,7 +525,12 @@ def _update_params(self, params: Dict[str, List[str]]):
         """
         logger.debug("Using experiment params '%s'", params)
 
-        from dvc.utils.hydra import apply_overrides
+        try:
+            from dvc.utils.hydra import apply_overrides
+        except ImportError:
+            if sys.version_info >= (3, 11):
+                logger.error("`--set-param` is not supported in Python>=3.11")
+            raise
 
         for path, overrides in params.items():
             apply_overrides(path, overrides)

pyproject.toml

@@ -33,8 +33,9 @@ log_level = "debug"
 addopts = "-ra"
 markers = [
     "needs_internet: Might need network access for the tests",
-"vscode: Tests verifying contract between DVC and VSCode plugin",
-"studio: Tests verifying contract between DVC and Studio"
+    "vscode: Tests verifying contract between DVC and VSCode plugin",
+    "studio: Tests verifying contract between DVC and Studio",
+    "requires_hydra: Tests need hydra and will be skipped if not installed",
 ]
 xfail_strict = true
 

setup.cfg

@@ -62,15 +62,13 @@ install_requires =
     rich>=10.13.0
     pyparsing>=2.4.7
     typing-extensions>=3.7.4
-    fsspec[http]>=2021.10.1
-    aiohttp-retry>=2.4.5
     scmrepo==0.0.25
     dvc-render==0.0.9
     dvc-task==0.1.2
     dvclive>=0.10.0
     dvc-data==0.2.0
-    dvc-http==2.19.0
-    hydra-core>=1.1.0
+    dvc-http==2.19.1
+    hydra-core>=1.1.0; python_version < '3.11'
 
 [options.extras_require]
 all =
@@ -91,7 +89,7 @@ dev =
 azure = dvc-azure==2.19.0
 gdrive = dvc-gdrive==2.19.0
 gs = dvc-gs==2.19.0
-hdfs = dvc-hdfs==2.19.0
+hdfs = dvc-hdfs==2.19.0; python_version < '3.11'
 oss = dvc-oss==2.19.0
 s3 = dvc-s3==2.19.0
 ssh = dvc-ssh==2.19.0

tests/conftest.py

@@ -127,6 +127,9 @@ def pytest_runtest_setup(item):
         # so we rerun them in case it fails.
         item.add_marker(pytest.mark.flaky(max_runs=5, min_passes=1))
 
+    if "requires_hydra" in item.keywords:
+        pytest.importorskip("hydra")
+
 
 @pytest.fixture(scope="session")
 def test_config(request):

tests/func/experiments/conftest.py

@@ -1,10 +1,29 @@
+import sys
 from textwrap import dedent
 
 import pytest
 
 from tests.func.test_repro_multistage import COPY_SCRIPT
 from tests.unit.repo.experiments.conftest import test_queue  # noqa
 
+
+@pytest.fixture(autouse=True)
+def mock_dvc_exp_run(mocker):
+    from dvc.repo.experiments.queue.base import BaseStashQueue
+
+    original = BaseStashQueue._update_params
+
+    def update_params(*args):
+        if sys.version_info >= (3, 11):
+            pytest.importorskip("hydra")
+        return original(*args)
+
+    mocker.patch(
+        "dvc.repo.experiments.queue.base.BaseStashQueue._update_params",
+        wraps=update_params,
+    )
+
+
 DEFAULT_ITERATIONS = 2
 CHECKPOINT_SCRIPT_FORMAT = dedent(
     """\

tests/func/test_live.py

@@ -127,6 +127,7 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage):
     assert "logs/images/1/image.jpg" in files
 
 
+@pytest.mark.requires_hydra
 @pytest.mark.parametrize("typ", ("live", "live_no_cache"))
 def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ):
     live_stage(**{typ: "logs"})
@@ -188,6 +189,7 @@ def checkpoints_metric(show_results, metric_file, metric_name):
     ]
 
 
+@pytest.mark.requires_hydra
 @pytest.mark.parametrize("typ", ("live", "live_no_cache"))
 def test_live_checkpoints_resume(
     tmp_dir, scm, dvc, live_checkpoint_stage, typ

tests/func/utils/test_hydra.py

@@ -1,7 +1,8 @@
 import pytest
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.utils.hydra import apply_overrides
+
+pytest.importorskip("hydra")
 
 
 @pytest.mark.parametrize("suffix", ["yaml", "toml", "json"])
@@ -90,6 +91,8 @@
     ],
 )
 def test_apply_overrides(tmp_dir, suffix, overrides, expected):
+    from dvc.utils.hydra import apply_overrides
+
     if suffix == "toml":
         if overrides in [
             ["foo=baz"],
@@ -114,6 +117,8 @@ def test_apply_overrides(tmp_dir, suffix, overrides, expected):
     [["foobar=2"], ["lorem=3,2"], ["+lorem=3"], ["foo[0]=bar"]],
 )
 def test_invalid_overrides(tmp_dir, overrides):
+    from dvc.utils.hydra import apply_overrides
+
     params_file = tmp_dir / "params.yaml"
     params_file.dump(
         {"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}

tests/unit/fs/test_fs.py

@@ -1,19 +1,25 @@
 import pytest
-from dvc_hdfs import HDFSFileSystem
 from dvc_http import HTTPFileSystem, HTTPSFileSystem
 from dvc_s3 import S3FileSystem
 from dvc_ssh import SSHFileSystem
 
 from dvc.config import RemoteNotFoundError
 from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
 
+try:
+    from dvc_hdfs import HDFSFileSystem
+except ImportError:
+    url_cls_pairs = []
+else:
+    url_cls_pairs = [("hdfs://example.com/dir/path", HDFSFileSystem)]
+
 
 @pytest.mark.parametrize(
     "url, cls",
     [
+        *url_cls_pairs,
         ("s3://bucket/path", S3FileSystem),
         ("ssh://example.com:/dir/path", SSHFileSystem),
-        ("hdfs://example.com/dir/path", HDFSFileSystem),
         ("http://example.com/path/to/file", HTTPFileSystem),
         ("https://example.com/path/to/file", HTTPSFileSystem),
         ("path/to/file", LocalFileSystem),