From 410b570fbbcaff6a4d0ba483b3eaeb68936f024e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Tue, 29 Oct 2024 08:16:38 +0545 Subject: [PATCH 1/2] bench: drop bench_datachain fixture --- tests/benchmarks/conftest.py | 124 ------------------------------- tests/benchmarks/test_ls.py | 4 +- tests/benchmarks/test_version.py | 4 +- 3 files changed, 4 insertions(+), 128 deletions(-) diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index d6838bd38..1483da0c1 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -1,12 +1,6 @@ -import os -import shutil from pathlib import Path -from subprocess import check_output import pytest -import virtualenv -from dulwich.porcelain import clone -from packaging import version @pytest.fixture @@ -14,124 +8,6 @@ def bucket(): return "s3://noaa-bathymetry-pds/" -def pytest_generate_tests(metafunc): - str_revs = metafunc.config.getoption("--datachain-revs") - revs = str_revs.split(",") if str_revs else [None] - if "datachain_rev" in metafunc.fixturenames: - metafunc.parametrize("datachain_rev", revs, scope="session") - - -class VirtualEnv: - def __init__(self, path) -> None: - self.path = path - self.bin = self.path / ("Scripts" if os.name == "nt" else "bin") - - def create(self) -> None: - virtualenv.cli_run([os.fspath(self.path)]) - - def run(self, cmd: str, *args: str, env=None) -> None: - exe = self.which(cmd) - check_output([exe, *args], env=env) # noqa: S603 - - def which(self, cmd: str) -> str: - assert self.bin.exists() - return shutil.which(cmd, path=self.bin) or cmd - - -@pytest.fixture(scope="session", name="make_datachain_venv") -def fixture_make_datachain_venv(tmp_path_factory): - def _make_datachain_venv(name): - venv_dir = tmp_path_factory.mktemp(f"datachain-venv-{name}") - venv = VirtualEnv(venv_dir) - venv.create() - return venv - - return _make_datachain_venv - - -@pytest.fixture(scope="session", name="datachain_venvs") -def fixture_datachain_venvs(): - return {} - - -@pytest.fixture(scope="session", name="datachain_git_repo") -def fixture_datachain_git_repo(tmp_path_factory, test_config): - url = test_config.datachain_git_repo - - if os.path.isdir(url): - return url - - tmp_path = os.fspath(tmp_path_factory.mktemp("datachain-git-repo")) - clone(url, tmp_path) - - return tmp_path - - -@pytest.fixture(scope="session", name="datachain_bin") -def fixture_datachain_bin( - datachain_rev, - datachain_venvs, - make_datachain_venv, - datachain_git_repo, - test_config, -): - if datachain_rev: - venv = datachain_venvs.get(datachain_rev) - if not venv: - venv = make_datachain_venv(datachain_rev) - venv.run("pip", "install", "-U", "pip") - venv.run( - "pip", "install", f"git+file://{datachain_git_repo}@{datachain_rev}" - ) - datachain_venvs[datachain_rev] = venv - datachain_bin = venv.which("datachain") - else: - datachain_bin = test_config.datachain_bin - - def _datachain_bin(*args): - return check_output([datachain_bin, *args], text=True) # noqa: S603 - - actual = version.parse(_datachain_bin("--version")) - _datachain_bin.version = (actual.major, actual.minor, actual.micro) - - return _datachain_bin - - -@pytest.fixture(scope="function", name="make_bench") -def fixture_make_bench(request): - def _make_bench(name): - import pytest_benchmark.plugin - - # hack from https://github.com/ionelmc/pytest-benchmark/issues/166 - bench = pytest_benchmark.plugin.benchmark.__pytest_wrapped__.obj(request) - - suffix = f"-{name}" - - def add_suffix(_name): - start, sep, end = _name.partition("[") - return start + suffix + sep + end - - bench.name = add_suffix(bench.name) - bench.fullname = add_suffix(bench.fullname) - - return bench - - return _make_bench - - -@pytest.fixture( - scope="function", params=[pytest.param(None, marks=pytest.mark.benchmark)] -) -def bench_datachain(datachain_bin, make_bench): - def _bench_datachain(*args, **kwargs): - name = kwargs.pop("name", None) - name = f"-{name}" if name else "" - bench = make_bench(args[0] + name) - return bench.pedantic(datachain_bin, args=args, **kwargs) - - return _bench_datachain - - @pytest.fixture def datasets(): return Path(__file__).parent / "datasets" diff --git a/tests/benchmarks/test_ls.py b/tests/benchmarks/test_ls.py index ed9a10494..3fdb41fcc 100644 --- a/tests/benchmarks/test_ls.py +++ b/tests/benchmarks/test_ls.py @@ -1,2 +1,2 @@ -def test_ls(bench_datachain, tmp_dir, bucket): - bench_datachain("ls", bucket, "--anon") +def test_ls(benchmark, tmp_dir, bucket): + benchmark("ls", bucket, "--anon") diff --git a/tests/benchmarks/test_version.py b/tests/benchmarks/test_version.py index fae3d6395..7701855b4 100644 --- a/tests/benchmarks/test_version.py +++ b/tests/benchmarks/test_version.py @@ -1,2 +1,2 @@ -def test_version(bench_datachain): - bench_datachain("--help", rounds=100) +def test_version(benchmark): + benchmark("--help", rounds=100) From 6f86e5d4a000022c36b07600a65f4edbce89d0fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Tue, 29 Oct 2024 08:29:26 +0545 Subject: [PATCH 2/2] bench: drop bench_datachain fixture --- noxfile.py | 3 +-- pyproject.toml | 3 +-- tests/benchmarks/conftest.py | 5 ----- tests/benchmarks/test_datachain.py | 3 --- tests/benchmarks/test_ls.py | 8 ++++++-- tests/benchmarks/test_version.py | 7 ++++++- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/noxfile.py b/noxfile.py index 7f71f85d4..72b8d9722 100644 --- a/noxfile.py +++ b/noxfile.py @@ -22,8 +22,7 @@ def bench(session: nox.Session) -> None: session.install(".[tests]") session.run( "pytest", - "-m", - "benchmark", + "--benchmark-only", "--benchmark-group-by", "func", *session.posargs, diff --git a/pyproject.toml b/pyproject.toml index 59e4017e4..b0aafc5f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,9 +127,8 @@ namespaces = false [tool.setuptools_scm] [tool.pytest.ini_options] -addopts = "-rfEs -m 'not benchmark and not examples'" +addopts = "-rfEs -m 'not examples' --benchmark-skip" markers = [ - "benchmark: benchmarks.", "e2e: End-to-end tests", "examples: All examples", "computer_vision: Computer vision examples", diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 1483da0c1..2e5153472 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -3,11 +3,6 @@ import pytest -@pytest.fixture -def bucket(): - return "s3://noaa-bathymetry-pds/" - - @pytest.fixture def datasets(): return Path(__file__).parent / "datasets" diff --git a/tests/benchmarks/test_datachain.py b/tests/benchmarks/test_datachain.py index 5103fbf25..3dc5b4e92 100644 --- a/tests/benchmarks/test_datachain.py +++ b/tests/benchmarks/test_datachain.py @@ -1,10 +1,7 @@ -import pytest - from datachain.lib.dc import DataChain from datachain.lib.webdataset_laion import process_laion_meta -@pytest.mark.benchmark def test_datachain(tmp_dir, test_session, datasets, benchmark): def run_script(uri, **kwargs): DataChain.from_storage(uri, session=test_session, **kwargs).gen( diff --git a/tests/benchmarks/test_ls.py b/tests/benchmarks/test_ls.py index 3fdb41fcc..19916a2a4 100644 --- a/tests/benchmarks/test_ls.py +++ b/tests/benchmarks/test_ls.py @@ -1,2 +1,6 @@ -def test_ls(benchmark, tmp_dir, bucket): - benchmark("ls", bucket, "--anon") +from datachain.cli import ls + + +def test_ls(benchmark, tmp_dir): + bucket = "s3://noaa-bathymetry-pds/" + benchmark.pedantic(ls, args=([bucket],), kwargs={"client_config": {"anon": True}}) diff --git a/tests/benchmarks/test_version.py b/tests/benchmarks/test_version.py index 7701855b4..54fcece4e 100644 --- a/tests/benchmarks/test_version.py +++ b/tests/benchmarks/test_version.py @@ -1,2 +1,7 @@ +import shutil +import subprocess + + def test_version(benchmark): - benchmark("--help", rounds=100) + bin = shutil.which("datachain") + benchmark(subprocess.check_call, [bin, "--help"])