From 9a86f0546d86d312e8182301d03fc2bb553217ba Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 25 Nov 2024 03:30:53 +0000 Subject: [PATCH 1/6] ENH: Add py-rattler to solve environments --- asv/plugin_manager.py | 2 +- asv/plugins/rattler.py | 135 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 asv/plugins/rattler.py diff --git a/asv/plugin_manager.py b/asv/plugin_manager.py index 16fd68e26..5de35952d 100644 --- a/asv/plugin_manager.py +++ b/asv/plugin_manager.py @@ -30,7 +30,7 @@ def load_plugins(self, package): self.init_plugin(mod) self._plugins.append(mod) except ModuleNotFoundError as err: - if any(keyword in name for keyword in [".mamba", ".virtualenv", ".conda"]): + if any(keyword in name for keyword in [".mamba", ".virtualenv", ".conda", ".rattler"]): continue # Fine to not have these else: log.error(f"Couldn't load {name} because\n{err}") diff --git a/asv/plugins/rattler.py b/asv/plugins/rattler.py new file mode 100644 index 000000000..5704b7a0e --- /dev/null +++ b/asv/plugins/rattler.py @@ -0,0 +1,135 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import os +import re +from pathlib import Path +import asyncio + +from .. import environment, util +from ..console import log + +from rattler import solve, install, VirtualPackage + + +class Rattler(environment.Environment): + """ + Manage an environment using py-rattler. + + Dependencies are installed using py-rattler. The benchmarked + project is installed using the build command specified. + """ + + tool_name = "rattler" + + def __init__(self, conf, python, requirements, tagged_env_vars): + """ + Parameters + ---------- + conf : Config instance + + python : str + Version of Python. Must be of the form "MAJOR.MINOR". + + requirements : dict + Dictionary mapping a PyPI package name to a version + identifier string. + """ + self._python = python + self._requirements = requirements + self._channels = conf.conda_channels + self._environment_file = None + + if conf.conda_environment_file == "IGNORE": + log.debug( + "Skipping environment file due to conda_environment_file set to IGNORE" + ) + self._environment_file = None + elif not conf.conda_environment_file: + if (Path("environment.yml")).exists(): + log.debug("Using environment.yml") + self._environment_file = "environment.yml" + else: + if (Path(conf.conda_environment_file)).exists(): + log.debug(f"Using {conf.conda_environment_file}") + self._environment_file = conf.conda_environment_file + else: + log.debug( + f"Environment file {conf.conda_environment_file} not found, ignoring" + ) + + super(Rattler, self).__init__(conf, python, requirements, tagged_env_vars) + # Rattler configuration things + self._pkg_cache = f"{self._env_dir}/pkgs" + + # TODO(haozeke): Do we need to handle mambarc or something? Have a way to provide channel priority, see mamba + + def _setup(self): + asyncio.run(self._async_setup()) + + async def _async_setup(self): + log.info(f"Creating environment for {self.name}") + + _args, pip_args = self._get_requirements() + _pkgs = ["python", "wheel", "pip"] # baseline, overwritten by env file + env = dict(os.environ) + env.update(self.build_env_vars) + if self._environment_file: + # For named environments + env_file_name = self._environment_file + env_data = load(Path(env_file_name).open(), Loader=Loader) + _pkgs = [x for x in env_data.get("dependencies", []) if isinstance(x, str)] + self._channels += [ + x for x in env_data.get("channels", []) if isinstance(x, str) + ] + self._channels = list(dict.fromkeys(self._channels).keys()) + # Handle possible pip keys + pip_maybe = [ + x for x in env_data.get("dependencies", []) if isinstance(x, dict) + ] + if len(pip_maybe) == 1: + try: + pip_args += pip_maybe[0]["pip"] + except KeyError: + raise KeyError("Only pip is supported as a secondary key") + _pkgs += _args + _pkgs = [util.replace_python_version(pkg, self._python) for pkg in _pkgs] + solved_records = await solve( + # Channels to use for solving + channels=self._channels, + # The specs to solve for + specs=_pkgs, + # Virtual packages define the specifications of the environment + virtual_packages=VirtualPackage.detect(), + ) + await install(records=solved_records, target_prefix=self._path) + if pip_args: + for declaration in pip_args: + parsed_declaration = util.ParsedPipDeclaration(declaration) + pip_call = util.construct_pip_call(self._run_pip, parsed_declaration) + pip_call() + + def _get_requirements(self): + _args = [] + pip_args = [] + + for key, val in {**self._requirements, **self._base_requirements}.items(): + if key.startswith("pip+"): + pip_args.append(f"{key[4:]} {val}") + else: + if val: + _args.append(f"{key}={val}") + else: + _args.append(key) + + return _args, pip_args + + def run_executable(self, executable, args, **kwargs): + return super(Rattler, self).run_executable(executable, args, **kwargs) + + def run(self, args, **kwargs): + log.debug(f"Running '{' '.join(args)}' in {self.name}") + return self.run_executable("python", args, **kwargs) + + def _run_pip(self, args, **kwargs): + # Run pip via python -m pip, so that it works on Windows when + # upgrading pip itself, and avoids shebang length limit on Linux + return self.run_executable("python", ["-mpip"] + list(args), **kwargs) From c34c2b8d5ff8db888140f2f1dff64a31b6d162c8 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 25 Nov 2024 03:31:08 +0000 Subject: [PATCH 2/6] BLD: Add py-rattler as a plugin --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0bda0c2c3..5422e2138 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ hg = [ ] plugs = [ "asv-bench-memray", + "py-rattler", ] testR = [ "rpy2; platform_system != 'Windows' and platform_python_implementation != 'PyPy'", From b1c1706e9abc316a70f09b5ebd28bee70e07d079 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 25 Nov 2024 03:34:07 +0000 Subject: [PATCH 3/6] DOC: Cleanup and use towncrier again Remember to double check with: pipx run towncrier build --draft --version 0.6.5 --date "$(date -u +%Y-%m-%d)" --- CHANGES.rst | 15 --------------- changelog.d/+cdf9af28.misc.rst | 1 + changelog.d/1426.bugfix.rst | 1 + changelog.d/1444.bugfix.rst | 1 + 4 files changed, 3 insertions(+), 15 deletions(-) create mode 100644 changelog.d/+cdf9af28.misc.rst create mode 100644 changelog.d/1426.bugfix.rst create mode 100644 changelog.d/1444.bugfix.rst diff --git a/CHANGES.rst b/CHANGES.rst index 6d9037ab9..e6bc0237e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,18 +1,3 @@ -0.6.5 (2024-08-12) ------------------- - -Bug Fixes -^^^^^^^^^ - -- Multiple Python versions are handled correctly (#1444) -- JSONC fixes (#1426) - -Other Changes and Additions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- New documentation design - - 0.6.4 (2024-08-12) ------------------ diff --git a/changelog.d/+cdf9af28.misc.rst b/changelog.d/+cdf9af28.misc.rst new file mode 100644 index 000000000..7e3792fc0 --- /dev/null +++ b/changelog.d/+cdf9af28.misc.rst @@ -0,0 +1 @@ +New documentation design diff --git a/changelog.d/1426.bugfix.rst b/changelog.d/1426.bugfix.rst new file mode 100644 index 000000000..ee2bce071 --- /dev/null +++ b/changelog.d/1426.bugfix.rst @@ -0,0 +1 @@ +JSONC fixes diff --git a/changelog.d/1444.bugfix.rst b/changelog.d/1444.bugfix.rst new file mode 100644 index 000000000..dff70d8ba --- /dev/null +++ b/changelog.d/1444.bugfix.rst @@ -0,0 +1 @@ +Multiple python versions are now handled correctly From 687a672db381d54b3776ef23075cb5eebba86c76 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 25 Nov 2024 03:42:04 +0000 Subject: [PATCH 4/6] MAINT: Appease linter and minor cleanup --- asv/plugin_manager.py | 3 ++- asv/plugins/rattler.py | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/asv/plugin_manager.py b/asv/plugin_manager.py index 5de35952d..d1c75ec5a 100644 --- a/asv/plugin_manager.py +++ b/asv/plugin_manager.py @@ -7,6 +7,7 @@ from . import commands, plugins from .console import log +ENV_PLUGINS = [".mamba", ".virtualenv", ".conda", ".rattler"] class PluginManager: """ @@ -30,7 +31,7 @@ def load_plugins(self, package): self.init_plugin(mod) self._plugins.append(mod) except ModuleNotFoundError as err: - if any(keyword in name for keyword in [".mamba", ".virtualenv", ".conda", ".rattler"]): + if any(keyword in name for keyword in ENV_PLUGINS): continue # Fine to not have these else: log.error(f"Couldn't load {name} because\n{err}") diff --git a/asv/plugins/rattler.py b/asv/plugins/rattler.py index 5704b7a0e..2b3af7919 100644 --- a/asv/plugins/rattler.py +++ b/asv/plugins/rattler.py @@ -1,14 +1,20 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import os -import re -from pathlib import Path import asyncio +from pathlib import Path -from .. import environment, util -from ..console import log +from yaml import load + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader from rattler import solve, install, VirtualPackage +from .. import environment, util +from ..console import log + class Rattler(environment.Environment): """ @@ -60,7 +66,7 @@ def __init__(self, conf, python, requirements, tagged_env_vars): # Rattler configuration things self._pkg_cache = f"{self._env_dir}/pkgs" - # TODO(haozeke): Do we need to handle mambarc or something? Have a way to provide channel priority, see mamba + # TODO(haozeke): Provide channel priority, see mamba def _setup(self): asyncio.run(self._async_setup()) From 9df531f3af9f4a2fb0d63cc341b8ea32da1ea14e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 25 Nov 2024 03:57:47 +0000 Subject: [PATCH 5/6] TST: Add rattler tests --- test/test_environment_bench.py | 48 +++++++++++++++++++++++++--------- test/tools.py | 7 +++++ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/test/test_environment_bench.py b/test/test_environment_bench.py index 7a957f8d8..a42f7a8bb 100644 --- a/test/test_environment_bench.py +++ b/test/test_environment_bench.py @@ -13,6 +13,8 @@ ENVIRONMENTS.append("conda") if tools.HAS_MAMBA: ENVIRONMENTS.append("mamba") +if tools.HAS_RATTLER: + ENVIRONMENTS.append("rattler") if len(ENVIRONMENTS) == 0: pytest.skip("No environments can be constructed", allow_module_level=True) @@ -128,26 +130,46 @@ def test_asv_benchmark(asv_project_factory, env): @pytest.mark.parametrize( - "config_modifier, expected_success, expected_error", + ("environment", "config_modifier", "expected_success", "expected_error"), [ pytest.param( + env, {"conda_channels": ["conda-forge", "nodefaults"]}, True, None, - id="with_conda_forge", - ), + id=f"with_conda_forge_{env}", + marks=[ + pytest.mark.skipif( + env == "mamba" and not tools.HAS_MAMBA, reason="needs mamba" + ), + pytest.mark.skipif( + env == "rattler" and not tools.HAS_RATTLER, reason="needs rattler" + ), + ], + ) + for env in ["mamba", "rattler"] + ] + + [ pytest.param( + env, {"conda_channels": []}, False, - "Solver could not find solution", - id="empty_conda_channels", - ), + ["Solver could not find solution", "Cannot solve the request"], + id=f"empty_conda_channels_{env}", + marks=[ + pytest.mark.skipif( + env == "mamba" and not tools.HAS_MAMBA, reason="needs mamba" + ), + pytest.mark.skipif( + env == "rattler" and not tools.HAS_RATTLER, reason="needs rattler" + ), + ], + ) + for env in ["mamba", "rattler"] ], ) -@pytest.mark.skipif(not tools.HAS_MAMBA, - reason="needs mamba") def test_asv_mamba( - asv_project_factory, config_modifier, expected_success, expected_error + environment, asv_project_factory, config_modifier, expected_success, expected_error ): """ Test running ASV benchmarks with various configurations, @@ -156,7 +178,7 @@ def test_asv_mamba( project_dir = asv_project_factory(custom_config=config_modifier) try: subprocess.run( - ["asv", "run", "--quick", "--dry-run", "--environment", "mamba"], + ["asv", "run", "--quick", "--dry-run", "--environment", environment], cwd=project_dir, check=True, capture_output=True, @@ -167,12 +189,13 @@ def test_asv_mamba( except subprocess.CalledProcessError as exc: if expected_success: pytest.fail(f"ASV benchmark unexpectedly failed: {exc.stderr}") - elif expected_error and expected_error not in exc.stderr: + elif expected_error and all([err not in exc.stderr for err in expected_error]): pytest.fail( f"Expected error '{expected_error}' not found in stderr: {exc.stderr}" ) +# TODO(haozeke): Add similar tests for rattler @pytest.mark.parametrize( "create_condarc, set_mambarc, expected_success, expected_error", [ @@ -199,8 +222,7 @@ def test_asv_mamba( ), ], ) -@pytest.mark.skipif(not tools.HAS_MAMBA, - reason="needs mamba") +@pytest.mark.skipif(not tools.HAS_MAMBA, reason="needs mamba") def test_asv_mamba_condarc( asv_project_factory, create_condarc, diff --git a/test/tools.py b/test/tools.py index 6d2480edf..980868153 100644 --- a/test/tools.py +++ b/test/tools.py @@ -79,6 +79,13 @@ def _check_conda(): HAS_VIRTUALENV = False +try: + import rattler # noqa F401 checking if installed + HAS_RATTLER = True +except ImportError: + HAS_RATTLER = False + + try: util.which(f'python{PYTHON_VER2}') HAS_PYTHON_VER2 = True From 2fdecc63f05b0b38d8927aef46b0ace92cb26126 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 25 Nov 2024 03:59:53 +0000 Subject: [PATCH 6/6] BLD: Rework environment management --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5422e2138..037aca036 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,12 +80,14 @@ hg = [ ] plugs = [ "asv-bench-memray", +] +envs = [ "py-rattler", ] testR = [ "rpy2; platform_system != 'Windows' and platform_python_implementation != 'PyPy'", ] -all = ["asv[doc,dev,hg,plugs]"] +all = ["asv[doc,dev,hg,envs]"] [build-system] requires = [ "wheel",