diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml index 59c08cf2..6d780eb4 100644 --- a/.github/workflows/upstream_tests.yml +++ b/.github/workflows/upstream_tests.yml @@ -438,7 +438,7 @@ jobs: - name: Setup environment working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE shell: bash -el {0} - timeout-minutes: 10 + timeout-minutes: 15 run: | # CONDA-LIBMAMBA-SOLVER CHANGE cat ../conda-libmamba-solver/dev/requirements.txt >> tests/requirements.txt diff --git a/conda_libmamba_solver/mamba_utils.py b/conda_libmamba_solver/mamba_utils.py index d6891394..2ab87976 100644 --- a/conda_libmamba_solver/mamba_utils.py +++ b/conda_libmamba_solver/mamba_utils.py @@ -9,6 +9,7 @@ # 2022.11.14: only keeping channel prioritization and context initialization logic now import logging +from functools import lru_cache from importlib.metadata import version from typing import Dict @@ -19,6 +20,7 @@ log = logging.getLogger(f"conda.{__name__}") +@lru_cache(maxsize=1) def mamba_version(): return version("libmambapy") diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py index b78a37dd..fc5efe6f 100644 --- a/conda_libmamba_solver/solver.py +++ b/conda_libmamba_solver/solver.py @@ -20,14 +20,7 @@ import libmambapy as api from boltons.setutils import IndexedSet from conda import __version__ as _conda_version -from conda.base.constants import ( - REPODATA_FN, - UNKNOWN_CHANNEL, - ChannelPriority, - DepsModifier, - UpdateModifier, - on_win, -) +from conda.base.constants import REPODATA_FN, UNKNOWN_CHANNEL, ChannelPriority, on_win from conda.base.context import context from conda.common.constants import NULL from conda.common.io import Spinner @@ -178,7 +171,9 @@ def solve_final_state( else: IndexHelper = LibMambaIndexHelper - if os.getenv("CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED"): + if os.getenv("CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED") or ( + getattr(context, "_argparse_args", None) or {} + ).get("override_channels"): # see https://github.com/conda/conda-libmamba-solver/issues/108 channels_from_installed = () else: @@ -301,7 +296,7 @@ def _solving_loop( self._command += "+last_solve_attempt" solved = self._solve_attempt(in_state, out_state, index) if not solved: - message = self._prepare_problems_message() + message = self._prepare_problems_message(pins=out_state.pins) exc = LibMambaUnsatisfiableError(message) exc.allow_retry = False raise exc @@ -326,13 +321,12 @@ def _log_info(self): def _setup_solver(self, index: LibMambaIndexHelper): self._solver_options = solver_options = [ (api.SOLVER_FLAG_ALLOW_DOWNGRADE, 1), - (api.SOLVER_FLAG_ALLOW_UNINSTALL, 1), (api.SOLVER_FLAG_INSTALL_ALSO_UPDATES, 1), - (api.SOLVER_FLAG_FOCUS_BEST, 1), - (api.SOLVER_FLAG_BEST_OBEY_POLICY, 1), ] if context.channel_priority is ChannelPriority.STRICT: solver_options.append((api.SOLVER_FLAG_STRICT_REPO_PRIORITY, 1)) + if self.specs_to_remove and self._command in ("remove", None, NULL): + solver_options.append((api.SOLVER_FLAG_ALLOW_UNINSTALL, 1)) self.solver = api.Solver(index._pool, self._solver_options) @@ -361,22 +355,32 @@ def _solve_attempt( log.debug("Computed specs: %s", out_state.specs) # ## Convert to tasks + out_state.pins.clear() + n_pins = 0 tasks = self._specs_to_tasks(in_state, out_state) for (task_name, task_type), specs in tasks.items(): log.debug("Adding task %s with specs %s", task_name, specs) + if task_name == "ADD_PIN": + # ## Add pins + for spec in specs: + n_pins += 1 + self.solver.add_pin(spec) + out_state.pins[f"pin-{n_pins}"] = spec + continue + try: self.solver.add_jobs(specs, task_type) except RuntimeError as exc: - if mamba_version() == "1.5.0": + if mamba_version().startswith("1.5."): for spec in specs: if spec in str(exc): break else: spec = f"One of {specs}" msg = ( - "This is a bug in libmamba 1.5.0 when using 'defaults::" - "' or 'pkgs/main::'. Please use '-c " - "defaults' instead." + f"This is a bug in libmamba {mamba_version()} when using " + "'defaults::' or 'pkgs/main::'. " + "Consider using '-c defaults' instead." ) raise InvalidMatchSpec(spec, msg) raise @@ -390,7 +394,7 @@ def _solve_attempt( problems = self.solver.problems_to_str() old_conflicts = out_state.conflicts.copy() - new_conflicts = self._maybe_raise_for_problems(problems, old_conflicts) + new_conflicts = self._maybe_raise_for_problems(problems, old_conflicts, out_state.pins) log.debug("Attempt failed with %s conflicts", len(new_conflicts)) out_state.conflicts.update(new_conflicts.items(), reason="New conflict found") return False @@ -418,16 +422,24 @@ def _spec_to_str(spec): return str(spec) def _specs_to_tasks_add(self, in_state: SolverInputState, out_state: SolverOutputState): - # These packages receive special protection, since they will be - # exempt from conflict treatment (ALLOWUNINSTALL) and if installed - # their updates will be considered ESSENTIAL and USERINSTALLED - protected = ( - ["python", "conda"] - + list(in_state.history.keys()) - + list(in_state.aggressive_updates.keys()) - ) + tasks = defaultdict(list) - # Fast-track python version changes + # Protect history and aggressive updates from being uninstalled if possible. From libsolv + # docs: "The matching installed packages are considered to be installed by a user, thus not + # installed to fulfill some dependency. This is needed input for the calculation of + # unneeded packages for jobs that have the SOLVER_CLEANDEPS flag set." + user_installed = { + pkg + for pkg in ( + *in_state.history, + *in_state.aggressive_updates, + *in_state.pinned, + *in_state.do_not_remove, + ) + if pkg in in_state.installed + } + + # Fast-track python version changes (Part 1/2) # ## When the Python version changes, this implies all packages depending on # ## python will be reinstalled too. This can mean that we'll have to try for every # ## installed package to result in a conflict before we get to actually solve everything @@ -439,93 +451,78 @@ def _specs_to_tasks_add(self, in_state: SolverInputState, out_state: SolverOutpu if installed_python and to_be_installed_python: python_version_might_change = not to_be_installed_python.match(installed_python) - tasks = defaultdict(list) + # Add specs to install for name, spec in out_state.specs.items(): if name.startswith("__"): - continue - spec = self._check_spec_compat(spec) - spec_str = self._spec_to_str(spec) - installed = in_state.installed.get(name) - - key = "INSTALL", api.SOLVER_INSTALL + continue # ignore virtual packages + spec: MatchSpec = self._check_spec_compat(spec) + installed: PackageRecord = in_state.installed.get(name) + if installed: + installed_spec_str = self._spec_to_str(installed.to_match_spec()) + else: + installed_spec_str = None + requested: MatchSpec = in_state.requested.get(name) + history: MatchSpec = in_state.history.get(name) + pinned: MatchSpec = in_state.pinned.get(name) + conflicting: MatchSpec = out_state.conflicts.get(name) + + if name in user_installed and not in_state.prune and not conflicting: + tasks[("USERINSTALLED", api.SOLVER_USERINSTALLED)].append(installed_spec_str) + + # These specs are explicit in some sort of way + if pinned: + # these are the EXPLICIT pins; conda also uses implicit pinning to + # constrain updates too but those can be overridden in case of conflicts. + if pinned.is_name_only_spec: + # pins need to constrain in some way, otherwide is undefined behaviour + pass + elif requested and not requested.match(pinned): + # We don't pin; requested and pinned are different and incompatible, + # requested wins and we let that happen in the next block + pass + else: + tasks[("ADD_PIN", api.SOLVER_NOOP)].append(self._spec_to_str(pinned)) - # Fast-track Python version changes: mark non-noarch Python-depending packages as - # conflicting (see `python_version_might_change` definition above for more details) - if python_version_might_change and installed is not None: - if installed.noarch is not None: - continue - for dep in installed.depends: - dep_spec = MatchSpec(dep) - if dep_spec.name in ("python", "python_abi"): - reason = "Python version might change and this package depends on Python" - out_state.conflicts.update( - {name: spec}, - reason=reason, - overwrite=False, - ) - break - - # ## Low-prio task ### - if name in out_state.conflicts and name not in protected: - tasks[("DISFAVOR", api.SOLVER_DISFAVOR)].append(spec_str) - tasks[("ALLOWUNINSTALL", api.SOLVER_ALLOWUNINSTALL)].append(spec_str) - - if installed is not None: - # ## Regular task ### - key = "UPDATE", api.SOLVER_UPDATE - - # ## Protect if installed AND history - if name in protected: - installed_spec = self._spec_to_str(installed.to_match_spec()) - tasks[("USERINSTALLED", api.SOLVER_USERINSTALLED)].append(installed_spec) - # This is "just" an essential job, so it gets higher priority in the solver - # conflict resolution. We do this because these are "protected" packages - # (history, aggressive updates) that we should try not messing with if - # conflicts appear - key = ("UPDATE | ESSENTIAL", api.SOLVER_UPDATE | api.SOLVER_ESSENTIAL) - - # ## Here we deal with the "bare spec update" problem - # ## I am only adding this for legacy / test compliancy reasons; forced updates - # ## like this should (imo) use constrained specs (e.g. conda install python=3) - # ## or the update command as in `conda update python`. however conda thinks - # ## differently of update vs install (quite counterintuitive): - # ## https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/installing-with-conda.html#conda-update-versus-conda-install # noqa - # ## this is tested in: - # ## tests/core/test_solve.py::test_pinned_1 - # ## tests/test_create.py::IntegrationTests::test_update_with_pinned_packages - # ## fixing this changes the outcome in other tests! - # let's say we have an environment with python 2.6 and we say `conda install - # python` libsolv will say we already have python and there's no reason to do - # anything else even if we force an update with essential, other packages in the - # environment (built for py26) will keep it in place. we offer two ways to deal - # with this libsolv behaviour issue: - # A) introduce an artificial version spec `python !=` - # B) use FORCEBEST -- this would be ideal, but sometimes in gets in the way, - # so we only use it as a last attempt effort. - # NOTE: This is a dirty-ish workaround... rethink? - requested = in_state.requested.get(name) - conditions = ( - requested, - spec == requested, - spec.strictness == 1, - self._command in ("update", "update+last_solve_attempt", None, NULL), - in_state.deps_modifier != DepsModifier.ONLY_DEPS, - in_state.update_modifier - not in (UpdateModifier.UPDATE_DEPS, UpdateModifier.FREEZE_INSTALLED), - ) - if all(conditions): - if "last_solve_attempt" in str(self._command): - key = ( - "UPDATE | ESSENTIAL | FORCEBEST", - api.SOLVER_UPDATE | api.SOLVER_ESSENTIAL | api.SOLVER_FORCEBEST, - ) - else: - # NOTE: This is ugly and there should be another way - spec_str = self._spec_to_str( - MatchSpec(spec, version=f">{installed.version}") + if requested: + spec_str = self._spec_to_str(requested) + if installed: + tasks[("UPDATE", api.SOLVER_UPDATE)].append(spec_str) + tasks[("ALLOW_UNINSTALL", api.SOLVER_ALLOWUNINSTALL)].append(name) + else: + tasks[("INSTALL", api.SOLVER_INSTALL)].append(spec_str) + elif name in in_state.always_update: + tasks[("UPDATE", api.SOLVER_UPDATE)].append(name) + tasks[("ALLOW_UNINSTALL", api.SOLVER_ALLOWUNINSTALL)].append(name) + # These specs are "implicit"; the solver logic massages them for better UX + # as long as they don't cause trouble + elif in_state.prune: + continue + elif name == "python" and installed and not pinned: + pyver = ".".join(installed.version.split(".")[:2]) + tasks[("ADD_PIN", api.SOLVER_NOOP)].append(f"python {pyver}.*") + elif history and not history.is_name_only_spec and not conflicting: + tasks[("ADD_PIN", api.SOLVER_NOOP)].append(self._spec_to_str(history)) + elif installed: + if conflicting: + tasks[("ALLOW_UNINSTALL", api.SOLVER_ALLOWUNINSTALL)].append(name) + else: + # we freeze everything else as installed + lock = True + if pinned and pinned.is_name_only_spec: + # name-only pins are treated as locks when installed + lock = True + elif in_state.update_modifier.UPDATE_ALL: + lock = False + if python_version_might_change and installed.noarch is None: + for dep in installed.depends: + if MatchSpec(dep).name in ("python", "python_abi"): + lock = False + break + if lock: + tasks[("LOCK", api.SOLVER_LOCK | api.SOLVER_WEAK)].append( + installed_spec_str ) - - tasks[key].append(spec_str) + tasks[("VERIFY", api.SOLVER_VERIFY | api.SOLVER_WEAK)].append(name) return dict(tasks) @@ -535,6 +532,7 @@ def _specs_to_tasks_remove(self, in_state: SolverInputState, out_state: SolverOu tasks = defaultdict(list) # Protect history and aggressive updates from being uninstalled if possible + for name, record in out_state.records.items(): if name in in_state.history or name in in_state.aggressive_updates: # MatchSpecs constructed from PackageRecords get parsed too @@ -613,7 +611,13 @@ def _parse_problems(cls, problems: str) -> Mapping[str, MatchSpec]: """ conflicts = [] not_found = [] - for line in problems.splitlines(): + if "1.4.5" <= mamba_version() < "1.5.0": + # 1.4.5 had a regression where it would return + # a single line with all the problems; fixed in 1.5.0 + problem_lines = [f" - {problem}" for problem in problems.split(" - ")[1:]] + else: + problem_lines = problems.splitlines()[1:] + for line in problem_lines: line = line.strip() words = line.split() if not line.startswith("- "): @@ -630,6 +634,18 @@ def _parse_problems(cls, problems: str) -> Mapping[str, MatchSpec]: conflicts.append(cls._str_to_matchspec(words[-1])) start = 3 if marker == 4 else 4 not_found.append(cls._str_to_matchspec(words[start:marker])) + elif "has constraint" in line and "conflicting with" in line: + # package libzlib-1.2.11-h4e544f5_1014 has constraint zlib 1.2.11 *_1014 + # conflicting with zlib-1.2.13-h998d150_0 + conflicts.append(cls._str_to_matchspec(words[-1])) + elif "cannot install both pin-" in line and "and pin-" in line: + # a pin is in conflict with another pin + pin_a = words[3].rsplit("-", 1)[0] + pin_b = words[5].rsplit("-", 1)[0] + conflicts.append(MatchSpec(pin_a)) + conflicts.append(MatchSpec(pin_b)) + else: + log.debug("! Problem line not recognized: %s", line) return { "conflicts": {s.name: s for s in conflicts}, @@ -640,6 +656,7 @@ def _maybe_raise_for_problems( self, problems: Optional[Union[str, Mapping]] = None, previous_conflicts: Mapping[str, MatchSpec] = None, + pins: Mapping[str, MatchSpec] = None, ): if self.solver is None: raise RuntimeError("Solver is not initialized. Call `._setup_solver()` first.") @@ -659,7 +676,7 @@ def _maybe_raise_for_problems( not_found = problems["not_found"] if not unsatisfiable and not_found: # This is not a conflict, but a missing package in the channel - exc = PackagesNotFoundError(not_found.values(), self.channels) + exc = PackagesNotFoundError(tuple(not_found.values()), tuple(self.channels)) exc.allow_retry = False raise exc @@ -675,26 +692,72 @@ def _maybe_raise_for_problems( if (previous and (previous_set == current_set)) or len(diff) >= 10: # We have same or more (up to 10) unsatisfiable now! Abort to avoid recursion - message = self._prepare_problems_message() + message = self._prepare_problems_message(pins=pins) exc = LibMambaUnsatisfiableError(message) # do not allow conda.cli.install to try more things exc.allow_retry = False raise exc return unsatisfiable - def _prepare_problems_message(self): + def _prepare_problems_message(self, pins=None): legacy_errors = self.solver.problems_to_str() + if " - " not in legacy_errors: + # This makes 'explain_problems()' crash. Anticipate. + return "Failed with empty error message." if "unsupported request" in legacy_errors: # This error makes 'explain_problems()' crash. Anticipate. log.info("Failed to explain problems. Unsupported request.") return legacy_errors + if ( + mamba_version() <= "1.4.1" + and "conflicting requests" in self.solver.all_problems_to_str() + ): + # This error makes 'explain_problems()' crash in libmamba <=1.4.1. + # Anticipate and return simpler error earlier. + log.info("Failed to explain problems. Conflicting requests.") + return legacy_errors try: explained_errors = self.solver.explain_problems() except Exception as exc: log.warning("Failed to explain problems", exc_info=exc) - return legacy_errors + return self._explain_with_pins(legacy_errors, pins) else: - return f"{legacy_errors}\n{explained_errors}" + msg = f"{legacy_errors}\n{explained_errors}" + return self._explain_with_pins(msg, pins) + + def _explain_with_pins(self, message, pins): + """ + Add info about pins to the error message. + This might be temporary as libmamba improves their error messages. + As of 1.5.0, if a pin introduces a conflict, it results in a cryptic error message like: + + ``` + Encountered problems while solving: + - cannot install both pin-1-1 and pin-1-1 + + Could not solve for environment specs + The following packages are incompatible + └─ pin-1 is installable with the potential options + ├─ pin-1 1, which can be installed; + └─ pin-1 1 conflicts with any installable versions previously reported. + ``` + + Since the pin-N name is masked, we add the following snippet underneath: + + ``` + Pins seem to be involved in the conflict. Currently pinned specs: + - python 2.7.* (labeled as 'pin-1') + + If Python is involved, try adding it explicitly to the command-line. + ``` + """ + if pins and "pin-1" in message: # add info about pins for easier debugging + pin_message = "Pins seem to be involved in the conflict. Currently pinned specs:\n" + for pin_name, spec in pins.items(): + pin_message += f" - {spec} (labeled as '{pin_name}')\n" + pin_message += "\nIf python is involved, try adding it explicitly to the command-line." + return f"{message}\n\n{pin_message}" + return message def _maybe_raise_for_conda_build( self, diff --git a/conda_libmamba_solver/state.py b/conda_libmamba_solver/state.py index a933de2f..e5135b07 100644 --- a/conda_libmamba_solver/state.py +++ b/conda_libmamba_solver/state.py @@ -198,6 +198,8 @@ def __init__( self._update_modifier = self._default_to_context_if_null( "update_modifier", update_modifier ) + if prune and self._update_modifier == UpdateModifier.FREEZE_INSTALLED: + self._update_modifier = UpdateModifier.UPDATE_SPECS # revert to default self._deps_modifier = self._default_to_context_if_null("deps_modifier", deps_modifier) self._ignore_pinned = self._default_to_context_if_null("ignore_pinned", ignore_pinned) self._force_remove = self._default_to_context_if_null("force_remove", force_remove) @@ -282,6 +284,24 @@ def aggressive_updates(self) -> Mapping[str, MatchSpec]: """ return MappingProxyType(self._aggressive_updates) + @property + def always_update(self) -> Mapping[str, MatchSpec]: + """ + Merged lists of packages that should always be updated, depending on the flags, including: + - aggressive_updates + - conda if auto_update_conda is true and we are on the base env + - almost all packages if update_all is true + - etc + """ + pkgs = {pkg: MatchSpec(pkg) for pkg in self.aggressive_updates if pkg in self.installed} + if context.auto_update_conda and paths_equal(self.prefix, context.root_prefix): + pkgs.setdefault("conda", MatchSpec("conda")) + if self.update_modifier.UPDATE_ALL: + for pkg in self.installed: + if pkg != "python" and pkg not in self.pinned: + pkgs.setdefault(pkg, MatchSpec(pkg)) + return MappingProxyType(pkgs) + @property def do_not_remove(self) -> Mapping[str, MatchSpec]: """ @@ -443,6 +463,8 @@ class SolverOutputState: If a solve attempt is not successful, conflicting specs are kept here for further relaxation of the version and build constrains. If not provided, their default value is a blank mapping. + pins + Packages that ended up being pinned. Mostly used for reporting and debugging. Notes ----- @@ -476,6 +498,7 @@ def __init__( for_history: Optional[Mapping[str, MatchSpec]] = None, neutered: Optional[Mapping[str, MatchSpec]] = None, conflicts: Optional[Mapping[str, MatchSpec]] = None, + pins: Optional[Mapping[str, MatchSpec]] = None, ): self.solver_input_state: SolverInputState = solver_input_state @@ -514,6 +537,10 @@ def __init__( "conflicts", data=(conflicts or {}), reason="From arguments" ) + self.pins: Mapping[str, MatchSpec] = TrackedMap( + "pins", data=(pins or {}), reason="From arguments" + ) + def _initialize_specs_from_input_state(self): """ Provide the initial value for the ``.specs`` mapping. This depends on whether @@ -521,8 +548,13 @@ def _initialize_specs_from_input_state(self): """ # Initialize specs following conda.core.solve._collect_all_metadata() - # First initialization depends on whether we have a history to work with or not - if self.solver_input_state.history: + if self.solver_input_state.prune: + pass # we do not initialize specs with history OR installed pkgs if we are pruning + # Otherwise, initialization depends on whether we have a history to work with or not + elif ( + self.solver_input_state.history + and not self.solver_input_state.update_modifier.UPDATE_ALL + ): # add in historically-requested specs self.specs.update(self.solver_input_state.history, reason="As in history") for name, record in self.solver_input_state.installed.items(): @@ -555,7 +587,7 @@ def _initialize_specs_from_input_state(self): # add everything in prefix if we have no history to work with (e.g. with --update-all) self.specs.update( {name: MatchSpec(name) for name in self.solver_input_state.installed}, - reason="Installed and no history available", + reason="Installed and no history available (prune=false)", ) # Add virtual packages so they are taken into account by the solver @@ -664,7 +696,7 @@ def _prepare_for_add(self, index: IndexHelper): ) else: # every other spec that matches something installed will be configured with - # only a target This is the case for conflicts, among others + # only a target. This is the case for conflicts, among others self.specs.set( name, MatchSpec(name, target=record.dist_str()), reason="Spec matches record" ) @@ -765,6 +797,13 @@ def _prepare_for_add(self, index: IndexHelper): reason="Update all, with history: treat pip installed " "stuff as explicitly installed", ) + elif name not in self.specs: + self.specs.set( + name, + MatchSpec(name), + reason="Update all, with history: " + "adding name-only spec from installed", + ) else: for name in sis.installed: if name in sis.pinned: @@ -1001,7 +1040,9 @@ def post_solve(self, solver: Type["Solver"]): history_spec = sis.history.get(name) if history_spec and spec.strictness < history_spec.strictness: self.neutered.set( - name, spec, reason="Spec needs less strict constrains than history" + name, + MatchSpec(name, version=history_spec.get("version")), + reason="Spec needs less strict constrains than history", ) # ## Add inconsistent packages back ### diff --git a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py index fcda36c2..78670a40 100644 --- a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py +++ b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py @@ -25,8 +25,12 @@ # Features / nomkl involved "test_features_solve_1", "test_prune_1", - # TODO: These ones need further investigation - "test_channel_priority_churn_minimized", + "test_update_prune_2", + "test_update_prune_3", + # Message expected, but libmamba does not report constraints + "test_update_prune_5", + # classic expects implicit update to channel with higher priority, including downgrades + # libmamba does not do this, it just stays in the same channel; should it change? "test_priority_1", # The following are known to fail upstream due to too strict expectations # We provide the same tests with adjusted checks in tests/test_modified_upstream.py @@ -41,15 +45,13 @@ "test_downgrade_python_prevented_with_sane_message", ], "tests/test_create.py": [ + # libmamba does not support features "test_remove_features", - # Inconsistency analysis not implemented yet - "test_conda_recovery_of_pip_inconsistent_env", # Known bug in mamba; see https://github.com/mamba-org/mamba/issues/1197 "test_offline_with_empty_index_cache", - "test_neutering_of_historic_specs", + # Adjusted in tests/test_modified_upstream.py + "test_install_features", "test_pinned_override_with_explicit_spec", - # TODO: Investigate why this fails on Windows now - "test_install_update_deps_only_deps_flags", # TODO: https://github.com/conda/conda-libmamba-solver/issues/141 "test_conda_pip_interop_conda_editable_package", ], @@ -86,27 +88,16 @@ "tests/conda_env/specs/test_requirements.py": [ "TestRequirements::test_environment", ], - # TODO: Known to fail; should be fixed by - # https://github.com/conda/conda-libmamba-solver/pull/242 + # Added to test_modified_upstream.py "tests/test_priority.py": ["test_reorder_channel_priority"], } -_broken_by_libmamba_1_4_2 = { + +_broken_by_libmamba_1_5_x = { # conda/tests - "tests/core/test_solve.py": [ - "test_force_remove_1", - "test_aggressive_update_packages", - "test_update_deps_2", - ], - "tests/test_create.py": [ - "test_list_with_pip_wheel", - "test_conda_pip_interop_dependency_satisfied_by_pip", # Linux-only - "test_conda_pip_interop_pip_clobbers_conda", # Linux-only - "test_install_tarball_from_local_channel", # Linux-only - ], - # conda-libmamba-solver/tests - "tests/test_modified_upstream.py": [ - "test_pinned_1", + "tests/test_export.py": [ + "test_explicit", + "test_export", ], } @@ -126,10 +117,15 @@ def pytest_collection_modifyitems(session, config, items): if item_name_no_brackets in _deselected_upstream_tests.get(path_key, []): deselected.append(item) continue - if version( - "libmambapy" - ) >= "1.4.2" and item_name_no_brackets in _broken_by_libmamba_1_4_2.get(path_key, []): - item.add_marker(pytest.mark.xfail(reason="Broken by libmamba 1.4.2; see #186")) + if version("libmambapy").startswith( + "1.5." + ) and item_name_no_brackets in _broken_by_libmamba_1_5_x.get(path_key, []): + item.add_marker( + pytest.mark.xfail( + reason="Broken in libmamba 1.5.x; " + "see https://github.com/mamba-org/mamba/issues/2431." + ) + ) selected.append(item) items[:] = selected config.hook.pytest_deselected(items=deselected) diff --git a/dev/requirements.txt b/dev/requirements.txt index 89b06f37..95b46620 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -2,8 +2,8 @@ pip # run-time boltons>=23.0.0 -conda>=23.3.1 -libmamba>=1.4.1 -libmambapy>=1.4.1 +conda>=23.7.3 +libmamba>=1.5.1 +libmambapy>=1.5.1 # be explicit about sqlite because sometimes it's removed from the env :shrug: sqlite diff --git a/news/270-tasks-and-prune b/news/270-tasks-and-prune new file mode 100644 index 00000000..9c843f75 --- /dev/null +++ b/news/270-tasks-and-prune @@ -0,0 +1,22 @@ +### Enhancements + +* Rewrite how we create tasks for `libsolv`, making use of `libmamba`'s `add_pin` features. (#270) + +### Bug fixes + +* Port logic from [conda/conda#9614](https://github.com/conda/conda/pull/9614), which fixes + a bug where the `--prune` flag was not working correctly in `conda env update` commands. + (#270) +* Ensure environments are not aggressively updated to higher priority channels under some conditions. (#240 via #270) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/pyproject.toml b/pyproject.toml index f108a4bf..43fc9278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,8 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "conda >=23.5.0", - "libmambapy >=1.4.1", + "conda >=23.7.3", + "libmambapy >=1.5.1", "boltons >=23.0.0", ] dynamic = [ diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 6cb7b8cd..6665e98a 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -21,8 +21,8 @@ requirements: - hatch-vcs run: - python >=3.8 - - conda >=23.5.0 - - libmambapy >=1.4.1 + - conda >=23.7.3 + - libmambapy >=1.5.1 - boltons >=23.0.0 test: diff --git a/tests/test_modified_upstream.py b/tests/test_modified_upstream.py index 0651d594..417db390 100644 --- a/tests/test_modified_upstream.py +++ b/tests/test_modified_upstream.py @@ -33,6 +33,13 @@ from conda.gateways.subprocess import subprocess_call_with_clean_env from conda.models.match_spec import MatchSpec from conda.models.version import VersionOrder +from conda.testing import ( + CondaCLIFixture, + TmpEnvFixture, + conda_cli, + path_factory, + tmp_env, +) from conda.testing.cases import BaseTestCase from conda.testing.helpers import ( add_subdir, @@ -52,6 +59,7 @@ package_is_installed, run_command, ) +from pytest import MonkeyPatch @pytest.mark.integration @@ -63,30 +71,6 @@ class PatchedCondaTestCreate(BaseTestCase): def setUp(self): PackageCacheData.clear() - # https://github.com/conda/conda/issues/9124 - @pytest.mark.skipif( - context.subdir != "linux-64", reason="lazy; package constraint here only valid on linux-64" - ) - def test_neutering_of_historic_specs(self): - with make_temp_env("psutil=5.6.3=py37h7b6447c_0") as prefix: - stdout, stderr, _ = run_command(Commands.INSTALL, prefix, "python=3.6") - with open(os.path.join(prefix, "conda-meta", "history")) as f: - d = f.read() - - ## MODIFIED - #  libmamba relaxes more aggressively sometimes - #  instead of relaxing from pkgname=version=build to pkgname=version, it - #  goes to just pkgname; this is because libmamba does not take into account - #  matchspec target and optionality (iow, MatchSpec.conda_build_form() does not) - #  Original check was stricter: - ### assert re.search(r"neutered specs:.*'psutil==5.6.3'\]", d) - assert re.search(r"neutered specs:.*'psutil'\]", d) - ## /MODIFIED - - # this would be unsatisfiable if the neutered specs were not being factored in correctly. - # If this command runs successfully (does not raise), then all is well. - stdout, stderr, _ = run_command(Commands.INSTALL, prefix, "imagesize") - def test_pinned_override_with_explicit_spec(self): with make_temp_env("python=3.6") as prefix: ## MODIFIED @@ -126,6 +110,32 @@ def test_install_update_deps_only_deps_flags(self): assert package_is_installed(prefix, "jinja2>3.0.1") +@pytest.mark.xfail(on_win, reason="nomkl not present on windows", strict=True) +def test_install_features(): + # MODIFIED: Added fixture manually + PackageCacheData.clear() + # /MODIFIED + with make_temp_env("python=2", "numpy=1.13", "nomkl", no_capture=True) as prefix: + assert package_is_installed(prefix, "numpy") + assert package_is_installed(prefix, "nomkl") + assert not package_is_installed(prefix, "mkl") + + with make_temp_env("python=2", "numpy=1.13") as prefix: + assert package_is_installed(prefix, "numpy") + assert not package_is_installed(prefix, "nomkl") + assert package_is_installed(prefix, "mkl") + + # run_command(Commands.INSTALL, prefix, "nomkl", no_capture=True) + run_command(Commands.INSTALL, prefix, "python=2", "nomkl", no_capture=True) + # MODIFIED ^: python=2 needed explicitly to trigger update + assert package_is_installed(prefix, "numpy") + assert package_is_installed(prefix, "nomkl") + assert package_is_installed(prefix, "blas=1.0=openblas") + assert not package_is_installed(prefix, "mkl_fft") + assert not package_is_installed(prefix, "mkl_random") + # assert not package_is_installed(prefix, "mkl") # pruned as an indirect dep + + # The following tests come from `conda/conda::tests/core/test_solve.py` @@ -325,7 +335,10 @@ def test_pinned_1(tmpdir): assert convert_to_dist_str(final_state_5) == order # now update without pinning - specs_to_add = (MatchSpec("python"),) + # MODIFIED: libmamba decides to stay in python=2.6 unless explicit + # specs_to_add = (MatchSpec("python"),) + specs_to_add = (MatchSpec("python=3"),) + # /MODIFIED history_specs = ( MatchSpec("python"), MatchSpec("system=5.8=0"), @@ -1276,3 +1289,52 @@ def test_downgrade_python_prevented_with_sane_message(tmpdir): assert "Encountered problems while solving" in error_msg assert "package unsatisfiable-with-py26-1.0-0 requires scikit-learn 0.13" in error_msg ## /MODIFIED + + +# The following tests come from tests/test_priority.py + + +@pytest.mark.integration +@pytest.mark.parametrize( + "pinned_package", + [ + pytest.param(True, id="with pinned_package"), + pytest.param(False, id="without pinned_package"), + ], +) +def test_reorder_channel_priority( + tmp_env: TmpEnvFixture, + monkeypatch: MonkeyPatch, + conda_cli: CondaCLIFixture, + pinned_package: bool, +): + # use "cheap" packages with no dependencies + package1 = "zlib" + package2 = "ca-certificates" + + # set pinned package + if pinned_package: + monkeypatch.setenv("CONDA_PINNED_PACKAGES", package1) + + # create environment with package1 and package2 + with tmp_env("--override-channels", "--channel=defaults", package1, package2) as prefix: + # check both packages are installed from defaults + PrefixData._cache_.clear() + assert PrefixData(prefix).get(package1).channel.name == "pkgs/main" + assert PrefixData(prefix).get(package2).channel.name == "pkgs/main" + + # update --all + out, err, retcode = conda_cli( + "update", + f"--prefix={prefix}", + "--override-channels", + "--channel=conda-forge", + "--all", + "--yes", + ) + # check pinned package is unchanged but unpinned packages are updated from conda-forge + PrefixData._cache_.clear() + expected_channel = "pkgs/main" if pinned_package else "conda-forge" + assert PrefixData(prefix).get(package1).channel.name == expected_channel + # assert PrefixData(prefix).get(package2).channel.name == "conda-forge" + # MODIFIED ^: Some packages do not change channels in libmamba diff --git a/tests/test_solvers.py b/tests/test_solvers.py index a02a51e9..a675cc0f 100644 --- a/tests/test_solvers.py +++ b/tests/test_solvers.py @@ -10,7 +10,7 @@ from uuid import uuid4 import pytest -from conda.common.compat import on_win +from conda.common.compat import on_linux, on_win from conda.core.prefix_data import PrefixData, get_python_version_for_prefix from conda.testing.integration import Commands, make_temp_env, run_command from conda.testing.solver_helpers import SolverTests @@ -18,6 +18,8 @@ from conda_libmamba_solver import LibMambaSolver from conda_libmamba_solver.mamba_utils import mamba_version +from .utils import conda_subprocess + class TestLibMambaSolver(SolverTests): @property @@ -90,7 +92,7 @@ def test_python_downgrade_reinstalls_noarch_packages(): @pytest.mark.xfail( - mamba_version() == "1.5.0", + mamba_version() in ("1.5.0", "1.5.1"), reason="Known bug. See https://github.com/mamba-org/mamba/issues/2431", ) def test_defaults_specs_work(): @@ -203,3 +205,34 @@ def test_update_from_latest_not_downgrade(tmpdir): ) update_python = PrefixData(prefix).get("python") assert original_python.version == update_python.version + + +@pytest.mark.skipif(not on_linux, reason="Linux only") +def test_too_aggressive_update_to_conda_forge_packages(): + """ + Comes from report in https://github.com/conda/conda-libmamba-solver/issues/240 + We expect a minimum change to the 'base' environment if we only ask for a single package. + conda classic would just change a few (<5) packages, but libmamba seemed to upgrade + EVERYTHING it can to conda-forge. + """ + with make_temp_env("conda", "python", "--override-channels", "--channel=defaults") as prefix: + cmd = ( + "install", + "-p", + prefix, + "-c", + "conda-forge", + "libzlib", + "--json", + "--dry-run", + "-y", + "-vvv", + ) + env = os.environ.copy() + env.pop("CONDA_SOLVER", None) + p_classic = conda_subprocess(*cmd, "--solver=classic", explain=True, env=env) + p_libmamba = conda_subprocess(*cmd, "--solver=libmamba", explain=True, env=env) + data_classic = json.loads(p_classic.stdout) + data_libmamba = json.loads(p_libmamba.stdout) + assert len(data_classic["actions"]["LINK"]) < 15 + assert len(data_libmamba["actions"]["LINK"]) <= len(data_classic["actions"]["LINK"]) diff --git a/tests/utils.py b/tests/utils.py index 5ace7629..179f002f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,7 +18,7 @@ def conda_subprocess(*args, explain=False, capture_output=True, **kwargs) -> Com text=kwargs.pop("text", capture_output), **kwargs, ) - if capture_output and p.returncode: + if capture_output and (explain or p.returncode): print(p.stdout) print(p.stderr, file=sys.stderr) p.check_returncode()