Alternative logic for solver creation tasks (#270)

* start branch * trigger ci * port logic from conda/conda#9614 * deselect a couple tests (accelerate and libmamba do not get along) * add news * Apply suggestions from code review * Alternative logic for solver creation tasks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix tests/core/test_solve.py::test_auto_update_conda * some progress with pins (but not quite there yet) * Increase timeouts in CI env setup * allow uninstall by default only on removals * handle prune * avoid crash * cleanup * only lock the installed spec * pre-commit * add SolverInputState.always_update helper mapping * allow uninstall of conflicts * use always_update list for simplicity * not update-all if pinned * explain pins if involved in errors * re-enable original test_neutering_of_historic_specs * bump minimum required conda and libmamba versions * fix tests/core/test_solve.py::test_force_remove_1 and tests/core/test_solve.py::test_pinned_1 * Re-enable tests that are failing * Require libmambapy >= 1.5.1 * pre-commit * amend news * Update __init__.py * fix test_neutering_of_historic_specs * consider 1.5.1 in warnings too * pre-commit * adjust test * remove comment * fix json serialization * xfail in 1.5.1 too * catch RuntimeError for the whole 1.5.x series * xfail test_explicit * fix test_pinned_1 * lock installed name-only pinned packages * do not add installed channels to list if --override-channels in use * add adjusted variant for test_priority * always define lock * explain some deselected tests * adjust test_install_features * xfail test_export * pre-commit --------- Co-authored-by: Jannis Leidel <jannis@leidel.info> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Chris Ostrouchov <chris.ostrouchov@gmail.com>
conda · Sep 15, 2023 · 03a9ad2 · 03a9ad2
1 parent 090ee32
commit 03a9ad2
Show file tree

Hide file tree

Showing 12 changed files with 404 additions and 185 deletions.
diff --git a/.github/workflows/upstream_tests.yml b/.github/workflows/upstream_tests.yml
@@ -438,7 +438,7 @@ jobs:
       - name: Setup environment
         working-directory: conda  # CONDA-LIBMAMBA-SOLVER CHANGE
         shell: bash -el {0}
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |
           # CONDA-LIBMAMBA-SOLVER CHANGE
           cat ../conda-libmamba-solver/dev/requirements.txt >> tests/requirements.txt

diff --git a/conda_libmamba_solver/mamba_utils.py b/conda_libmamba_solver/mamba_utils.py
@@ -9,6 +9,7 @@
 # 2022.11.14: only keeping channel prioritization and context initialization logic now
 
 import logging
+from functools import lru_cache
 from importlib.metadata import version
 from typing import Dict
 
@@ -19,6 +20,7 @@
 log = logging.getLogger(f"conda.{__name__}")
 
 
+@lru_cache(maxsize=1)
 def mamba_version():
     return version("libmambapy")
 

diff --git a/conda_libmamba_solver/solver.py b/conda_libmamba_solver/solver.py
diff --git a/conda_libmamba_solver/state.py b/conda_libmamba_solver/state.py
@@ -198,6 +198,8 @@ def __init__(
         self._update_modifier = self._default_to_context_if_null(
             "update_modifier", update_modifier
         )
+        if prune and self._update_modifier == UpdateModifier.FREEZE_INSTALLED:
+            self._update_modifier = UpdateModifier.UPDATE_SPECS  # revert to default
         self._deps_modifier = self._default_to_context_if_null("deps_modifier", deps_modifier)
         self._ignore_pinned = self._default_to_context_if_null("ignore_pinned", ignore_pinned)
         self._force_remove = self._default_to_context_if_null("force_remove", force_remove)
@@ -282,6 +284,24 @@ def aggressive_updates(self) -> Mapping[str, MatchSpec]:
         """
         return MappingProxyType(self._aggressive_updates)
 
+    @property
+    def always_update(self) -> Mapping[str, MatchSpec]:
+        """
+        Merged lists of packages that should always be updated, depending on the flags, including:
+        - aggressive_updates
+        - conda if auto_update_conda is true and we are on the base env
+        - almost all packages if update_all is true
+        - etc
+        """
+        pkgs = {pkg: MatchSpec(pkg) for pkg in self.aggressive_updates if pkg in self.installed}
+        if context.auto_update_conda and paths_equal(self.prefix, context.root_prefix):
+            pkgs.setdefault("conda", MatchSpec("conda"))
+        if self.update_modifier.UPDATE_ALL:
+            for pkg in self.installed:
+                if pkg != "python" and pkg not in self.pinned:
+                    pkgs.setdefault(pkg, MatchSpec(pkg))
+        return MappingProxyType(pkgs)
+
     @property
     def do_not_remove(self) -> Mapping[str, MatchSpec]:
         """
@@ -443,6 +463,8 @@ class SolverOutputState:
         If a solve attempt is not successful, conflicting specs are kept here for further
         relaxation of the version and build constrains. If not provided, their default value is a
         blank mapping.
+    pins
+        Packages that ended up being pinned. Mostly used for reporting and debugging.
 
     Notes
     -----
@@ -476,6 +498,7 @@ def __init__(
         for_history: Optional[Mapping[str, MatchSpec]] = None,
         neutered: Optional[Mapping[str, MatchSpec]] = None,
         conflicts: Optional[Mapping[str, MatchSpec]] = None,
+        pins: Optional[Mapping[str, MatchSpec]] = None,
     ):
         self.solver_input_state: SolverInputState = solver_input_state
 
@@ -514,15 +537,24 @@ def __init__(
             "conflicts", data=(conflicts or {}), reason="From arguments"
         )
 
+        self.pins: Mapping[str, MatchSpec] = TrackedMap(
+            "pins", data=(pins or {}), reason="From arguments"
+        )
+
     def _initialize_specs_from_input_state(self):
         """
         Provide the initial value for the ``.specs`` mapping. This depends on whether
         there's a history available (existing prefix) or not (new prefix).
         """
         # Initialize specs following conda.core.solve._collect_all_metadata()
 
-        # First initialization depends on whether we have a history to work with or not
-        if self.solver_input_state.history:
+        if self.solver_input_state.prune:
+            pass  # we do not initialize specs with history OR installed pkgs if we are pruning
+        # Otherwise, initialization depends on whether we have a history to work with or not
+        elif (
+            self.solver_input_state.history
+            and not self.solver_input_state.update_modifier.UPDATE_ALL
+        ):
             # add in historically-requested specs
             self.specs.update(self.solver_input_state.history, reason="As in history")
             for name, record in self.solver_input_state.installed.items():
@@ -555,7 +587,7 @@ def _initialize_specs_from_input_state(self):
             # add everything in prefix if we have no history to work with (e.g. with --update-all)
             self.specs.update(
                 {name: MatchSpec(name) for name in self.solver_input_state.installed},
-                reason="Installed and no history available",
+                reason="Installed and no history available (prune=false)",
             )
 
         # Add virtual packages so they are taken into account by the solver
@@ -664,7 +696,7 @@ def _prepare_for_add(self, index: IndexHelper):
                 )
             else:
                 # every other spec that matches something installed will be configured with
-                # only a target This is the case for conflicts, among others
+                # only a target. This is the case for conflicts, among others
                 self.specs.set(
                     name, MatchSpec(name, target=record.dist_str()), reason="Spec matches record"
                 )
@@ -765,6 +797,13 @@ def _prepare_for_add(self, index: IndexHelper):
                             reason="Update all, with history: treat pip installed "
                             "stuff as explicitly installed",
                         )
+                    elif name not in self.specs:
+                        self.specs.set(
+                            name,
+                            MatchSpec(name),
+                            reason="Update all, with history: "
+                            "adding name-only spec from installed",
+                        )
             else:
                 for name in sis.installed:
                     if name in sis.pinned:
@@ -1001,7 +1040,9 @@ def post_solve(self, solver: Type["Solver"]):
             history_spec = sis.history.get(name)
             if history_spec and spec.strictness < history_spec.strictness:
                 self.neutered.set(
-                    name, spec, reason="Spec needs less strict constrains than history"
+                    name,
+                    MatchSpec(name, version=history_spec.get("version")),
+                    reason="Spec needs less strict constrains than history",
                 )
 
         # ## Add inconsistent packages back ###

diff --git a/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py b/dev/collect_upstream_conda_tests/collect_upstream_conda_tests.py
@@ -25,8 +25,12 @@
         # Features / nomkl involved
         "test_features_solve_1",
         "test_prune_1",
-        # TODO: These ones need further investigation
-        "test_channel_priority_churn_minimized",
+        "test_update_prune_2",
+        "test_update_prune_3",
+        # Message expected, but libmamba does not report constraints
+        "test_update_prune_5",
+        # classic expects implicit update to channel with higher priority, including downgrades
+        # libmamba does not do this, it just stays in the same channel; should it change?
         "test_priority_1",
         # The following are known to fail upstream due to too strict expectations
         # We provide the same tests with adjusted checks in tests/test_modified_upstream.py
@@ -41,15 +45,13 @@
         "test_downgrade_python_prevented_with_sane_message",
     ],
     "tests/test_create.py": [
+        # libmamba does not support features
         "test_remove_features",
-        # Inconsistency analysis not implemented yet
-        "test_conda_recovery_of_pip_inconsistent_env",
         # Known bug in mamba; see https://github.com/mamba-org/mamba/issues/1197
         "test_offline_with_empty_index_cache",
-        "test_neutering_of_historic_specs",
+        # Adjusted in tests/test_modified_upstream.py
+        "test_install_features",
         "test_pinned_override_with_explicit_spec",
-        # TODO: Investigate why this fails on Windows now
-        "test_install_update_deps_only_deps_flags",
         # TODO: https://github.com/conda/conda-libmamba-solver/issues/141
         "test_conda_pip_interop_conda_editable_package",
     ],
@@ -86,27 +88,16 @@
     "tests/conda_env/specs/test_requirements.py": [
         "TestRequirements::test_environment",
     ],
-    # TODO: Known to fail; should be fixed by
-    # https://github.com/conda/conda-libmamba-solver/pull/242
+    # Added to test_modified_upstream.py
     "tests/test_priority.py": ["test_reorder_channel_priority"],
 }
 
-_broken_by_libmamba_1_4_2 = {
+
+_broken_by_libmamba_1_5_x = {
     # conda/tests
-    "tests/core/test_solve.py": [
-        "test_force_remove_1",
-        "test_aggressive_update_packages",
-        "test_update_deps_2",
-    ],
-    "tests/test_create.py": [
-        "test_list_with_pip_wheel",
-        "test_conda_pip_interop_dependency_satisfied_by_pip",  # Linux-only
-        "test_conda_pip_interop_pip_clobbers_conda",  # Linux-only
-        "test_install_tarball_from_local_channel",  # Linux-only
-    ],
-    # conda-libmamba-solver/tests
-    "tests/test_modified_upstream.py": [
-        "test_pinned_1",
+    "tests/test_export.py": [
+        "test_explicit",
+        "test_export",
     ],
 }
 
@@ -126,10 +117,15 @@ def pytest_collection_modifyitems(session, config, items):
         if item_name_no_brackets in _deselected_upstream_tests.get(path_key, []):
             deselected.append(item)
             continue
-        if version(
-            "libmambapy"
-        ) >= "1.4.2" and item_name_no_brackets in _broken_by_libmamba_1_4_2.get(path_key, []):
-            item.add_marker(pytest.mark.xfail(reason="Broken by libmamba 1.4.2; see #186"))
+        if version("libmambapy").startswith(
+            "1.5."
+        ) and item_name_no_brackets in _broken_by_libmamba_1_5_x.get(path_key, []):
+            item.add_marker(
+                pytest.mark.xfail(
+                    reason="Broken in libmamba 1.5.x; "
+                    "see https://github.com/mamba-org/mamba/issues/2431."
+                )
+            )
         selected.append(item)
     items[:] = selected
     config.hook.pytest_deselected(items=deselected)
diff --git a/dev/requirements.txt b/dev/requirements.txt
@@ -2,8 +2,8 @@
 pip
 # run-time
 boltons>=23.0.0
-conda>=23.3.1
-libmamba>=1.4.1
-libmambapy>=1.4.1
+conda>=23.7.3
+libmamba>=1.5.1
+libmambapy>=1.5.1
 # be explicit about sqlite because sometimes it's removed from the env :shrug:
 sqlite
diff --git a/news/270-tasks-and-prune b/news/270-tasks-and-prune
@@ -0,0 +1,22 @@
+### Enhancements
+
+* Rewrite how we create tasks for `libsolv`, making use of `libmamba`'s `add_pin` features. (#270)
+
+### Bug fixes
+
+* Port logic from [conda/conda#9614](https://github.com/conda/conda/pull/9614), which fixes
+  a bug where the `--prune` flag was not working correctly in `conda env update` commands.
+  (#270)
+* Ensure environments are not aggressively updated to higher priority channels under some conditions. (#240 via #270)
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>
diff --git a/pyproject.toml b/pyproject.toml
@@ -23,8 +23,8 @@ classifiers = [
 ]
 requires-python = ">=3.8"
 dependencies = [
-  "conda >=23.5.0",
-  "libmambapy >=1.4.1",
+  "conda >=23.7.3",
+  "libmambapy >=1.5.1",
   "boltons >=23.0.0",
 ]
 dynamic = [

diff --git a/recipe/meta.yaml b/recipe/meta.yaml
@@ -21,8 +21,8 @@ requirements:
     - hatch-vcs
   run:
     - python >=3.8
-    - conda >=23.5.0
-    - libmambapy >=1.4.1
+    - conda >=23.7.3
+    - libmambapy >=1.5.1
     - boltons >=23.0.0
 
 test: