fix lockfiles with overrides

conan/api/subapi/lockfile.py

@@ -2,6 +2,7 @@
 
 from conan.api.output import ConanOutput
 from conan.cli import make_abs_path
+from conans.client.graph.graph import Overrides
 from conans.errors import ConanException
 from conans.model.graph_lock import Lockfile, LOCKFILE
 
@@ -12,7 +13,7 @@ def __init__(self, conan_api):
         self.conan_api = conan_api
 
     @staticmethod
-    def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False):
+    def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False, overrides=None):
         """ obtain a lockfile, following this logic:
         - If lockfile is explicitly defined, it would be either absolute or relative to cwd and
           the lockfile file must exist. If lockfile="" (empty string) the default "conan.lock"
@@ -26,6 +27,7 @@ def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False):
         :param cwd: the current working dir, if None, os.getcwd() will be used
         :param conanfile_path: The full path to the conanfile, if existing
         :param lockfile: the name of the lockfile file
+        :param overrides: Dictionary of overrides {overriden: [new_ref1, new_ref2]}
         """
         if lockfile == "":
             # Allow a way with ``--lockfile=""`` to optout automatic usage of conan.lock
@@ -37,6 +39,8 @@ def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False):
             base_path = os.path.dirname(conanfile_path) if conanfile_path else cwd
             lockfile_path = make_abs_path(LOCKFILE, base_path)
             if not os.path.isfile(lockfile_path):
+                if overrides:
+                    raise ConanException("Cannot define overrides without a lockfile")
                 return
         else:  # explicit lockfile given
             lockfile_path = make_abs_path(lockfile, cwd)
@@ -45,6 +49,9 @@ def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False):
 
         graph_lock = Lockfile.load(lockfile_path)
         graph_lock.partial = partial
+
+        if overrides:
+            graph_lock._overrides = Overrides.deserialize(overrides)
         ConanOutput().info("Using lockfile: '{}'".format(lockfile_path))
         return graph_lock
 

conan/cli/args.py

@@ -32,6 +32,8 @@ def add_lockfile_args(parser):
                         help="Lock package-id and package-revision information")
     parser.add_argument("--lockfile-clean", action="store_true",
                         help="Remove unused entries from the lockfile")
+    parser.add_argument("--lockfile-overrides",
+                        help="Overwrite lockfile overrides")
 
 
 def add_common_install_arguments(parser):

conan/cli/commands/build.py

@@ -29,11 +29,12 @@ def build(conan_api, parser, *args):
     path = conan_api.local.get_conanfile_path(args.path, cwd, py=True)
     folder = os.path.dirname(path)
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
-
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
                                                conanfile_path=path,
                                                cwd=cwd,
-                                               partial=args.lockfile_partial)
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
     print_profiles(profile_host, profile_build)
 

conan/cli/commands/create.py

@@ -36,11 +36,12 @@ def create(conan_api, parser, *args):
     cwd = os.getcwd()
     path = conan_api.local.get_conanfile_path(args.path, cwd, py=True)
     test_conanfile_path = _get_test_conanfile_path(args.test_folder, path)
-
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
                                                conanfile_path=path,
                                                cwd=cwd,
-                                               partial=args.lockfile_partial)
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
 

conan/cli/commands/export_pkg.py

@@ -43,8 +43,10 @@ def export_pkg(conan_api, parser, *args):
     cwd = os.getcwd()
     path = conan_api.local.get_conanfile_path(args.path, cwd, py=True)
     test_conanfile_path = _get_test_conanfile_path(args.test_folder, path)
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path,
-                                               cwd=cwd, partial=args.lockfile_partial)
+                                               cwd=cwd, partial=args.lockfile_partial,
+                                               overrides=overrides)
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
 

conan/cli/commands/graph.py

@@ -51,10 +51,12 @@ def graph_build_order(conan_api, parser, subparser, *args):
 
     # Basic collaborators, remotes, lockfile, profiles
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
                                                conanfile_path=path,
                                                cwd=cwd,
-                                               partial=args.lockfile_partial)
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
 
     if path:
@@ -132,10 +134,12 @@ def graph_info(conan_api, parser, subparser, *args):
 
     # Basic collaborators, remotes, lockfile, profiles
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
                                                conanfile_path=path,
                                                cwd=cwd,
-                                               partial=args.lockfile_partial)
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
 
     if path:

conan/cli/commands/install.py

@@ -56,10 +56,12 @@ def install(conan_api, parser, *args):
 
     # Basic collaborators, remotes, lockfile, profiles
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
                                                conanfile_path=path,
                                                cwd=cwd,
-                                               partial=args.lockfile_partial)
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
     print_profiles(profile_host, profile_build)
     if path:

conan/cli/commands/lock.py

@@ -33,8 +33,9 @@ def lock_create(conan_api, parser, subparser, *args):
     cwd = os.getcwd()
     path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path,
-                                               cwd=cwd, partial=True)
+                                               cwd=cwd, partial=True, overrides=overrides)
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
 
     if path:

conan/cli/commands/test.py

@@ -25,10 +25,12 @@ def test(conan_api, parser, *args):
     cwd = os.getcwd()
     ref = RecipeReference.loads(args.reference)
     path = conan_api.local.get_conanfile_path(args.path, cwd, py=True)
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
     lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
                                                conanfile_path=path,
                                                cwd=cwd,
-                                               partial=args.lockfile_partial)
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
     remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
     profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
 

conans/client/graph/graph.py

@@ -227,6 +227,23 @@ def serialize(self):
         result["requires"] = {n.id: n.ref.repr_notime() for n in self.neighbors()}
         return result
 
+    def overrides(self):
+
+        def transitive_subgraph():
+            result = set()
+            opened = {self}
+            while opened:
+                new_opened = set()
+                for o in opened:
+                    result.add(o)
+                    new_opened.update(set(o.neighbors()).difference(result))
+                opened = new_opened
+
+            return result
+
+        nodes = transitive_subgraph()
+        return Overrides.create(nodes)
+
 
 class Edge(object):
     def __init__(self, src, dst, require):
@@ -242,6 +259,9 @@ def __init__(self):
     def __bool__(self):
         return bool(self._overrides)
 
+    def __repr__(self):
+        return repr(self.serialize())
+
     @staticmethod
     def create(nodes):
         overrides = {}
@@ -250,7 +270,7 @@ def create(nodes):
                 if r.override:
                     continue
                 if r.overriden_ref and not r.force:
-                    overrides.setdefault(r.overriden_ref, set()).add(r.ref)
+                    overrides.setdefault(r.overriden_ref, set()).add(r.override_ref)
                 else:
                     overrides.setdefault(r.ref, set()).add(None)
 

conans/client/graph/graph_builder.py

@@ -79,6 +79,7 @@ def _expand_require(self, require, node, graph, profile_host, profile_build, gra
             prev_ref = prev_node.ref if prev_node else prev_require.ref
             if prev_require.force or prev_require.override:  # override
                 require.overriden_ref = require.ref  # Store that the require has been overriden
+                require.override_ref = prev_ref
                 require.ref = prev_ref
             else:
                 self._conflicting_version(require, node, prev_require, prev_node,

conans/client/graph/install_graph.py

@@ -4,7 +4,7 @@
 
 from conan.api.output import ConanOutput
 from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_SKIP, \
-    BINARY_MISSING, BINARY_INVALID
+    BINARY_MISSING, BINARY_INVALID, Overrides, BINARY_BUILD
 from conans.errors import ConanInvalidConfiguration, ConanException
 from conans.model.recipe_ref import RecipeReference
 from conans.util.files import load
@@ -29,17 +29,21 @@ def __init__(self):
         # to cross compile, there will be a dependency from the current "self" (host context)
         # to that "build" package_id.
         self.depends = []  # List of package_ids of dependencies to other binaries of the same ref
+        self.overrides = Overrides()
+        self.ref = None
 
     @staticmethod
     def create(node):
         result = _InstallPackageReference()
+        result.ref = node.ref
         result.package_id = node.pref.package_id
         result.prev = node.pref.revision
         result.binary = node.binary
         result.context = node.context
         # self_options are the minimum to reproduce state
         result.options = node.conanfile.self_options.dumps().splitlines()
         result.nodes.append(node)
+        result.overrides = node.overrides()
         return result
 
     def add(self, node):
@@ -50,25 +54,40 @@ def add(self, node):
         # assert self.context == node.context
         self.nodes.append(node)
 
+    def _build_args(self):
+        if self.binary != BINARY_BUILD:
+            return None
+        cmd = f"--require={self.ref}" if self.context == "host" else f"--tool-require={self.ref}"
+        cmd += f" --build={self.ref}"
+        if self.options:
+            cmd += " " + " ".join(f"-o {o}" for o in self.options)
+        if self.overrides:
+            cmd += f' --lockfile-overrides="{self.overrides}"'
+        return cmd
+
     def serialize(self):
         return {"package_id": self.package_id,
                 "prev": self.prev,
                 "context": self.context,
                 "binary": self.binary,
                 "options": self.options,
                 "filenames": self.filenames,
-                "depends": self.depends}
+                "depends": self.depends,
+                "overrides": self.overrides.serialize(),
+                "build_args": self._build_args()}
 
     @staticmethod
-    def deserialize(data, filename):
+    def deserialize(data, filename, ref):
         result = _InstallPackageReference()
+        result.ref = ref
         result.package_id = data["package_id"]
         result.prev = data["prev"]
         result.binary = data["binary"]
         result.context = data["context"]
         result.options = data["options"]
         result.filenames = data["filenames"] or [filename]
         result.depends = data["depends"]
+        result.overrides = Overrides.deserialize(data["overrides"])
         return result
 
 
@@ -154,7 +173,7 @@ def deserialize(data, filename):
             result.depends.append(RecipeReference.loads(d))
         for level in data["packages"]:
             for p in level:
-                install_node = _InstallPackageReference.deserialize(p, filename)
+                install_node = _InstallPackageReference.deserialize(p, filename, result.ref)
                 result.packages[install_node.package_id] = install_node
         return result
 

conans/model/graph_lock.py

@@ -2,7 +2,8 @@
 import os
 from collections import OrderedDict
 
-from conans.client.graph.graph import RECIPE_VIRTUAL, RECIPE_CONSUMER, CONTEXT_BUILD
+from conan.api.output import ConanOutput
+from conans.client.graph.graph import RECIPE_VIRTUAL, RECIPE_CONSUMER, CONTEXT_BUILD, Overrides
 from conans.errors import ConanException
 from conans.model.recipe_ref import RecipeReference
 from conans.util.files import load, save
@@ -87,6 +88,7 @@ def __init__(self, deps_graph=None, lock_packages=False):
         self._python_requires = _LockRequires()
         self._build_requires = _LockRequires()
         self._alias = {}
+        self._overrides = Overrides()
         self.partial = False
 
         if deps_graph is None:
@@ -112,6 +114,7 @@ def update_lock(self, deps_graph, lock_packages=False):
                 self._requires.add(graph_node.ref, pids)
 
         self._alias.update(deps_graph.aliased)
+        self._overrides.update(deps_graph.overrides())
 
         self._requires.sort()
         self._build_requires.sort()
@@ -147,6 +150,7 @@ def merge(self, other):
         self._build_requires.merge(other._build_requires)
         self._python_requires.merge(other._python_requires)
         self._alias.update(other._alias)
+        self._overrides.update(other._overrides)
 
     def add(self, requires=None, build_requires=None, python_requires=None):
         """ adding new things manually will trigger the sort() of the locked list, so lockfiles
@@ -185,6 +189,8 @@ def deserialize(data):
         if "alias" in data:
             graph_lock._alias = {RecipeReference.loads(k): RecipeReference.loads(v)
                                  for k, v in data["alias"].items()}
+        if "overrides" in data:
+            graph_lock._overrides = Overrides.deserialize(data["overrides"])
         return graph_lock
 
     def serialize(self):
@@ -200,14 +206,34 @@ def serialize(self):
             result["python_requires"] = self._python_requires.serialize()
         if self._alias:
             result["alias"] = {repr(k): repr(v) for k, v in self._alias.items()}
+        if self._overrides:
+            result["overrides"] = self._overrides.serialize()
         return result
 
     def resolve_locked(self, node, require, resolve_prereleases):
         if require.build or node.context == CONTEXT_BUILD:
             locked_refs = self._build_requires.refs()
         else:
             locked_refs = self._requires.refs()
-        self._resolve(require, locked_refs, resolve_prereleases)
+        self._resolve_overrides(require)
+        try:
+            self._resolve(require, locked_refs, resolve_prereleases)
+        except ConanException:
+            overrides = self._overrides.get(require.ref)
+            if overrides is not None and len(overrides) > 1:
+                msg = f"Override defined for {require.ref}, but multiple possible overrides" \
+                      f" {overrides}. You might need to apply the 'conan graph build-order'" \
+                      f" overrides for correctly building this package with this lockfile"
+                ConanOutput().error(msg)
+            raise
+
+    def _resolve_overrides(self, require):
+        existing = self._overrides.get(require.ref)
+        if existing is not None and len(existing) == 1:
+            require.overriden_ref = require.ref  # Store that the require has been overriden
+            ref = next(iter(existing))
+            require.ref = ref
+            require.override_ref = ref
 
     def resolve_prev(self, node):
         if node.context == CONTEXT_BUILD:

conans/model/requires.py

@@ -29,6 +29,7 @@ def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visib
         self._direct = direct
         self.options = options
         self.overriden_ref = None  # to store if the requirement has been overriden (store old ref)
+        self.override_ref = None  # to store if the requirement has been overriden (store new ref)
 
     @property
     def skip(self):