[develop2] test_package --build=missing (#13117)

* wip

* wip

* add tests

* Update conans/test/integration/command/test_package_test.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

---------

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

conan/cli/args.py

@@ -2,11 +2,10 @@
 
 
 _help_build_policies = '''Optional, specify which packages to build from source. Combining multiple
-    '--build' options on one command line is allowed. For dependencies, the optional 'build_policy'
-    attribute in their conanfile.py takes precedence over the command line parameter.
-    Possible parameters:
+    '--build' options on one command line is allowed.
+    Possible values:
 
-    --build="*"        Force build for all packages, do not use binary packages.
+    --build="*"        Force build from source for all packages.
     --build=never      Disallow build for all packages, use binary packages or fail if a binary
                        package is not found. Cannot be combined with other '--build' options.
     --build=missing    Build packages from source whose binary package is not found.
@@ -16,9 +15,8 @@
                        pattern uses 'fnmatch' style wildcards.
     --build=![pattern] Excluded packages, which will not be built from the source, whose package
                        reference matches the pattern. The pattern uses 'fnmatch' style wildcards.
-
-    Default behavior: If you omit the '--build' option, the 'build_policy' attribute in conanfile.py
-    will be used if it exists, otherwise the behavior is like '--build={}'.
+    --build=missing:[pattern] Build from source if a compatible binary does not exist, only for
+                              packages matching pattern.
 '''
 
 
@@ -35,23 +33,20 @@ def add_lockfile_args(parser):
     parser.add_argument("--lockfile-clean", action="store_true", help="remove unused")
 
 
-def _add_common_install_arguments(parser, build_help, update_help=None):
-    if build_help:
-        parser.add_argument("-b", "--build", action="append", help=build_help)
+def add_common_install_arguments(parser):
+    parser.add_argument("-b", "--build", action="append", help=_help_build_policies)
 
     group = parser.add_mutually_exclusive_group()
     group.add_argument("-r", "--remote", action="append", default=None,
                        help='Look in the specified remote or remotes server')
     group.add_argument("-nr", "--no-remote", action="store_true",
                        help='Do not use remote, resolve exclusively in the cache')
 
-    if not update_help:
-        update_help = ("Will check the remote and in case a newer version and/or revision of "
-                       "the dependencies exists there, it will install those in the local cache. "
-                       "When using version ranges, it will install the latest version that "
-                       "satisfies the range. Also, if using revisions, it will update to the "
-                       "latest revision for the resolved version range.")
-
+    update_help = ("Will check the remote and in case a newer version and/or revision of "
+                   "the dependencies exists there, it will install those in the local cache. "
+                   "When using version ranges, it will install the latest version that "
+                   "satisfies the range. Also, if using revisions, it will update to the "
+                   "latest revision for the resolved version range.")
     parser.add_argument("-u", "--update", action='store_true', default=False,
                         help=update_help)
     add_profiles_args(parser)
@@ -120,5 +115,5 @@ def common_graph_args(subparser):
                            help='Directly provide requires instead of a conanfile')
     subparser.add_argument("--tool-requires", action='append',
                            help='Directly provide tool-requires instead of a conanfile')
-    _add_common_install_arguments(subparser, build_help=_help_build_policies.format("never"))
+    add_common_install_arguments(subparser)
     add_lockfile_args(subparser)

conan/cli/commands/build.py

@@ -3,8 +3,7 @@
 from conan.api.output import ConanOutput
 from conan.cli.command import conan_command
 from conan.cli.commands import make_abs_path
-from conan.cli.args import add_lockfile_args, _add_common_install_arguments, add_reference_args, \
-    _help_build_policies
+from conan.cli.args import add_lockfile_args, add_common_install_arguments, add_reference_args
 from conan.internal.conan_app import ConanApp
 from conan.cli.printers.graph import print_graph_packages
 from conans.client.conanfile.build import run_build_method
@@ -23,7 +22,7 @@ def build(conan_api, parser, *args):
     # TODO: Missing --build-require argument and management
     parser.add_argument("-of", "--output-folder",
                         help='The root output folder for generated and build files')
-    _add_common_install_arguments(parser, build_help=_help_build_policies.format("never"))
+    add_common_install_arguments(parser)
     add_lockfile_args(parser)
     args = parser.parse_args(*args)
 

conan/cli/commands/create.py

@@ -5,7 +5,7 @@
 from conan.api.output import ConanOutput, cli_out_write
 from conan.cli.command import conan_command, OnceArgument
 from conan.cli.commands.export import common_args_export
-from conan.cli.args import add_lockfile_args, _add_common_install_arguments, _help_build_policies
+from conan.cli.args import add_lockfile_args, add_common_install_arguments
 from conan.internal.conan_app import ConanApp
 from conan.cli.printers.graph import print_graph_packages
 from conans.client.conanfile.build import run_build_method
@@ -26,7 +26,7 @@ def create(conan_api, parser, *args):
     """
     common_args_export(parser)
     add_lockfile_args(parser)
-    _add_common_install_arguments(parser, build_help=_help_build_policies.format("never"))
+    add_common_install_arguments(parser)
     parser.add_argument("--build-require", action='store_true', default=False,
                         help='The provided reference is a build-require')
     parser.add_argument("-tf", "--test-folder", action=OnceArgument,
@@ -92,12 +92,11 @@ def create(conan_api, parser, *args):
 
     if test_conanfile_path:
         # TODO: We need arguments for:
-        #  - decide build policy for test_package deps "--test_package_build=missing"
         #  - decide update policy "--test_package_update"
         tested_python_requires = ref.repr_notime() if is_python_require else None
         from conan.cli.commands.test import run_test
         deps_graph = run_test(conan_api, test_conanfile_path, ref, profile_host, profile_build,
-                              remotes, lockfile, update=False, build_modes=None,
+                              remotes, lockfile, update=False, build_modes=args.build,
                               tested_python_requires=tested_python_requires)
         lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages,
                                                       clean=args.lockfile_clean)

conan/cli/commands/test.py

@@ -3,7 +3,7 @@
 from conan.api.output import ConanOutput
 from conan.cli.command import conan_command, OnceArgument
 from conan.cli.commands.create import test_package, _check_tested_reference_matches
-from conan.cli.args import add_lockfile_args, _add_common_install_arguments
+from conan.cli.args import add_lockfile_args, add_common_install_arguments
 from conan.cli.printers.graph import print_graph_basic, print_graph_packages
 from conans.model.recipe_ref import RecipeReference
 
@@ -17,7 +17,7 @@ def test(conan_api, parser, *args):
                         help="Path to a test_package folder containing a conanfile.py")
     parser.add_argument("reference", action=OnceArgument,
                         help='Provide a package reference to test')
-    _add_common_install_arguments(parser, build_help=False)  # Used packages must exist
+    add_common_install_arguments(parser)
     add_lockfile_args(parser)
     args = parser.parse_args(*args)
 
@@ -39,7 +39,7 @@ def test(conan_api, parser, *args):
     out.info(profile_build.dumps())
 
     deps_graph = run_test(conan_api, path, ref, profile_host, profile_build, remotes, lockfile,
-                          args.update, build_modes=None)
+                          args.update, build_modes=args.build)
     lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages,
                                                   clean=args.lockfile_clean)
     conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, os.path.dirname(path))

conan/cli/printers/graph.py

@@ -17,16 +17,16 @@ def print_graph_basic(graph):
     for node in graph.nodes:
         if hasattr(node.conanfile, "python_requires"):
             for r in node.conanfile.python_requires._pyrequires.values():  # TODO: improve interface
-                python_requires[r.ref] = r.recipe, r.remote
+                python_requires[r.ref] = r.recipe, r.remote, False
         if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
             continue
         if node.context == CONTEXT_BUILD:
-            build_requires[node.ref] = node.recipe, node.remote
+            build_requires[node.ref] = node.recipe, node.remote, node.test_package
         else:
             if node.test:
-                test_requires[node.ref] = node.recipe, node.remote
+                test_requires[node.ref] = node.recipe, node.remote, node.test_package
             else:
-                requires[node.ref] = node.recipe, node.remote
+                requires[node.ref] = node.recipe, node.remote, node.test_package
         if node.conanfile.deprecated:
             deprecated[node.ref] = node.conanfile.deprecated
 
@@ -38,9 +38,11 @@ def _format_requires(title, reqs_to_print):
         if not reqs_to_print:
             return
         output.info(title, Color.BRIGHT_YELLOW)
-        for ref, (recipe, remote) in sorted(reqs_to_print.items()):
+        for ref, (recipe, remote, test_package) in sorted(reqs_to_print.items()):
             if remote is not None:
                 recipe = "{} ({})".format(recipe, remote.name)
+            if test_package:
+                recipe = f"(tp) {recipe}"
             output.info("    {} - {}".format(ref.repr_notime(), recipe), Color.BRIGHT_CYAN)
 
     _format_requires("Requirements", requires)

conans/client/graph/graph.py

@@ -57,6 +57,7 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False):
         self.binary_remote = None
         self.context = context
         self.test = test
+        self.test_package = False  # True if it is a test_package only package
 
         # real graph model
         self.transitive_deps = OrderedDict()  # of _TransitiveRequirement

conans/client/graph/graph_binaries.py

@@ -300,10 +300,16 @@ def _evaluate_package_id(self, node):
                 conanfile.layout()
 
     def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update):
-        self._selected_remotes = remotes or []# TODO: A bit dirty interfaz, pass as arg instead
+        self._selected_remotes = remotes or []  # TODO: A bit dirty interfaz, pass as arg instead
         self._update = update  # TODO: Dirty, fix it
-        build_mode = BuildMode(build_mode)
+        test_package = deps_graph.root.conanfile.tested_reference_str is not None
+        if test_package:
+            main_mode = BuildMode(["never"])
+            test_mode = BuildMode(build_mode)
+        else:
+            main_mode = test_mode = BuildMode(build_mode)
         for node in deps_graph.ordered_iterate():
+            build_mode = test_mode if node.test_package else main_mode
             if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
                 if node.path is not None and node.path.endswith(".py"):
                     # For .py we keep evaluating the package_id, validate(), etc

conans/client/graph/graph_builder.py

@@ -56,6 +56,7 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None):
                                              for r in reversed(new_node.conanfile.requires.values()))
             self._remove_overrides(dep_graph)
             check_graph_provides(dep_graph)
+            self._compute_test_package_deps(dep_graph)
         except GraphError as e:
             dep_graph.error = e
         dep_graph.resolved_ranges = self._resolver.resolved_ranges
@@ -291,3 +292,31 @@ def _remove_overrides(dep_graph):
             to_remove = [r for r in node.transitive_deps if r.override]
             for r in to_remove:
                 node.transitive_deps.pop(r)
+
+    @staticmethod
+    def _compute_test_package_deps(graph):
+        """ compute and tag the graph nodes that belong exclusively to test_package
+        dependencies but not the main graph
+        """
+        root_node = graph.root
+        tested_ref = root_node.conanfile.tested_reference_str
+        if tested_ref is None:
+            return
+        tested_ref = RecipeReference.loads(root_node.conanfile.tested_reference_str)
+        tested_ref = str(tested_ref)
+        # We classify direct dependencies in the "tested" main ones and the "test_package" specific
+        direct_nodes = [n.node for n in root_node.transitive_deps.values() if n.require.direct]
+        main_nodes = [n for n in direct_nodes if tested_ref == str(n.ref)]
+        test_package_nodes = [n for n in direct_nodes if tested_ref != str(n.ref)]
+
+        # Accumulate the transitive dependencies of the 2 subgraphs ("main", and "test_package")
+        main_graph_nodes = set(main_nodes)
+        for n in main_nodes:
+            main_graph_nodes.update(t.node for t in n.transitive_deps.values())
+        test_graph_nodes = set(test_package_nodes)
+        for n in test_package_nodes:
+            test_graph_nodes.update(t.node for t in n.transitive_deps.values())
+        # Some dependencies in "test_package" might be "main" graph too, "main" prevails
+        test_package_only = test_graph_nodes.difference(main_graph_nodes)
+        for t in test_package_only:
+            t.test_package = True

conans/test/integration/command/test_package_test.py

@@ -2,8 +2,6 @@
 import textwrap
 import unittest
 
-import pytest
-
 from conans.model.recipe_ref import RecipeReference
 from conans.paths import CONANFILE
 from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, GenConanfile
@@ -114,6 +112,46 @@ def test(self):
         self.assertIn("hello/0.1 (test package): TEST HELLO VERSION 0.1", client.out)
 
 
+class TestPackageBuild:
+    def test_build_all(self):
+        c = TestClient()
+        c.save({"tool/conanfile.py": GenConanfile("tool", "0.1"),
+                "dep/conanfile.py": GenConanfile("dep", "0.1"),
+                "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_requires("dep/0.1"),
+                "pkg/test_package/conanfile.py": GenConanfile().with_tool_requires("tool/0.1")
+                                                               .with_test("pass")})
+        c.run("export tool")
+        c.run("export dep")
+        c.run("create pkg --build=*")
+        c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"),
+                                "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")})
+        c.assert_listed_require({"tool/0.1": "(tp) Cache"}, build=True, test_package=True)
+        c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")},
+                               build=True, test_package=True)
+        c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"),
+                                "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Cache")},
+                               test_package=True)
+
+    def test_build_missing(self):
+        c = TestClient()
+        c.save({"tool/conanfile.py": GenConanfile("tool", "0.1"),
+                "dep/conanfile.py": GenConanfile("dep", "0.1"),
+                "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_requires("dep/0.1"),
+                "pkg/test_package/conanfile.py": GenConanfile().with_tool_requires("tool/0.1")
+                                                               .with_test("pass")})
+        c.run("export tool")
+        c.run("create dep")
+        c.run("create pkg --build=missing")
+        c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"),
+                                "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")})
+        c.assert_listed_require({"tool/0.1": "(tp) Cache"}, build=True, test_package=True)
+        c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")},
+                               build=True, test_package=True)
+        c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"),
+                                "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Cache")},
+                               test_package=True)
+
+
 class ConanTestTest(unittest.TestCase):
 
     def test_partial_reference(self):

conans/test/utils/tools.py

@@ -669,10 +669,14 @@ def package_exists(self, pref):
         prev = self.cache.get_package_revisions_references(pref)
         return True if prev else False
 
-    def assert_listed_require(self, requires, build=False, python=False, test=False):
+    def assert_listed_require(self, requires, build=False, python=False, test=False,
+                              test_package=False):
         """ parses the current command output, and extract the first "Requirements" section
         """
         lines = self.out.splitlines()
+        if test_package:
+            line_req = lines.index("-------- test_package: Computing dependency graph --------")
+            lines = lines[line_req:]
         header = "Requirements" if not build else "Build requirements"
         if python:
             header = "Python requires"
@@ -691,11 +695,14 @@ def assert_listed_require(self, requires, build=False, python=False, test=False)
             else:
                 raise AssertionError(f"Cant find {r}-{kind} in {reqs}")
 
-    def assert_listed_binary(self, requires, build=False, test=False):
+    def assert_listed_binary(self, requires, build=False, test=False, test_package=False):
         """ parses the current command output, and extract the second "Requirements" section
         belonging to the computed package binaries
         """
         lines = self.out.splitlines()
+        if test_package:
+            line_req = lines.index("-------- test_package: Computing dependency graph --------")
+            lines = lines[line_req:]
         line_req = lines.index("-------- Computing necessary packages --------")
         header = "Requirements" if not build else "Build requirements"
         if test: