expose a v2 ruleset for BinaryToolBase (#8859)

### Problem

*One step towards a conclusion for #7790.*

v2 rules currently have no way to make use of the extremely useful infrastructure we have for downloading tools hosted by ourselves or others, exposed via the `NativeTool` and `Script` subclasses of `BinaryToolBase`. This requires us to hard-code the digest and url to fetch from for `cloc` and `pex`, e.g.: https://github.com/pantsbuild/pants/blob/fd469cb62f330b73e7814f86e94558d16cfc9da3/src/python/pants/backend/python/rules/download_pex_bin.py#L67-L73

### Solution

- Add a v2 ruleset in `binary_tool.py`, which exposes a `BinaryToolFetchRequest` object which wraps a `BinaryToolBase` instance and converts into a `UrlToFetch`.
- Add a `default_digest` field to `BinaryToolBase`, along with the `--fingerprint` and `--size-bytes` options, so that tools can specify the expected digest for their `default_version`, while giving the user the ability to set `--fingerprint` and `--size-bytes` if they override the `--version`.
- Make cloc and pex use `BinaryToolFetchRequest` instead of hardcoding the url and digest.

### Result

It should be much, much easier to integrate outside tools into v2 `@rule`s! The above pex `@rule` can now look like:
```python
@rule
async def download_pex_bin(pex_binary_tool: DownloadedPexBin.Factory) -> DownloadedPexBin:
  snapshot = await Get[Snapshot](BinaryToolFetchRequest(pex_binary_tool))
  return DownloadedPexBin(SingleFileExecutable(snapshot))
```

src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py

@@ -24,17 +24,24 @@
 from pants.engine.selectors import Params
 from pants.rules.core import strip_source_root
 from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.subsystem.util import init_subsystems
 from pants.testutil.test_base import TestBase
 
 
 class TestPythonAWSLambdaCreation(TestBase):
 
+  def setUp(self):
+    super().setUp()
+    init_subsystems([download_pex_bin.DownloadedPexBin.Factory])
+
   @classmethod
   def rules(cls):
     return (
       *super().rules(),
       *awslambda_python_rules(),
-      *download_pex_bin.rules(),
+      # If we pull in the subsystem_rule() as well from this file, we get an error saying the scope
+      # 'download-pex-bin' was not found when trying to fetch the appropriate scope.
+      download_pex_bin.download_pex_bin,
       *inject_init.rules(),
       *pex.rules(),
       *pex_from_target_closure.rules(),
@@ -43,6 +50,7 @@ def rules(cls):
       *strip_source_root.rules(),
       *subprocess_environment.rules(),
       RootRule(PythonAWSLambdaAdaptor),
+      RootRule(download_pex_bin.DownloadedPexBin.Factory),
     )
 
   def create_python_awslambda(self, addr: str) -> Tuple[str, bytes]:
@@ -52,6 +60,7 @@ def create_python_awslambda(self, addr: str) -> Tuple[str, bytes]:
       Params(
         target.adaptor,
         create_options_bootstrapper(args=["--backend-packages2=pants.backend.awslambda.python"]),
+        download_pex_bin.DownloadedPexBin.Factory.global_instance(),
       ),
     )
     files_content = list(self.request_single_product(FilesContent,

src/python/pants/backend/graph_info/subsystems/BUILD

@@ -4,6 +4,7 @@
 python_library(
   dependencies = [
     'src/python/pants/binaries',
+    'src/python/pants/engine:fs',
   ],
   tags = {"partially_type_checked"},
 )

src/python/pants/backend/graph_info/subsystems/cloc_binary.py

@@ -1,7 +1,9 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.binaries.binary_tool import Script
+from pants.binaries.binary_tool import Script, ToolForPlatform, ToolVersion
+from pants.engine.fs import Digest
+from pants.engine.platform import PlatformConstraint
 
 
 class ClocBinary(Script):
@@ -12,3 +14,11 @@ class ClocBinary(Script):
 
   replaces_scope = 'cloc'
   replaces_name = 'version'
+
+  default_versions_and_digests = {
+    PlatformConstraint.none: ToolForPlatform(
+      digest=Digest('2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d',
+                    546279),
+      version=ToolVersion('1.80'),
+    ),
+  }

src/python/pants/backend/python/lint/bandit/rules_integration_test.py

@@ -7,6 +7,8 @@
 
 from pants.backend.python.lint.bandit.rules import BanditTarget
 from pants.backend.python.lint.bandit.rules import rules as bandit_rules
+from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.subsystems import python_native_code, subprocess_environment
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.build_graph.address import Address
 from pants.build_graph.build_file_aliases import BuildFileAliases
@@ -17,10 +19,16 @@
 from pants.rules.core.lint import LintResult
 from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.subsystem.util import init_subsystems
 from pants.testutil.test_base import TestBase
 
 
 class BanditIntegrationTest(TestBase):
+
+  def setUp(self):
+    super().setUp()
+    init_subsystems([download_pex_bin.DownloadedPexBin.Factory])
+
   good_source = FileContent(path="test/good.py", content=b"hashlib.sha256()\n")
   bad_source = FileContent(path="test/bad.py", content=b"hashlib.md5()\n")
   # MD5 is a insecure hashing function
@@ -31,7 +39,16 @@ def alias_groups(cls) -> BuildFileAliases:
 
   @classmethod
   def rules(cls):
-    return (*super().rules(), *bandit_rules(), RootRule(BanditTarget))
+    return (
+      *super().rules(),
+      *bandit_rules(),
+      download_pex_bin.download_pex_bin,
+      *pex.rules(),
+      *python_native_code.rules(),
+      *subprocess_environment.rules(),
+      RootRule(BanditTarget),
+      RootRule(download_pex_bin.DownloadedPexBin.Factory),
+    )
 
   def run_bandit(
     self,
@@ -60,7 +77,11 @@ def run_bandit(
       )
     )
     return self.request_single_product(
-      LintResult, Params(target, create_options_bootstrapper(args=args)),
+      LintResult, Params(
+        target,
+        create_options_bootstrapper(args=args),
+        download_pex_bin.DownloadedPexBin.Factory.global_instance(),
+      ),
     )
 
   def test_single_passing_source(self) -> None:

src/python/pants/backend/python/lint/black/rules_integration_test.py

@@ -7,6 +7,8 @@
 
 from pants.backend.python.lint.black.rules import BlackTarget
 from pants.backend.python.lint.black.rules import rules as black_rules
+from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.subsystems import python_native_code, subprocess_environment
 from pants.build_graph.address import Address
 from pants.engine.fs import Digest, FileContent, InputFilesContent, Snapshot
 from pants.engine.legacy.structs import TargetAdaptor
@@ -16,11 +18,16 @@
 from pants.rules.core.lint import LintResult
 from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.subsystem.util import init_subsystems
 from pants.testutil.test_base import TestBase
 
 
 class BlackIntegrationTest(TestBase):
 
+  def setUp(self):
+    super().setUp()
+    init_subsystems([download_pex_bin.DownloadedPexBin.Factory])
+
   good_source = FileContent(path="test/good.py", content=b'animal = "Koala"\n')
   bad_source = FileContent(path="test/bad.py", content=b'name=    "Anakin"\n')
   fixed_bad_source = FileContent(path="test/bad.py", content=b'name = "Anakin"\n')
@@ -30,7 +37,16 @@ class BlackIntegrationTest(TestBase):
 
   @classmethod
   def rules(cls):
-    return (*super().rules(), *black_rules(), RootRule(BlackTarget))
+    return (
+      *super().rules(),
+      *black_rules(),
+      download_pex_bin.download_pex_bin,
+      *pex.rules(),
+      *python_native_code.rules(),
+      *subprocess_environment.rules(),
+      RootRule(BlackTarget),
+      RootRule(download_pex_bin.DownloadedPexBin.Factory),
+    )
 
   def run_black(
     self,
@@ -56,8 +72,16 @@ def run_black(
     lint_target = BlackTarget(target_adaptor)
     fmt_target = BlackTarget(target_adaptor, prior_formatter_result_digest=input_snapshot.directory_digest)
     options_bootstrapper = create_options_bootstrapper(args=args)
-    lint_result = self.request_single_product(LintResult, Params(lint_target, options_bootstrapper))
-    fmt_result = self.request_single_product(FmtResult, Params(fmt_target, options_bootstrapper))
+    lint_result = self.request_single_product(LintResult, Params(
+      lint_target,
+      options_bootstrapper,
+      download_pex_bin.DownloadedPexBin.Factory.global_instance(),
+    ))
+    fmt_result = self.request_single_product(FmtResult, Params(
+      fmt_target,
+      options_bootstrapper,
+      download_pex_bin.DownloadedPexBin.Factory.global_instance(),
+    ))
     return lint_result, fmt_result
 
   def get_digest(self, source_files: List[FileContent]) -> Digest:

src/python/pants/backend/python/lint/flake8/rules_integration_test.py

@@ -7,6 +7,8 @@
 
 from pants.backend.python.lint.flake8.rules import Flake8Target
 from pants.backend.python.lint.flake8.rules import rules as flake8_rules
+from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.subsystems import python_native_code, subprocess_environment
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.build_graph.address import Address
 from pants.build_graph.build_file_aliases import BuildFileAliases
@@ -18,10 +20,16 @@
 from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants.testutil.interpreter_selection_utils import skip_unless_python27_and_python3_present
 from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.subsystem.util import init_subsystems
 from pants.testutil.test_base import TestBase
 
 
 class Flake8IntegrationTest(TestBase):
+
+  def setUp(self):
+    super().setUp()
+    init_subsystems([download_pex_bin.DownloadedPexBin.Factory])
+
   good_source = FileContent(path="test/good.py", content=b"print('Nothing suspicious here..')\n")
   bad_source = FileContent(path="test/bad.py", content=b"import typing\n")  # unused import
   py3_only_source = FileContent(path="test/py3.py", content=b"version: str = 'Py3 > Py2'\n")
@@ -32,7 +40,16 @@ def alias_groups(cls) -> BuildFileAliases:
 
   @classmethod
   def rules(cls):
-    return (*super().rules(), *flake8_rules(), RootRule(Flake8Target))
+    return (
+      *super().rules(),
+      *flake8_rules(),
+      download_pex_bin.download_pex_bin,
+      *pex.rules(),
+      *python_native_code.rules(),
+      *subprocess_environment.rules(),
+      RootRule(Flake8Target),
+      RootRule(download_pex_bin.DownloadedPexBin.Factory),
+    )
 
   def run_flake8(
     self,
@@ -61,7 +78,11 @@ def run_flake8(
       )
     )
     return self.request_single_product(
-      LintResult, Params(target, create_options_bootstrapper(args=args)),
+      LintResult, Params(
+        target,
+        create_options_bootstrapper(args=args),
+        download_pex_bin.DownloadedPexBin.Factory.global_instance(),
+      ),
     )
 
   def test_single_passing_source(self) -> None:

src/python/pants/backend/python/lint/isort/rules_integration_test.py

@@ -5,6 +5,8 @@
 
 from pants.backend.python.lint.isort.rules import IsortTarget
 from pants.backend.python.lint.isort.rules import rules as isort_rules
+from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.subsystems import python_native_code, subprocess_environment
 from pants.build_graph.address import Address
 from pants.engine.fs import Digest, FileContent, InputFilesContent, Snapshot
 from pants.engine.legacy.structs import TargetAdaptor
@@ -14,11 +16,16 @@
 from pants.rules.core.lint import LintResult
 from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.subsystem.util import init_subsystems
 from pants.testutil.test_base import TestBase
 
 
 class IsortIntegrationTest(TestBase):
 
+  def setUp(self):
+    super().setUp()
+    init_subsystems([download_pex_bin.DownloadedPexBin.Factory])
+
   good_source = FileContent(path="test/good.py", content=b'from animals import cat, dog\n')
   bad_source = FileContent(path="test/bad.py", content=b'from colors import green, blue\n')
   fixed_bad_source = FileContent(path="test/bad.py", content=b'from colors import blue, green\n')
@@ -36,7 +43,16 @@ class IsortIntegrationTest(TestBase):
 
   @classmethod
   def rules(cls):
-    return (*super().rules(), *isort_rules(), RootRule(IsortTarget))
+    return (
+      *super().rules(),
+      *isort_rules(),
+      download_pex_bin.download_pex_bin,
+      *pex.rules(),
+      *python_native_code.rules(),
+      *subprocess_environment.rules(),
+      RootRule(IsortTarget),
+      RootRule(download_pex_bin.DownloadedPexBin.Factory),
+    )
 
   def run_isort(
     self,
@@ -64,8 +80,16 @@ def run_isort(
       target_adaptor, prior_formatter_result_digest=input_snapshot.directory_digest,
     )
     options_bootstrapper = create_options_bootstrapper(args=args)
-    lint_result = self.request_single_product(LintResult, Params(lint_target, options_bootstrapper))
-    fmt_result = self.request_single_product(FmtResult, Params(fmt_target, options_bootstrapper))
+    lint_result = self.request_single_product(LintResult, Params(
+      lint_target,
+      options_bootstrapper,
+      download_pex_bin.DownloadedPexBin.Factory.global_instance(),
+    ))
+    fmt_result = self.request_single_product(FmtResult, Params(
+      fmt_target,
+      options_bootstrapper,
+      download_pex_bin.DownloadedPexBin.Factory.global_instance(),
+    ))
     return lint_result, fmt_result
 
   def get_digest(self, source_files: List[FileContent]) -> Digest:

src/python/pants/backend/python/lint/python_format_target_integration_test.py

@@ -11,6 +11,8 @@
   _ConcretePythonFormatTarget,
   format_python_target,
 )
+from pants.backend.python.rules import download_pex_bin, pex
+from pants.backend.python.subsystems import python_native_code, subprocess_environment
 from pants.build_graph.address import Address
 from pants.engine.fs import Digest, FileContent, InputFilesContent, Snapshot
 from pants.engine.legacy.structs import TargetAdaptor
@@ -19,21 +21,31 @@
 from pants.rules.core.fmt import AggregatedFmtResults
 from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants.testutil.option.util import create_options_bootstrapper
+from pants.testutil.subsystem.util import init_subsystems
 from pants.testutil.test_base import TestBase
 
 
 class PythonFormatTargetIntegrationTest(TestBase):
 
+  def setUp(self):
+    super().setUp()
+    init_subsystems([download_pex_bin.DownloadedPexBin.Factory])
+
   @classmethod
   def rules(cls):
     return (
       *super().rules(),
       format_python_target,
       *black_rules(),
       *isort_rules(),
+      download_pex_bin.download_pex_bin,
+      *pex.rules(),
+      *python_native_code.rules(),
+      *subprocess_environment.rules(),
       RootRule(_ConcretePythonFormatTarget),
       RootRule(BlackTarget),
       RootRule(IsortTarget),
+      RootRule(download_pex_bin.DownloadedPexBin.Factory),
     )
 
   def run_black_and_isort(
@@ -55,7 +67,8 @@ def run_black_and_isort(
           args=[
             "--backend-packages2=['pants.backend.python.lint.black', 'pants.backend.python.lint.isort']"
           ],
-        )
+        ),
+        download_pex_bin.DownloadedPexBin.Factory.global_instance(),
       )
     )
     return results