From 07a308e5932e051628dbb56bc75772ceacb27f1d Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Wed, 27 Oct 2021 12:38:49 -0700 Subject: [PATCH] [internal] Don't download Go third-party dependencies multiple times (#13352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns out that https://github.com/pantsbuild/pants/pull/13339 didn't actually work - we are redownloading the same modules several times with Go! Downloads happen when: 1. determining `GoModInfo` (once per `go.mod`) 2. `AllDownloadedModules` (once per `go.mod`) 3. Determining metadata for each third-party package (once per third-party module) 4. Determining metadata for each first-party package (once per first-party package/directory) This PR fixes it so that we only download modules a single time, once per `go.mod`. To fix this, we stop treating third-party modules like first-party modules, i.e. we stop `cd`-ing into its downloaded directory and running `go list` directly in it, using its own `go.mod` and `go.sum`. That requires that the chroot has all of the module's transitive dependencies present, and it also resulted in issues like https://github.com/pantsbuild/pants/issues/13138. Instead, the much simpler thing to do is run `go list '...'` to do all third-party analysis in a single swoop. That gets us all the analysis we need. We also extract the relevant `.go` files from all of the downloaded `GOPATH`, i.e. all the downloaded modules. For compilation, all we need is the `.go` files + the metadata we had earlier calculated. Compilation doesn't need access to anything else like other package's. For first-party analysis, we copy the whole `GOPATH` into the chroot. (This is really slow! We need something like https://github.com/pantsbuild/pants/issues/12716 to fix this.) ## Benchmark Running in https://github.com/toolchainlabs/remote-api-tools. Before: ``` ❯ hyperfine -r 5 './pants_from_sources --no-process-execution-local-cache --no-pantsd package ::' Time (mean ± σ): 36.467 s ± 0.603 s [User: 41.109 s, System: 38.095 s] Range (min … max): 35.518 s … 37.137 s 5 runs ``` Fixing only third-party analysis: ``` ❯ hyperfine -r 5 --show-output './pants_from_sources --no-process-execution-local-cache --no-pantsd package ::' Time (mean ± σ): 29.880 s ± 0.901 s [User: 29.564 s, System: 15.281 s] Range (min … max): 28.835 s … 31.312 s 5 runs ``` Fixing everything: ``` ❯ hyperfine -r 5 './pants_from_sources --no-process-execution-local-cache --no-pantsd package ::' Time (mean ± σ): 26.633 s ± 2.283 s [User: 24.115 s, System: 30.453 s] Range (min … max): 24.570 s … 30.037 s 5 runs ``` [ci skip-rust] [ci skip-build-wheels] --- .../pants/backend/go/target_type_rules.py | 35 +- .../backend/go/target_type_rules_test.py | 90 +- src/python/pants/backend/go/target_types.py | 29 +- .../pants/backend/go/util_rules/build_pkg.py | 17 +- .../backend/go/util_rules/build_pkg_test.py | 6 +- .../backend/go/util_rules/first_party_pkg.py | 16 +- .../go/util_rules/first_party_pkg_test.py | 25 +- .../pants/backend/go/util_rules/go_mod.py | 55 +- .../backend/go/util_rules/go_mod_test.py | 82 +- src/python/pants/backend/go/util_rules/sdk.py | 1 + .../backend/go/util_rules/third_party_pkg.py | 394 ++++----- .../go/util_rules/third_party_pkg_test.py | 795 ++++++++++-------- 12 files changed, 677 insertions(+), 868 deletions(-) diff --git a/src/python/pants/backend/go/target_type_rules.py b/src/python/pants/backend/go/target_type_rules.py index 8d698bef3d4..41039dcd428 100644 --- a/src/python/pants/backend/go/target_type_rules.py +++ b/src/python/pants/backend/go/target_type_rules.py @@ -19,8 +19,6 @@ GoImportPathField, GoModPackageSourcesField, GoModTarget, - GoThirdPartyModulePathField, - GoThirdPartyModuleVersionField, GoThirdPartyPackageDependenciesField, GoThirdPartyPackageTarget, ) @@ -32,8 +30,8 @@ from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest from pants.backend.go.util_rules.import_analysis import GoStdLibImports from pants.backend.go.util_rules.third_party_pkg import ( - ThirdPartyModuleInfo, - ThirdPartyModuleInfoRequest, + AllThirdPartyPackages, + AllThirdPartyPackagesRequest, ThirdPartyPkgInfo, ThirdPartyPkgInfoRequest, ) @@ -153,12 +151,7 @@ async def inject_go_third_party_package_dependencies( tgt = wrapped_target.target pkg_info = await Get( ThirdPartyPkgInfo, - ThirdPartyPkgInfoRequest( - module_path=tgt[GoThirdPartyModulePathField].value, - version=tgt[GoThirdPartyModuleVersionField].value, - import_path=tgt[GoImportPathField].value, - go_mod_stripped_digest=go_mod_info.stripped_digest, - ), + ThirdPartyPkgInfoRequest(tgt[GoImportPathField].value, go_mod_info.stripped_digest), ) inferred_dependencies = [] @@ -214,16 +207,9 @@ async def generate_targets_from_go_mod( request.generator[GoModPackageSourcesField].path_globs(files_not_found_behavior), ), ) - all_module_info = await MultiGet( - Get( - ThirdPartyModuleInfo, - ThirdPartyModuleInfoRequest( - module_path=module_descriptor.path, - version=module_descriptor.version, - go_mod_stripped_digest=go_mod_info.stripped_digest, - ), - ) - for module_descriptor in go_mod_info.modules + all_third_party_packages = await Get( + AllThirdPartyPackages, + AllThirdPartyPackagesRequest(go_mod_info.stripped_digest), ) dir_to_filenames = group_by_dir(go_paths.files) @@ -251,11 +237,7 @@ def create_first_party_package_tgt(dir: str) -> GoFirstPartyPackageTarget: def create_third_party_package_tgt(pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyPackageTarget: return GoThirdPartyPackageTarget( - { - GoThirdPartyModulePathField.alias: pkg_info.module_path, - GoThirdPartyModuleVersionField.alias: pkg_info.version, - GoImportPathField.alias: pkg_info.import_path, - }, + {GoImportPathField.alias: pkg_info.import_path}, # E.g. `src/go:mod#github.com/google/uuid`. generator_addr.create_generated(pkg_info.import_path), union_membership, @@ -264,8 +246,7 @@ def create_third_party_package_tgt(pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyP third_party_pkgs = ( create_third_party_package_tgt(pkg_info) - for module_info in all_module_info - for pkg_info in module_info.values() + for pkg_info in all_third_party_packages.import_paths_to_pkg_info.values() ) return GeneratedTargets(request.generator, (*first_party_pkgs, *third_party_pkgs)) diff --git a/src/python/pants/backend/go/target_type_rules_test.py b/src/python/pants/backend/go/target_type_rules_test.py index e746e17534c..2c385631f51 100644 --- a/src/python/pants/backend/go/target_type_rules_test.py +++ b/src/python/pants/backend/go/target_type_rules_test.py @@ -24,8 +24,6 @@ GoFirstPartyPackageTarget, GoImportPathField, GoModTarget, - GoThirdPartyModulePathField, - GoThirdPartyModuleVersionField, GoThirdPartyPackageTarget, ) from pants.backend.go.util_rules import first_party_pkg, go_mod, sdk, third_party_pkg @@ -79,23 +77,21 @@ def test_go_package_dependency_inference(rule_runner: RuleRunner) -> None: module go.example.com/foo go 1.17 - require github.com/google/go-cmp v0.4.0 + require github.com/google/uuid v1.3.0 """ ), "foo/go.sum": dedent( """\ - github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= - github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - """ + github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= + github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= + """ ), "foo/pkg/foo.go": dedent( """\ package pkg - import "github.com/google/go-cmp/cmp" - func grok(left, right string) bool { - return cmp.Equal(left, right) + import "github.com/google/uuid" + func Grok() string { + return uuid.Foo() } """ ), @@ -126,7 +122,7 @@ def test_go_package_dependency_inference(rule_runner: RuleRunner) -> None: [InferGoPackageDependenciesRequest(tgt2[GoFirstPartyPackageSourcesField])], ) assert inferred_deps2.dependencies == FrozenOrderedSet( - [Address("foo", generated_name="github.com/google/go-cmp/cmp")] + [Address("foo", generated_name="github.com/google/uuid")] ) # Compilation failures should not blow up Pants. @@ -154,6 +150,7 @@ def test_generate_package_targets(rule_runner: RuleRunner) -> None: require ( github.com/google/go-cmp v0.4.0 github.com/google/uuid v1.2.0 + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect ) """ ), @@ -189,15 +186,9 @@ def gen_first_party_tgt(rel_dir: str, sources: list[str]) -> GoFirstPartyPackage residence_dir=os.path.join("src/go", rel_dir).rstrip("/"), ) - def gen_third_party_tgt( - mod_path: str, version: str, import_path: str - ) -> GoThirdPartyPackageTarget: + def gen_third_party_tgt(import_path: str) -> GoThirdPartyPackageTarget: return GoThirdPartyPackageTarget( - { - GoImportPathField.alias: import_path, - GoThirdPartyModulePathField.alias: mod_path, - GoThirdPartyModuleVersionField.alias: version, - }, + {GoImportPathField.alias: import_path}, Address("src/go", generated_name=import_path), ) @@ -207,44 +198,21 @@ def gen_third_party_tgt( gen_first_party_tgt("", ["hello.go"]), gen_first_party_tgt("subdir", ["subdir/f.go", "subdir/f2.go"]), gen_first_party_tgt("another_dir/subdir", ["another_dir/subdir/f.go"]), - gen_third_party_tgt("github.com/google/uuid", "v1.2.0", "github.com/google/uuid"), - gen_third_party_tgt( - "github.com/google/go-cmp", "v0.4.0", "github.com/google/go-cmp/cmp" - ), - gen_third_party_tgt( - "github.com/google/go-cmp", "v0.4.0", "github.com/google/go-cmp/cmp/cmpopts" - ), - gen_third_party_tgt( - "github.com/google/go-cmp", "v0.4.0", "github.com/google/go-cmp/cmp/internal/diff" - ), - gen_third_party_tgt( - "github.com/google/go-cmp", "v0.4.0", "github.com/google/go-cmp/cmp/internal/flags" - ), - gen_third_party_tgt( - "github.com/google/go-cmp", - "v0.4.0", - "github.com/google/go-cmp/cmp/internal/function", - ), - gen_third_party_tgt( - "github.com/google/go-cmp", - "v0.4.0", - "github.com/google/go-cmp/cmp/internal/testprotos", - ), - gen_third_party_tgt( - "github.com/google/go-cmp", - "v0.4.0", - "github.com/google/go-cmp/cmp/internal/teststructs", - ), - gen_third_party_tgt( - "github.com/google/go-cmp", "v0.4.0", "github.com/google/go-cmp/cmp/internal/value" - ), - gen_third_party_tgt( - "golang.org/x/xerrors", "v0.0.0-20191204190536-9bdfabe68543", "golang.org/x/xerrors" - ), - gen_third_party_tgt( - "golang.org/x/xerrors", - "v0.0.0-20191204190536-9bdfabe68543", - "golang.org/x/xerrors/internal", + *( + gen_third_party_tgt(pkg) + for pkg in ( + "github.com/google/uuid", + "github.com/google/go-cmp/cmp", + "github.com/google/go-cmp/cmp/cmpopts", + "github.com/google/go-cmp/cmp/internal/diff", + "github.com/google/go-cmp/cmp/internal/flags", + "github.com/google/go-cmp/cmp/internal/function", + "github.com/google/go-cmp/cmp/internal/testprotos", + "github.com/google/go-cmp/cmp/internal/teststructs", + "github.com/google/go-cmp/cmp/internal/value", + "golang.org/x/xerrors", + "golang.org/x/xerrors/internal", + ) ), }, ) @@ -261,11 +229,7 @@ def test_package_targets_cannot_be_manually_created() -> None: ) with pytest.raises(InvalidTargetException): GoThirdPartyPackageTarget( - { - GoImportPathField.alias: "foo", - GoThirdPartyModulePathField.alias: "foo", - GoThirdPartyModuleVersionField.alias: "foo", - }, + {GoImportPathField.alias: "foo"}, Address("foo"), ) diff --git a/src/python/pants/backend/go/target_types.py b/src/python/pants/backend/go/target_types.py index 8ebdf9eff62..95a15bf9542 100644 --- a/src/python/pants/backend/go/target_types.py +++ b/src/python/pants/backend/go/target_types.py @@ -188,36 +188,9 @@ class GoThirdPartyPackageDependenciesField(Dependencies): pass -class GoThirdPartyModulePathField(StringField): - alias = "module_path" - help = ( - "The module path of the third-party module this package comes from, " - "e.g. `github.com/google/go-cmp`.\n\n" - "This field should not be overridden; use the value from target generation." - ) - required = True - value: str - - -class GoThirdPartyModuleVersionField(StringField): - alias = "version" - help = ( - "The version of the third-party module this package comes from, e.g. `v0.4.0`.\n\n" - "This field should not be overridden; use the value from target generation." - ) - required = True - value: str - - class GoThirdPartyPackageTarget(Target): alias = "go_third_party_package" - core_fields = ( - *COMMON_TARGET_FIELDS, - GoThirdPartyPackageDependenciesField, - GoThirdPartyModulePathField, - GoThirdPartyModuleVersionField, - GoImportPathField, - ) + core_fields = (*COMMON_TARGET_FIELDS, GoThirdPartyPackageDependenciesField, GoImportPathField) help = ( "A package from a third-party Go module.\n\n" "You should not explicitly create this target in BUILD files. Instead, add a `go_mod` " diff --git a/src/python/pants/backend/go/util_rules/build_pkg.py b/src/python/pants/backend/go/util_rules/build_pkg.py index 4e1b19e39ae..00601079999 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg.py +++ b/src/python/pants/backend/go/util_rules/build_pkg.py @@ -9,8 +9,7 @@ GoFirstPartyPackageSourcesField, GoFirstPartyPackageSubpathField, GoImportPathField, - GoThirdPartyModulePathField, - GoThirdPartyModuleVersionField, + GoThirdPartyPackageDependenciesField, ) from pants.backend.go.util_rules.assembly import ( AssemblyPostCompilation, @@ -32,7 +31,6 @@ from pants.engine.process import FallibleProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import Dependencies, DependenciesRequest, UnexpandedTargets, WrappedTarget -from pants.util.dirutil import fast_relpath from pants.util.frozendict import FrozenDict from pants.util.strutil import path_safe @@ -262,22 +260,17 @@ async def setup_build_go_package_target_request( go_file_names += _first_party_pkg_info.test_files s_file_names = _first_party_pkg_info.s_files - elif target.has_field(GoThirdPartyModulePathField): - _module_path = target[GoThirdPartyModulePathField].value - subpath = fast_relpath(import_path, _module_path) - + elif target.has_field(GoThirdPartyPackageDependenciesField): _go_mod_address = target.address.maybe_convert_to_target_generator() _go_mod_info = await Get(GoModInfo, GoModInfoRequest(_go_mod_address)) _third_party_pkg_info = await Get( ThirdPartyPkgInfo, ThirdPartyPkgInfoRequest( - import_path=import_path, - module_path=_module_path, - version=target[GoThirdPartyModuleVersionField].value, - go_mod_stripped_digest=_go_mod_info.stripped_digest, + import_path=import_path, go_mod_stripped_digest=_go_mod_info.stripped_digest ), ) + subpath = _third_party_pkg_info.subpath digest = _third_party_pkg_info.digest go_file_names = _third_party_pkg_info.go_files s_file_names = _third_party_pkg_info.s_files @@ -297,7 +290,7 @@ async def setup_build_go_package_target_request( for tgt in all_deps if ( tgt.has_field(GoFirstPartyPackageSourcesField) - or tgt.has_field(GoThirdPartyModulePathField) + or tgt.has_field(GoThirdPartyPackageDependenciesField) ) ) direct_dependencies = [] diff --git a/src/python/pants/backend/go/util_rules/build_pkg_test.py b/src/python/pants/backend/go/util_rules/build_pkg_test.py index 935d41bd3ea..dd36415c381 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_test.py @@ -293,7 +293,7 @@ def test_build_third_party_pkg_target(rule_runner: RuleRunner) -> None: rule_runner, Address("", target_name="mod", generated_name=import_path), expected_import_path=import_path, - expected_subpath="", + expected_subpath="github.com/google/uuid@v1.3.0", expected_go_file_names=[ "dce.go", "doc.go", @@ -378,7 +378,7 @@ def test_build_target_with_dependencies(rule_runner: RuleRunner) -> None: rule_runner, Address("", target_name="mod", generated_name=xerrors_internal_import_path), expected_import_path=xerrors_internal_import_path, - expected_subpath="internal", + expected_subpath="golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/internal", expected_go_file_names=["internal.go"], expected_direct_dependency_import_paths=[], expected_transitive_dependency_import_paths=[], @@ -389,7 +389,7 @@ def test_build_target_with_dependencies(rule_runner: RuleRunner) -> None: rule_runner, Address("", target_name="mod", generated_name=xerrors_import_path), expected_import_path=xerrors_import_path, - expected_subpath="", + expected_subpath="golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543", expected_go_file_names=[ "adaptor.go", "doc.go", diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg.py b/src/python/pants/backend/go/util_rules/first_party_pkg.py index 22b82c20d4f..867bfca1114 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg.py @@ -15,6 +15,10 @@ ) from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest from pants.backend.go.util_rules.sdk import GoSdkProcess +from pants.backend.go.util_rules.third_party_pkg import ( + AllThirdPartyPackages, + AllThirdPartyPackagesRequest, +) from pants.build_graph.address import Address from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import Digest, MergeDigests @@ -80,11 +84,15 @@ async def compute_first_party_package_info( import_path = target[GoImportPathField].value subpath = target[GoFirstPartyPackageSubpathField].value - pkg_sources = await Get( - HydratedSources, HydrateSourcesRequest(target[GoFirstPartyPackageSourcesField]) + pkg_sources, all_third_party_packages = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(target[GoFirstPartyPackageSourcesField])), + Get(AllThirdPartyPackages, AllThirdPartyPackagesRequest(go_mod_info.stripped_digest)), ) input_digest = await Get( - Digest, MergeDigests([pkg_sources.snapshot.digest, go_mod_info.digest]) + Digest, + MergeDigests( + [pkg_sources.snapshot.digest, go_mod_info.digest, all_third_party_packages.digest] + ), ) result = await Get( @@ -93,7 +101,7 @@ async def compute_first_party_package_info( input_digest=input_digest, command=("list", "-json", f"./{subpath}"), description=f"Determine metadata for {request.address}", - working_dir=request.address.spec_path, + working_dir=request.address.spec_path, # i.e. the `go.mod`'s directory. ), ) if result.exit_code != 0: diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py index 195df14940a..7e06581f0d9 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py @@ -44,12 +44,33 @@ def test_package_info(rule_runner: RuleRunner) -> None: "foo/go.mod": dedent( """\ module go.example.com/foo - go 1.17 + go 1.16 + require github.com/google/uuid v1.3.0 + require ( + rsc.io/quote v1.5.2 + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c // indirect + rsc.io/sampler v1.3.0 // indirect + ) + """ + ), + "foo/go.sum": dedent( + """\ + github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= + github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= + rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= + rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= + rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= """ ), "foo/pkg/foo.go": dedent( """\ package pkg + import "github.com/google/uuid" + import "rsc.io/quote" + func Grok() string { return "Hello World" } @@ -107,7 +128,7 @@ def assert_info( assert_info( "pkg", - imports=[], + imports=["github.com/google/uuid", "rsc.io/quote"], test_imports=[], xtest_imports=[], go_files=["foo.go"], diff --git a/src/python/pants/backend/go/util_rules/go_mod.py b/src/python/pants/backend/go/util_rules/go_mod.py index b78c47305d1..ca39f3965a4 100644 --- a/src/python/pants/backend/go/util_rules/go_mod.py +++ b/src/python/pants/backend/go/util_rules/go_mod.py @@ -7,8 +7,6 @@ import os from dataclasses import dataclass -import ijson - from pants.backend.go.target_types import GoModSourcesField from pants.backend.go.util_rules.sdk import GoSdkProcess from pants.build_graph.address import Address @@ -17,25 +15,15 @@ from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import HydratedSources, HydrateSourcesRequest, WrappedTarget -from pants.util.ordered_set import FrozenOrderedSet logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class ModuleDescriptor: - path: str - version: str - - @dataclass(frozen=True) class GoModInfo: # Import path of the Go module, based on the `module` in `go.mod`. import_path: str - # Modules referenced by this go.mod with resolved versions. - modules: FrozenOrderedSet[ModuleDescriptor] - # Digest containing the full paths to `go.mod` and `go.sum`. digest: Digest @@ -51,31 +39,6 @@ def debug_hint(self) -> str: return self.address.spec -def parse_module_descriptors(raw_json: bytes) -> list[ModuleDescriptor]: - """Parse the JSON output of `go list -m`.""" - if not raw_json: - return [] - - module_descriptors = [] - for raw_module_descriptor in ijson.items(raw_json, "", multiple_values=True): - if raw_module_descriptor.get("Main", False): - continue - path = raw_module_descriptor["Path"] - if "Replace" in raw_module_descriptor: - if raw_module_descriptor["Replace"]["Path"] != raw_module_descriptor["Path"]: - raise NotImplementedError( - "Pants does not yet support replace directives that change the import path. " - "Please open an issue at https://github.com/pantsbuild/pants/issues/new/choose " - "with this error message so that we know to prioritize adding support:\n\n" - f"{raw_module_descriptor}" - ) - version = raw_module_descriptor["Replace"]["Version"] - else: - version = raw_module_descriptor["Version"] - module_descriptors.append(ModuleDescriptor(path, version)) - return module_descriptors - - @rule async def determine_go_mod_info( request: GoModInfoRequest, @@ -88,6 +51,7 @@ async def determine_go_mod_info( # Get the `go.mod` (and `go.sum`) and strip so the file has no directory prefix. hydrated_sources = await Get(HydratedSources, HydrateSourcesRequest(sources_field)) sources_digest = hydrated_sources.snapshot.digest + stripped_source_get = Get(Digest, RemovePrefix(sources_digest, go_mod_dir)) mod_json_get = Get( ProcessResult, @@ -98,26 +62,11 @@ async def determine_go_mod_info( description=f"Parse {go_mod_path}", ), ) - list_modules_get = Get( - ProcessResult, - GoSdkProcess( - command=("list", "-m", "-json", "all"), - input_digest=sources_digest, - working_dir=go_mod_dir, - description=f"List modules in {go_mod_path}", - ), - ) - - stripped_source_get = Get(Digest, RemovePrefix(sources_digest, go_mod_dir)) - mod_json, list_modules, stripped_sources = await MultiGet( - mod_json_get, list_modules_get, stripped_source_get - ) + mod_json, stripped_sources = await MultiGet(mod_json_get, stripped_source_get) module_metadata = json.loads(mod_json.stdout) - modules = parse_module_descriptors(list_modules.stdout) return GoModInfo( import_path=module_metadata["Module"]["Path"], - modules=FrozenOrderedSet(modules), digest=sources_digest, stripped_digest=stripped_sources, ) diff --git a/src/python/pants/backend/go/util_rules/go_mod_test.py b/src/python/pants/backend/go/util_rules/go_mod_test.py index 59e989dbe42..efe8ffcc848 100644 --- a/src/python/pants/backend/go/util_rules/go_mod_test.py +++ b/src/python/pants/backend/go/util_rules/go_mod_test.py @@ -9,7 +9,7 @@ from pants.backend.go.target_types import GoModTarget from pants.backend.go.util_rules import go_mod, sdk -from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest, ModuleDescriptor +from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest from pants.build_graph.address import Address from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @@ -30,72 +30,26 @@ def rule_runner() -> RuleRunner: def test_go_mod_info(rule_runner: RuleRunner) -> None: + go_mod_content = dedent( + """\ + module go.example.com/foo + go 1.17 + require github.com/golang/protobuf v1.4.2 + """ + ) + go_sum_content = "does not matter" rule_runner.write_files( - { - "foo/pkg/foo.go": "package pkg\n", - "foo/go.mod": dedent( - """\ - module go.example.com/foo - go 1.17 - require github.com/golang/protobuf v1.4.2 - """ - ), - "foo/go.sum": dedent( - """\ - github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= - github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= - github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= - github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= - github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= - github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= - github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= - github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= - github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= - github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= - github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= - google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= - google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= - google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= - google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= - google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= - """ - ), - "foo/main.go": "package main\nfunc main() { }\n", - "foo/BUILD": "go_mod()", - } + {"foo/go.mod": go_mod_content, "foo/go.sum": go_sum_content, "foo/BUILD": "go_mod()"} ) go_mod_info = rule_runner.request(GoModInfo, [GoModInfoRequest(Address("foo"))]) assert go_mod_info.import_path == "go.example.com/foo" - assert go_mod_info.modules - assert any( - module_descriptor.path == "github.com/golang/protobuf" - for module_descriptor in go_mod_info.modules + assert ( + go_mod_info.digest + == rule_runner.make_snapshot( + {"foo/go.mod": go_mod_content, "foo/go.sum": go_sum_content} + ).digest ) - - -def test_go_mod_replace_version(rule_runner: RuleRunner) -> None: - rule_runner.write_files( - { - "go.mod": dedent( - """\ - module go.example.com/foo - go 1.17 - require github.com/google/uuid v1.0.0 - - replace github.com/google/uuid => github.com/google/uuid v1.3.0 - """ - ), - "go.sum": dedent( - """\ - github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= - github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= - """ - ), - "BUILD": "go_mod(name='mod')", - } + assert ( + go_mod_info.stripped_digest + == rule_runner.make_snapshot({"go.mod": go_mod_content, "go.sum": go_sum_content}).digest ) - go_mod_info = rule_runner.request(GoModInfo, [GoModInfoRequest(Address("", target_name="mod"))]) - assert go_mod_info.import_path == "go.example.com/foo" - assert set(go_mod_info.modules) == {ModuleDescriptor("github.com/google/uuid", "v1.3.0")} diff --git a/src/python/pants/backend/go/util_rules/sdk.py b/src/python/pants/backend/go/util_rules/sdk.py index 82aad83857b..680b6aaa54b 100644 --- a/src/python/pants/backend/go/util_rules/sdk.py +++ b/src/python/pants/backend/go/util_rules/sdk.py @@ -96,6 +96,7 @@ async def setup_go_sdk_process( argv=[bash.path, go_sdk_run.script.path, *request.command], env={ GoSdkRunSetup.CHDIR_ENV: request.working_dir or "", + **request.env, }, input_digest=input_digest, description=request.description, diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index 3c3dc79b53f..92d017ff407 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -3,213 +3,39 @@ from __future__ import annotations -import os +import logging +import os.path from dataclasses import dataclass -from typing import Tuple import ijson from pants.backend.go.util_rules.sdk import GoSdkProcess from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( - CreateDigest, + AddPrefix, Digest, - DigestContents, - DigestEntries, DigestSubset, - FileEntry, GlobMatchErrorBehavior, - MergeDigests, PathGlobs, RemovePrefix, - Snapshot, ) from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.frozendict import FrozenDict -from pants.util.strutil import strip_v2_chroot_path +from pants.util.strutil import strip_prefix, strip_v2_chroot_path -# ----------------------------------------------------------------------------------------------- -# Download modules -# ----------------------------------------------------------------------------------------------- - - -class _AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): - """A mapping of each downloaded (module, version) to its digest. - - Each digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and - `go.sum` for the particular module. This means that you can operate on the module (e.g. `go - list`) directly, without needing to set the working_dir etc. - """ - - -@dataclass(frozen=True) -class _AllDownloadedModulesRequest: - """Download all modules from the `go.mod`. - - The `go.mod` and `go.sum` must already be up-to-date. - """ - - go_mod_stripped_digest: Digest - - -@dataclass(frozen=True) -class _DownloadedModule: - """A downloaded module's directory. - - The digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and - `go.sum` for the particular module. This means that you can operate on the module (e.g. `go - list`) directly, without needing to set the working_dir etc. - """ - - digest: Digest - - -@dataclass(frozen=True) -class _DownloadedModuleRequest(EngineAwareParameter): - module_path: str - version: str - go_mod_stripped_digest: Digest - - def debug_hint(self) -> str: - return f"{self.module_path}@{self.version}" - - -@rule -async def download_third_party_modules( - request: _AllDownloadedModulesRequest, -) -> _AllDownloadedModules: - # TODO: Clean this up. - input_digest_entries = await Get(DigestEntries, Digest, request.go_mod_stripped_digest) - assert len(input_digest_entries) == 2 - go_sum_file_digest = next( - file_entry.file_digest - for file_entry in input_digest_entries - if isinstance(file_entry, FileEntry) and file_entry.path == "go.sum" - ) - - download_result = await Get( - ProcessResult, - GoSdkProcess( - command=("mod", "download", "-json", "all"), - input_digest=request.go_mod_stripped_digest, - # TODO: make this more descriptive: point to the actual `go_mod` target or path. - description="Download all third-party Go modules", - output_files=("go.mod", "go.sum"), - output_directories=("gopath",), - allow_downloads=True, - ), - ) - - # Check that the root `go.mod` and `go.sum` did not change. - result_go_mod_digest = await Get( - Digest, DigestSubset(download_result.output_digest, PathGlobs(["go.mod", "go.sum"])) - ) - if result_go_mod_digest != request.go_mod_stripped_digest: - # TODO: make this a more informative error. - contents = await Get(DigestContents, Digest, result_go_mod_digest) - - raise Exception( - "`go.mod` and/or `go.sum` changed! Please run `go mod tidy`.\n\n" - f"{contents[0].content.decode()}\n\n" - f"{contents[1].content.decode()}\n\n" - ) - - download_snapshot = await Get(Snapshot, Digest, download_result.output_digest) - all_downloaded_files = set(download_snapshot.files) - - # To analyze each module via `go list`, we need its own `go.mod`, along with a `go.sum` that - # includes it and its deps: - # - # * If the module does not already have `go.mod`, Go will have generated it. - # * Our `go.sum` should be a superset of each module, so we can simply use that. Note that we - # eagerly error if the `go.sum` changed during the download, so we can be confident - # that the on-disk `go.sum` is comprehensive. TODO(#13093): subset this somehow? - module_paths_and_versions_to_dirs = {} - missing_go_sums = [] - generated_go_mods_to_module_dirs = {} - for module_metadata in ijson.items(download_result.stdout, "", multiple_values=True): - download_dir = strip_v2_chroot_path(module_metadata["Dir"]) - module_paths_and_versions_to_dirs[ - (module_metadata["Path"], module_metadata["Version"]) - ] = download_dir - _go_sum = os.path.join(download_dir, "go.sum") - if _go_sum not in all_downloaded_files: - missing_go_sums.append(FileEntry(_go_sum, go_sum_file_digest)) - if os.path.join(download_dir, "go.mod") not in all_downloaded_files: - generated_go_mod = strip_v2_chroot_path(module_metadata["GoMod"]) - generated_go_mods_to_module_dirs[generated_go_mod] = download_dir - - digest_entries = await Get(DigestEntries, Digest, download_result.output_digest) - go_mod_requests = [] - for entry in digest_entries: - if isinstance(entry, FileEntry) and entry.path in generated_go_mods_to_module_dirs: - module_dir = generated_go_mods_to_module_dirs[entry.path] - go_mod_requests.append(FileEntry(os.path.join(module_dir, "go.mod"), entry.file_digest)) - - generated_digest = await Get(Digest, CreateDigest([*missing_go_sums, *go_mod_requests])) - full_digest = await Get(Digest, MergeDigests([download_result.output_digest, generated_digest])) - - subsets = await MultiGet( - Get( - Digest, - DigestSubset( - full_digest, - PathGlobs( - [f"{module_dir}/**"], - glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin=f"downloading {module}@{version}", - ), - ), - ) - for (module, version), module_dir in module_paths_and_versions_to_dirs.items() - ) - stripped_subsets = await MultiGet( - Get(Digest, RemovePrefix(digest, module_dir)) - for digest, module_dir in zip(subsets, module_paths_and_versions_to_dirs.values()) - ) - module_paths_and_versions_to_digests = { - mod_and_version: digest - for mod_and_version, digest in zip( - module_paths_and_versions_to_dirs.keys(), stripped_subsets - ) - } - return _AllDownloadedModules(module_paths_and_versions_to_digests) - - -@rule -async def extract_module_from_downloaded_modules( - request: _DownloadedModuleRequest, -) -> _DownloadedModule: - all_modules = await Get( - _AllDownloadedModules, _AllDownloadedModulesRequest(request.go_mod_stripped_digest) - ) - digest = all_modules.get((request.module_path, request.version)) - if digest is None: - raise AssertionError( - f"The module {request.module_path}@{request.version} was not downloaded. This should " - "not happen: please open an issue at " - "https://github.com/pantsbuild/pants/issues/new/choose with this error message.\n\n" - f"{all_modules}" - ) - return _DownloadedModule(digest) - - -# ----------------------------------------------------------------------------------------------- -# Determine package info -# ----------------------------------------------------------------------------------------------- +logger = logging.getLogger(__name__) @dataclass(frozen=True) class ThirdPartyPkgInfo: - """All the info needed to build a third-party package. + """All the info and files needed to build a third-party package. - The digest is stripped of the `gopath` prefix. + The digest only contains the files for the package, with all prefixes stripped. """ import_path: str - module_path: str - version: str + subpath: str digest: Digest @@ -230,97 +56,165 @@ class ThirdPartyPkgInfoRequest(EngineAwareParameter): """ import_path: str - module_path: str - version: str go_mod_stripped_digest: Digest def debug_hint(self) -> str: return self.import_path -class ThirdPartyModuleInfo(FrozenDict[str, ThirdPartyPkgInfo]): - """A mapping of the import path for each package in the module to its - `ThirdPartyPackageInfo`.""" - - @dataclass(frozen=True) -class ThirdPartyModuleInfoRequest(EngineAwareParameter): - """Request info for every package contained in a third-party module. +class AllThirdPartyPackages(FrozenDict[str, ThirdPartyPkgInfo]): + """All the packages downloaded from a go.mod, along with a digest of the downloaded files. - The module must be included in the input `go.mod`/`go.sum`. + The digest has files in the format `gopath/pkg/mod`, which is what `GoSdkProcess` sets `GOPATH` + to. This means that you can include the digest in a process and Go will properly consume it as + the `GOPATH`. """ - module_path: str - version: str - go_mod_stripped_digest: Digest + digest: Digest + import_paths_to_pkg_info: FrozenDict[str, ThirdPartyPkgInfo] - def debug_hint(self) -> str: - return f"{self.module_path}@{self.version}" + +@dataclass(frozen=True) +class AllThirdPartyPackagesRequest: + go_mod_stripped_digest: Digest @rule -async def compute_third_party_module_metadata( - request: ThirdPartyModuleInfoRequest, -) -> ThirdPartyModuleInfo: - downloaded_module = await Get( - _DownloadedModule, - _DownloadedModuleRequest( - request.module_path, request.version, request.go_mod_stripped_digest - ), +async def download_and_analyze_third_party_packages( + request: AllThirdPartyPackagesRequest, +) -> AllThirdPartyPackages: + # NB: We download all modules to GOPATH=$(pwd)/gopath. Running `go list ...` from $(pwd) would + # naively try analyzing the contents of the GOPATH like they were first-party packages. This + # results in errors like this: + # + # package /gopath/pkg/mod/golang.org/x/text@v0.3.0/unicode: can only use + # path@version syntax with 'go get' and 'go install' in module-aware mode + # + # Instead, we run `go list` from a subdirectory of the chroot. It can still access the + # contents of `GOPATH`, but won't incorrectly treat its contents as first-party packages. + go_mod_prefix = "go_mod_prefix" + go_mod_prefixed_digest = await Get( + Digest, AddPrefix(request.go_mod_stripped_digest, go_mod_prefix) ) - json_result = await Get( + + list_argv = ( + "list", + # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace. + # Instead, we expect them to be well-formed already. + # + # It would be convenient to set `-mod=mod` to allow edits, and then compare the resulting + # files to the input so that we could print a diff for the user to know how to update. But + # `-mod=mod` results in more packages being downloaded and added to `go.mod` than is + # actually necessary. + # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a + # message from Go and won't be intuitive for Pants users what to do. + "-mod=readonly", + # There may be some packages in the transitive closure that cannot be built, but we should + # not blow up Pants. + # + # For example, a package that sets the special value `package documentation` and has no + # source files would naively error due to `build constraints exclude all Go files`, even + # though we should not error on that package. + "-e", + "-json", + # This matches all packages. `all` only matches first-party packages and complains that + # there are no `.go` files. + "...", + ) + list_result = await Get( ProcessResult, GoSdkProcess( - input_digest=downloaded_module.digest, - command=("list", "-mod=readonly", "-json", "./..."), - description=( - "Determine metadata for Go third-party module " - f"{request.module_path}@{request.version}" - ), + command=list_argv, + # TODO: make this more descriptive: point to the actual `go_mod` target or path. + description="Download and analyze all third-party Go packages", + input_digest=go_mod_prefixed_digest, + output_directories=("gopath/pkg/mod",), + working_dir=go_mod_prefix, + allow_downloads=True, ), ) + stripped_result_digest = await Get( + Digest, RemovePrefix(list_result.output_digest, "gopath/pkg/mod") + ) - # Some modules don't have any Go code in them, meaning they have no packages. - if not json_result.stdout: - return ThirdPartyModuleInfo() - - import_path_to_info = {} - for metadata in ijson.items(json_result.stdout, "", multiple_values=True): - import_path = metadata["ImportPath"] - pkg_info = ThirdPartyPkgInfo( - import_path=import_path, - module_path=request.module_path, - version=request.version, - digest=downloaded_module.digest, - imports=tuple(metadata.get("Imports", ())), - go_files=tuple(metadata.get("GoFiles", ())), - s_files=tuple(metadata.get("SFiles", ())), - unsupported_sources_error=maybe_create_error_for_invalid_sources( - metadata, import_path, request.module_path, request.version - ), + all_digest_subset_gets = [] + all_pkg_info_kwargs = [] + for pkg_json in ijson.items(list_result.stdout, "", multiple_values=True): + if "Standard" in pkg_json: + continue + import_path = pkg_json["ImportPath"] + if import_path == "...": + if "Error" not in pkg_json: + raise AssertionError( + "`go list` included the import path `...`, but there was no `Error` attached. " + "Please open an issue at https://github.com/pantsbuild/pants/issues/new/choose " + f"with this error message:\n\n{pkg_json}" + ) + # TODO: Improve this error message, such as better instructions if `go.sum` is stale. + raise Exception(pkg_json["Error"]["Err"]) + + if "Error" in pkg_json: + err = pkg_json["Error"]["Err"] + if "build constraints exclude all Go files" in err: + logger.debug( + f"Skipping the Go third-party package `{import_path}` because of this " + f"error: {err}" + ) + continue + + raise AssertionError( + f"`go list` failed for the import path `{import_path}`. Please open an issue at " + f"https://github.com/pantsbuild/pants/issues/new/choose so that we can figure out " + f"how to support this:\n\n{err}" + ) + + dir_path = strip_prefix(strip_v2_chroot_path(pkg_json["Dir"]), "gopath/pkg/mod/") + all_pkg_info_kwargs.append( + dict( + import_path=import_path, + subpath=dir_path, + imports=tuple(pkg_json.get("Imports", ())), + go_files=tuple(pkg_json.get("GoFiles", ())), + s_files=tuple(pkg_json.get("SFiles", ())), + unsupported_sources_error=maybe_create_error_for_invalid_sources( + pkg_json, import_path + ), + ) + ) + all_digest_subset_gets.append( + Get( + Digest, + DigestSubset( + stripped_result_digest, + PathGlobs( + [os.path.join(dir_path, "*")], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=f"downloading {import_path}", + ), + ), + ) ) - import_path_to_info[import_path] = pkg_info - return ThirdPartyModuleInfo(import_path_to_info) + + all_digest_subsets = await MultiGet(all_digest_subset_gets) + import_path_to_info = { + pkg_info_kwargs["import_path"]: ThirdPartyPkgInfo(digest=digest_subset, **pkg_info_kwargs) + for pkg_info_kwargs, digest_subset in zip(all_pkg_info_kwargs, all_digest_subsets) + } + return AllThirdPartyPackages(list_result.output_digest, FrozenDict(import_path_to_info)) @rule -async def extract_package_info_from_module_info( - request: ThirdPartyPkgInfoRequest, -) -> ThirdPartyPkgInfo: - module_info = await Get( - ThirdPartyModuleInfo, - ThirdPartyModuleInfoRequest( - request.module_path, request.version, request.go_mod_stripped_digest - ), +async def extract_package_info(request: ThirdPartyPkgInfoRequest) -> ThirdPartyPkgInfo: + all_packages = await Get( + AllThirdPartyPackages, AllThirdPartyPackagesRequest(request.go_mod_stripped_digest) ) - pkg_info = module_info.get(request.import_path) + pkg_info = all_packages.import_paths_to_pkg_info.get(request.import_path) if pkg_info is None: raise AssertionError( - f"The package {request.import_path} does not belong to the module " - f"{request.module_path}@{request.version}. This should not happen: please open an " - "issue at https://github.com/pantsbuild/pants/issues/new/choose with this error " - "message.\n\n" - f"{module_info}" + f"The package `{request.import_path}` was not downloaded, but Pants tried using it. " + "This should not happen. Please open an issue at " + "https://github.com/pantsbuild/pants/issues/new/choose with this error message." ) # We error if trying to _use_ a package with unsupported sources (vs. only generating the @@ -332,7 +226,7 @@ async def extract_package_info_from_module_info( def maybe_create_error_for_invalid_sources( - go_list_json: dict, import_path: str, module_path: str, version: str + go_list_json: dict, import_path: str ) -> NotImplementedError | None: for key in ( "CgoFiles", @@ -351,10 +245,8 @@ def maybe_create_error_for_invalid_sources( f"The third-party package {import_path} includes `{key}`, which Pants does " "not yet support. Please open a feature request at " "https://github.com/pantsbuild/pants/issues/new/choose so that we know to " - "prioritize adding support. Please include this metadata:\n\n" - f"package: {import_path}\n" - f"module: {module_path}\n" - f"version: {version}" + "prioritize adding support. Please include this error message and the version of " + "the third-party module." ) return None diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py index 3a4102f4a2a..7527eb4e7f7 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os.path from textwrap import dedent import pytest @@ -10,16 +11,12 @@ from pants.backend.go.target_types import GoModTarget from pants.backend.go.util_rules import sdk, third_party_pkg from pants.backend.go.util_rules.third_party_pkg import ( - ThirdPartyModuleInfo, - ThirdPartyModuleInfoRequest, + AllThirdPartyPackages, + AllThirdPartyPackagesRequest, ThirdPartyPkgInfo, ThirdPartyPkgInfoRequest, - _AllDownloadedModules, - _AllDownloadedModulesRequest, - _DownloadedModule, - _DownloadedModuleRequest, ) -from pants.engine.fs import Digest, PathGlobs, Snapshot +from pants.engine.fs import Digest, Snapshot from pants.engine.process import ProcessExecutionFailure from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner, engine_error @@ -31,9 +28,7 @@ def rule_runner() -> RuleRunner: rules=[ *sdk.rules(), *third_party_pkg.rules(), - QueryRule(_AllDownloadedModules, [_AllDownloadedModulesRequest]), - QueryRule(_DownloadedModule, [_DownloadedModuleRequest]), - QueryRule(ThirdPartyModuleInfo, [ThirdPartyModuleInfoRequest]), + QueryRule(AllThirdPartyPackages, [AllThirdPartyPackagesRequest]), QueryRule(ThirdPartyPkgInfo, [ThirdPartyPkgInfoRequest]), ], target_types=[GoModTarget], @@ -45,22 +40,9 @@ def rule_runner() -> RuleRunner: GO_MOD = dedent( """\ module example.com/third-party-module - go 1.17 + go 1.16 - // No dependencies, already has `go.mod`. require github.com/google/uuid v1.3.0 - - // No dependencies, but missing `go.mod`. - require cloud.google.com/go v0.26.0 - - // Has dependencies, already has a `go.sum`. - require ( - github.com/google/go-cmp v0.5.6 - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect - ) - - // Has dependencies, missing `go.sum`. This causes `go list` to fail in that directory unless - // we add `go.sum`. require ( rsc.io/quote v1.5.2 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c // indirect @@ -71,16 +53,10 @@ def rule_runner() -> RuleRunner: GO_SUM = dedent( """\ - cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= - cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= - github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= - github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= @@ -89,369 +65,466 @@ def rule_runner() -> RuleRunner: ) -# ----------------------------------------------------------------------------------------------- -# Download modules -# ----------------------------------------------------------------------------------------------- +def set_up_go_mod(rule_runner: RuleRunner, go_mod: str, go_sum: str) -> Digest: + return rule_runner.make_snapshot({"go.mod": go_mod, "go.sum": go_sum}).digest -def test_download_modules(rule_runner: RuleRunner) -> None: +def test_download_and_analyze_all_packages(rule_runner: RuleRunner) -> None: input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest - downloaded_modules = rule_runner.request( - _AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)] + all_packages = rule_runner.request( + AllThirdPartyPackages, [AllThirdPartyPackagesRequest(input_digest)] ) - assert len(downloaded_modules) == 7 - - def assert_module(module: str, version: str, sample_file: str) -> None: - assert (module, version) in downloaded_modules - digest = downloaded_modules[(module, version)] - snapshot = rule_runner.request(Snapshot, [digest]) - assert "go.mod" in snapshot.files - assert "go.sum" in snapshot.files - assert sample_file in snapshot.files - - extracted_module = rule_runner.request( - _DownloadedModule, [_DownloadedModuleRequest(module, version, input_digest)] - ) - extracted_snapshot = rule_runner.request(Snapshot, [extracted_module.digest]) - assert extracted_snapshot == snapshot - - assert_module("cloud.google.com/go", "v0.26.0", "bigtable/filter.go") - assert_module("github.com/google/uuid", "v1.3.0", "uuid.go") - assert_module("github.com/google/go-cmp", "v0.5.6", "cmp/cmpopts/errors_go113.go") - assert_module("golang.org/x/text", "v0.0.0-20170915032832-14c0d48ead0c", "width/transform.go") - assert_module("golang.org/x/xerrors", "v0.0.0-20191204190536-9bdfabe68543", "wrap.go") - assert_module("rsc.io/quote", "v1.5.2", "quote.go") - assert_module("rsc.io/sampler", "v1.3.0", "sampler.go") - - -def test_download_modules_missing_module(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest - with engine_error( - AssertionError, contains="The module some_project.org/project@v1.1 was not downloaded" - ): - rule_runner.request( - _DownloadedModule, - [_DownloadedModuleRequest("some_project.org/project", "v1.1", input_digest)], - ) - - -def test_download_modules_invalid_go_sum(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot( - { - "go.mod": dedent( - """\ - module example.com/third-party-module - go 1.17 - require github.com/google/uuid v1.3.0 - """ - ), - "go.sum": dedent( - """\ - github.com/google/uuid v1.3.0 h1:00000gmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= - github.com/google/uuid v1.3.0/go.mod h1:00000e4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= - """ - ), - } - ).digest - with engine_error(ProcessExecutionFailure, contains="SECURITY ERROR"): - rule_runner.request(_AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)]) - - -def test_download_modules_missing_go_sum(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot( - { - "go.mod": dedent( - """\ - module example.com/third-party-module - go 1.17 - require github.com/google/uuid v1.3.0 - """ - ), - # `go.sum` is for a different module. - "go.sum": dedent( - """\ - cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= - cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= - """ - ), + assert set(all_packages.import_paths_to_pkg_info.keys()) == { + "golang.org/x/text/encoding/japanese", + "golang.org/x/text/message/catalog", + "golang.org/x/text/internal/testtext", + "golang.org/x/text/encoding/ianaindex", + "golang.org/x/text/cmd/gotext", + "golang.org/x/text/width", + "golang.org/x/text/internal/format", + "rsc.io/sampler", + "golang.org/x/text/internal/tag", + "golang.org/x/text/unicode/norm", + "golang.org/x/text/number", + "golang.org/x/text/transform", + "golang.org/x/text/internal", + "golang.org/x/text/internal/utf8internal", + "golang.org/x/text/language/display", + "golang.org/x/text/internal/stringset", + "golang.org/x/text/encoding/korean", + "golang.org/x/text/internal/triegen", + "golang.org/x/text/secure/bidirule", + "golang.org/x/text/secure/precis", + "golang.org/x/text/language", + "golang.org/x/text/encoding/unicode/utf32", + "golang.org/x/text/internal/colltab", + "golang.org/x/text/unicode/rangetable", + "golang.org/x/text/encoding/htmlindex", + "golang.org/x/text/internal/export/idna", + "golang.org/x/text/encoding/charmap", + "golang.org/x/text/unicode/cldr", + "golang.org/x/text/secure", + "golang.org/x/text/internal/ucd", + "golang.org/x/text/feature/plural", + "golang.org/x/text/unicode", + "golang.org/x/text/encoding/traditionalchinese", + "golang.org/x/text/runes", + "golang.org/x/text/internal/catmsg", + "rsc.io/quote/buggy", + "golang.org/x/text/encoding/simplifiedchinese", + "golang.org/x/text/cases", + "golang.org/x/text/encoding/internal", + "github.com/google/uuid", + "golang.org/x/text/encoding/internal/enctest", + "golang.org/x/text/collate/build", + "golang.org/x/text", + "golang.org/x/text/unicode/bidi", + "golang.org/x/text/search", + "golang.org/x/text/unicode/runenames", + "golang.org/x/text/message", + "golang.org/x/text/encoding", + "golang.org/x/text/encoding/unicode", + "rsc.io/quote", + "golang.org/x/text/currency", + "golang.org/x/text/internal/number", + "golang.org/x/text/collate/tools/colcmp", + "golang.org/x/text/encoding/internal/identifier", + "golang.org/x/text/collate", + "golang.org/x/text/internal/gen", + } + + def assert_pkg_info( + import_path: str, + subpath: str, + imports: tuple[str, ...], + go_files: tuple[str, ...], + extra_files: tuple[str, ...], + ) -> None: + assert import_path in all_packages.import_paths_to_pkg_info + pkg_info = all_packages.import_paths_to_pkg_info[import_path] + assert pkg_info.import_path == import_path + assert pkg_info.subpath == subpath + assert pkg_info.imports == imports + assert pkg_info.go_files == go_files + assert not pkg_info.s_files + snapshot = rule_runner.request(Snapshot, [pkg_info.digest]) + assert set(snapshot.files) == { + os.path.join(subpath, file_name) for file_name in (*go_files, *extra_files) } - ).digest - with engine_error(contains="`go.mod` and/or `go.sum` changed!"): - rule_runner.request(_AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)]) - - -# ----------------------------------------------------------------------------------------------- -# Determine package info -# ----------------------------------------------------------------------------------------------- - -def test_determine_pkg_info(rule_runner: RuleRunner) -> None: - rule_runner.write_files({"go.mod": GO_MOD, "go.sum": GO_SUM, "BUILD": "go_mod(name='mod')"}) - input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])]) - - def assert_module( - module: str, - version: str, - expected: list[str] | dict[str, ThirdPartyPkgInfo], - *, - check_subset: bool = False, - skip_checking_pkg_info: bool = False, - ) -> None: - module_info = rule_runner.request( - ThirdPartyModuleInfo, [ThirdPartyModuleInfoRequest(module, version, input_digest)] - ) - # If `check_subset`, check that the expected import_paths are included. - if check_subset: - assert isinstance(expected, list) - assert set(expected).issubset(module_info.keys()) - else: - # If expected is a dict, check that the ThirdPartyPkgInfo is correct for each package. - if isinstance(expected, dict): - assert dict(module_info) == expected - # Else, only check that the import paths are present. - else: - assert list(module_info.keys()) == expected - - # Check our subsetting logic. - if not skip_checking_pkg_info: - for pkg_info in module_info.values(): - extracted_pkg = rule_runner.request( - ThirdPartyPkgInfo, - [ThirdPartyPkgInfoRequest(pkg_info.import_path, module, version, input_digest)], - ) - assert extracted_pkg == pkg_info - - assert_module( - "cloud.google.com/go", - "v0.26.0", - ["cloud.google.com/go/bigquery", "cloud.google.com/go/firestore"], - check_subset=True, + assert_pkg_info( + import_path="github.com/google/uuid", + subpath="github.com/google/uuid@v1.3.0", + imports=( + "bytes", + "crypto/md5", + "crypto/rand", + "crypto/sha1", + "database/sql/driver", + "encoding/binary", + "encoding/hex", + "encoding/json", + "errors", + "fmt", + "hash", + "io", + "net", + "os", + "strings", + "sync", + "time", + ), + go_files=( + "dce.go", + "doc.go", + "hash.go", + "marshal.go", + "node.go", + "node_net.go", + "null.go", + "sql.go", + "time.go", + "util.go", + "uuid.go", + "version1.go", + "version4.go", + ), + extra_files=( + ".travis.yml", + "CONTRIBUTING.md", + "CONTRIBUTORS", + "LICENSE", + "README.md", + "go.mod", + "json_test.go", + "node_js.go", + "null_test.go", + "seq_test.go", + "sql_test.go", + "uuid_test.go", + ), ) - - uuid_mod = "github.com/google/uuid" - uuid_version = "v1.3.0" - uuid_digest = rule_runner.request( - _DownloadedModule, [_DownloadedModuleRequest(uuid_mod, uuid_version, input_digest)] - ).digest - assert_module( - uuid_mod, - uuid_version, - { - uuid_mod: ThirdPartyPkgInfo( - import_path=uuid_mod, - module_path=uuid_mod, - version=uuid_version, - digest=uuid_digest, - imports=( - "bytes", - "crypto/md5", - "crypto/rand", - "crypto/sha1", - "database/sql/driver", - "encoding/binary", - "encoding/hex", - "encoding/json", - "errors", - "fmt", - "hash", - "io", - "net", - "os", - "strings", - "sync", - "time", - ), - go_files=( - "dce.go", - "doc.go", - "hash.go", - "marshal.go", - "node.go", - "node_net.go", - "null.go", - "sql.go", - "time.go", - "util.go", - "uuid.go", - "version1.go", - "version4.go", - ), - s_files=(), - ) - }, + assert_pkg_info( + import_path="golang.org/x/text/unicode/bidi", + subpath="golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c/unicode/bidi", + imports=("container/list", "fmt", "log", "sort", "unicode/utf8"), + go_files=("bidi.go", "bracket.go", "core.go", "prop.go", "tables.go", "trieval.go"), + extra_files=( + "core_test.go", + "gen.go", + "gen_ranges.go", + "gen_trieval.go", + "ranges_test.go", + "tables_test.go", + ), ) - assert_module( - "github.com/google/go-cmp", - "v0.5.6", - [ - "github.com/google/go-cmp/cmp", - "github.com/google/go-cmp/cmp/cmpopts", - "github.com/google/go-cmp/cmp/internal/diff", - "github.com/google/go-cmp/cmp/internal/flags", - "github.com/google/go-cmp/cmp/internal/function", - "github.com/google/go-cmp/cmp/internal/testprotos", - "github.com/google/go-cmp/cmp/internal/teststructs", - "github.com/google/go-cmp/cmp/internal/teststructs/foo1", - "github.com/google/go-cmp/cmp/internal/teststructs/foo2", - "github.com/google/go-cmp/cmp/internal/value", - ], + +def test_invalid_go_sum(rule_runner: RuleRunner) -> None: + digest = set_up_go_mod( + rule_runner, + dedent( + """\ + module example.com/third-party-module + go 1.17 + require github.com/google/uuid v1.3.0 + """ + ), + dedent( + """\ + github.com/google/uuid v1.3.0 h1:00000gmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= + github.com/google/uuid v1.3.0/go.mod h1:00000e4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= + """ + ), ) - assert_module( - "golang.org/x/text", - "v0.0.0-20170915032832-14c0d48ead0c", - ["golang.org/x/text/cmd/gotext", "golang.org/x/text/collate"], - check_subset=True, - skip_checking_pkg_info=True, # Contains unsupported `.cgo` files. + with engine_error(ProcessExecutionFailure, contains="SECURITY ERROR"): + rule_runner.request(AllThirdPartyPackages, [AllThirdPartyPackagesRequest(digest)]) + + +def test_missing_go_sum(rule_runner: RuleRunner) -> None: + digest = set_up_go_mod( + rule_runner, + dedent( + """\ + module example.com/third-party-module + go 1.17 + require github.com/google/uuid v1.3.0 + """ + ), + # `go.sum` is for a different module. + dedent( + """\ + cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= + cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= + """ + ), ) - assert_module( - "golang.org/x/xerrors", - "v0.0.0-20191204190536-9bdfabe68543", - ["golang.org/x/xerrors", "golang.org/x/xerrors/internal"], + with engine_error(contains="github.com/google/uuid@v1.3.0: missing go.sum entry"): + rule_runner.request(AllThirdPartyPackages, [AllThirdPartyPackagesRequest(digest)]) + + +def test_stale_go_mod(rule_runner: RuleRunner) -> None: + digest = set_up_go_mod( + rule_runner, + # Go 1.17+ expects indirect dependencies to be included in the `go.mod`, i.e. + # `golang.org/x/xerrors `. + dedent( + """\ + module example.com/third-party-module + go 1.17 + require github.com/google/go-cmp v0.5.6 + """ + ), + dedent( + """\ + github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= + github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + """ + ), ) - - assert_module("rsc.io/quote", "v1.5.2", ["rsc.io/quote", "rsc.io/quote/buggy"]) - assert_module("rsc.io/sampler", "v1.3.0", ["rsc.io/sampler"]) + with engine_error(ProcessExecutionFailure, contains="updates to go.mod needed"): + rule_runner.request(AllThirdPartyPackages, [AllThirdPartyPackagesRequest(digest)]) -def test_determine_pkg_info_missing(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest +def test_pkg_missing(rule_runner: RuleRunner) -> None: + digest = set_up_go_mod(rule_runner, GO_MOD, GO_SUM) with engine_error( - AssertionError, - contains=( - "The package another_project.org/foo does not belong to the module " - "github.com/google/uuid@v1.3.0" - ), + AssertionError, contains="The package `another_project.org/foo` was not downloaded" ): rule_runner.request( - ThirdPartyPkgInfo, - [ - ThirdPartyPkgInfoRequest( - "another_project.org/foo", "github.com/google/uuid", "v1.3.0", input_digest - ) - ], + ThirdPartyPkgInfo, [ThirdPartyPkgInfoRequest("another_project.org/foo", digest)] ) -def test_determine_pkg_info_no_packages(rule_runner) -> None: - rule_runner.write_files( - { - "go.mod": dedent( - """\ - module example.com/third-party-module - go 1.17 - - require github.com/Azure/go-autorest v13.3.2+incompatible - """ - ), - "go.sum": dedent( - """\ - github.com/Azure/go-autorest v13.3.2+incompatible h1:VxzPyuhtnlBOzc4IWCZHqpyH2d+QMLQEuy3wREyY4oc= - github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= - """ - ), - "BUILD": "go_mod(name='mod')", - } +def test_module_with_no_packages(rule_runner) -> None: + digest = set_up_go_mod( + rule_runner, + dedent( + """\ + module example.com/third-party-module + go 1.17 + require github.com/Azure/go-autorest v13.3.2+incompatible + """ + ), + dedent( + """\ + github.com/Azure/go-autorest v13.3.2+incompatible h1:VxzPyuhtnlBOzc4IWCZHqpyH2d+QMLQEuy3wREyY4oc= + github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= + """ + ), ) - input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])]) - module_info = rule_runner.request( - ThirdPartyModuleInfo, - [ - ThirdPartyModuleInfoRequest( - "github.com/Azure/go-autorest", "v13.3.2+incompatible", input_digest - ) - ], + all_packages = rule_runner.request( + AllThirdPartyPackages, [AllThirdPartyPackagesRequest(digest)] ) - assert not module_info + assert not all_packages.import_paths_to_pkg_info def test_unsupported_sources(rule_runner: RuleRunner) -> None: # `golang.org/x/mobile/bind/objc` uses `.h` files on both Linux and macOS. - mobile_version = "v0.0.0-20210924032853-1c027f395ef7" - input_digest = rule_runner.make_snapshot( - { - "go.mod": dedent( - f"""\ - module example.com/unsupported - go 1.17 - require golang.org/x/mobile {mobile_version} - """ - ), - "go.sum": dedent( - """\ - github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= - github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= - github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= - github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= - golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= - golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= - golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 h1:estk1glOnSVeJ9tdEZZc5mAMDZk5lNJNyJ6DvrBkTEU= - golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= - golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= - golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= - golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= - golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= - golang.org/x/mobile v0.0.0-20210924032853-1c027f395ef7 h1:CyFUjc175y/mbMjxe+WdqI72jguLyjQChKCDe9mfTvg= - golang.org/x/mobile v0.0.0-20210924032853-1c027f395ef7/go.mod h1:c4YKU3ZylDmvbw+H/PSvm42vhdWbuxCzbonauEAP9B8= - golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= - golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= - golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= - golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= - golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= - golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= - golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= - golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= - golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= - golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= - golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= - golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= - golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= - golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= - golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= - golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= - golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= - golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= - golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= - golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= - golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= - golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= - golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= - golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= - golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= - golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= - golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - """ - ), - } - ).digest + digest = set_up_go_mod( + rule_runner, + dedent( + """\ + module example.com/unsupported + go 1.16 + require golang.org/x/mobile v0.0.0-20210924032853-1c027f395ef7 + """ + ), + dedent( + """\ + github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= + github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= + github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= + github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= + golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= + golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 h1:estk1glOnSVeJ9tdEZZc5mAMDZk5lNJNyJ6DvrBkTEU= + golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= + golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= + golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= + golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= + golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= + golang.org/x/mobile v0.0.0-20210924032853-1c027f395ef7 h1:CyFUjc175y/mbMjxe+WdqI72jguLyjQChKCDe9mfTvg= + golang.org/x/mobile v0.0.0-20210924032853-1c027f395ef7/go.mod h1:c4YKU3ZylDmvbw+H/PSvm42vhdWbuxCzbonauEAP9B8= + golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= + golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= + golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= + golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= + golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= + golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= + golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= + golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= + golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= + golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= + golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= + golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= + golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= + golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= + golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= + golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= + golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= + golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= + golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= + golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + """ + ), + ) - # Nothing should error when computing `ThirdPartyModuleInfo`, we only create an exception to + # Nothing should error when computing `AllThirdPartyPackages`, we only create an exception to # maybe raise later. - module_info = rule_runner.request( - ThirdPartyModuleInfo, - [ThirdPartyModuleInfoRequest("golang.org/x/mobile", mobile_version, input_digest)], + all_packages = rule_runner.request( + AllThirdPartyPackages, [AllThirdPartyPackagesRequest(digest)] + ) + assert ( + all_packages.import_paths_to_pkg_info[ + "golang.org/x/mobile/bind/objc" + ].unsupported_sources_error + is not None ) - assert module_info["golang.org/x/mobile/bind/objc"].unsupported_sources_error is not None # Error when requesting the `ThirdPartyPkgInfo`. with engine_error(NotImplementedError): rule_runner.request( ThirdPartyPkgInfo, - [ - ThirdPartyPkgInfoRequest( - "golang.org/x/mobile/bind/objc", - "golang.org/x/mobile", - mobile_version, - input_digest, - ) - ], + [ThirdPartyPkgInfoRequest("golang.org/x/mobile/bind/objc", digest)], ) + + +def test_determine_pkg_info_module_with_replace_directive(rule_runner: RuleRunner) -> None: + """Regression test for https://github.com/pantsbuild/pants/issues/13138.""" + digest = set_up_go_mod( + rule_runner, + dedent( + """\ + module example.com/third-party-module + go 1.16 + require github.com/hashicorp/consul/api v1.3.0 + """ + ), + dedent( + """\ + github.com/Azure/go-autorest v13.3.2+incompatible h1:VxzPyuhtnlBOzc4IWCZHqpyH2d+QMLQEuy3wREyY4oc= + github.com/Azure/go-autorest v13.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= + github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= + github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= + github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= + github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= + github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= + github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= + github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= + github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= + github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= + github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= + github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= + github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= + github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= + github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= + github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= + github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= + github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= + github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= + github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= + github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= + github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= + github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= + github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= + github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= + github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= + github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= + github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= + github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= + github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= + github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= + github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= + github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= + github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= + github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= + github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= + github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= + github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= + github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= + github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= + github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= + github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= + github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= + github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= + github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= + github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= + github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= + github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= + github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= + github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= + github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= + github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= + github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= + github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= + github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= + github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= + github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= + github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= + github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= + github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= + github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= + github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= + github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= + github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= + github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= + github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= + github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= + github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= + github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= + github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= + github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= + github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= + github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= + github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= + github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= + github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= + github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= + github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= + github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= + github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= + github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= + github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= + github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= + github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= + github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= + github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= + golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3 h1:KYQXGkl6vs02hK7pK4eIbw0NpNPedieTSTEiJ//bwGs= + golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= + golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= + golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= + golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= + golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= + golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= + golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5 h1:x6r4Jo0KNzOOzYd8lbcRsqjuqEASK6ob3auvWYM4/8U= + golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= + """ + ), + ) + pkg_info = rule_runner.request( + ThirdPartyPkgInfo, + [ThirdPartyPkgInfoRequest("github.com/hashicorp/consul/api", digest)], + ) + assert pkg_info.subpath == "github.com/hashicorp/consul/api@v1.3.0" + assert "raw.go" in pkg_info.go_files