From d4f73c636ef99a5c9c2c0c064e4c442270fe492f Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 10 Jan 2022 17:03:57 +0100 Subject: [PATCH 01/11] rewrite third party dep analysis to not use `go list` [ci skip-rust] --- .../pants/backend/experimental/go/register.py | 2 + .../pants/backend/go/goals/check_test.py | 2 + .../goals/package_binary_integration_test.py | 2 + .../pants/backend/go/goals/tailor_test.py | 2 + .../pants/backend/go/goals/test_test.py | 2 + .../go/lint/gofmt/rules_integration_test.py | 2 + .../go/lint/vet/rules_integration_test.py | 2 + .../pants/backend/go/target_type_rules.py | 6 +- .../backend/go/target_type_rules_test.py | 2 + .../util_rules/assembly_integration_test.py | 2 + .../backend/go/util_rules/build_pkg_target.py | 7 +- .../go/util_rules/build_pkg_target_test.py | 8 +- .../backend/go/util_rules/build_pkg_test.py | 2 + .../go/util_rules/embed_integration_test.py | 2 + .../backend/go/util_rules/first_party_pkg.py | 10 +- .../go/util_rules/first_party_pkg_test.py | 2 + .../backend/go/util_rules/pkg_analyzer.py | 36 ++ .../backend/go/util_rules/third_party_pkg.py | 471 +++++++++++++----- .../go/util_rules/third_party_pkg_test.py | 72 ++- 19 files changed, 456 insertions(+), 178 deletions(-) create mode 100644 src/python/pants/backend/go/util_rules/pkg_analyzer.py diff --git a/src/python/pants/backend/experimental/go/register.py b/src/python/pants/backend/experimental/go/register.py index f3eb1ffdb9f..729666e2f9f 100644 --- a/src/python/pants/backend/experimental/go/register.py +++ b/src/python/pants/backend/experimental/go/register.py @@ -22,6 +22,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, tests_analysis, third_party_pkg, @@ -44,6 +45,7 @@ def rules(): *go_mod.rules(), *first_party_pkg.rules(), *link.rules(), + *pkg_analyzer.rules(), *sdk.rules(), *tests_analysis.rules(), *tailor.rules(), diff --git a/src/python/pants/backend/go/goals/check_test.py b/src/python/pants/backend/go/goals/check_test.py index 33c3d3e1885..48f0a70d4a3 100644 --- a/src/python/pants/backend/go/goals/check_test.py +++ b/src/python/pants/backend/go/goals/check_test.py @@ -19,6 +19,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -39,6 +40,7 @@ def rule_runner() -> RuleRunner: *import_analysis.rules(), *link.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/goals/package_binary_integration_test.py b/src/python/pants/backend/go/goals/package_binary_integration_test.py index ad5e8779046..0cd9f9daa1d 100644 --- a/src/python/pants/backend/go/goals/package_binary_integration_test.py +++ b/src/python/pants/backend/go/goals/package_binary_integration_test.py @@ -21,6 +21,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -42,6 +43,7 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *link.rules(), *target_type_rules.rules(), *third_party_pkg.rules(), diff --git a/src/python/pants/backend/go/goals/tailor_test.py b/src/python/pants/backend/go/goals/tailor_test.py index 9a4a6ce439a..62d13d195b8 100644 --- a/src/python/pants/backend/go/goals/tailor_test.py +++ b/src/python/pants/backend/go/goals/tailor_test.py @@ -16,6 +16,7 @@ first_party_pkg, go_mod, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -35,6 +36,7 @@ def rule_runner() -> RuleRunner: rules=[ *go_tailor_rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *sdk.rules(), diff --git a/src/python/pants/backend/go/goals/test_test.py b/src/python/pants/backend/go/goals/test_test.py index 4d60dcc44c7..581dea99b04 100644 --- a/src/python/pants/backend/go/goals/test_test.py +++ b/src/python/pants/backend/go/goals/test_test.py @@ -19,6 +19,7 @@ first_party_pkg, go_mod, link, + pkg_analyzer, sdk, tests_analysis, third_party_pkg, @@ -40,6 +41,7 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *link.rules(), *sdk.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py b/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py index 6278b508e15..62de818bc8d 100644 --- a/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py +++ b/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py @@ -18,6 +18,7 @@ first_party_pkg, go_mod, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -44,6 +45,7 @@ def rule_runner() -> RuleRunner: *third_party_pkg.rules(), *sdk.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *build_pkg.rules(), *link.rules(), *assembly.rules(), diff --git a/src/python/pants/backend/go/lint/vet/rules_integration_test.py b/src/python/pants/backend/go/lint/vet/rules_integration_test.py index 9f51b0e6a4a..3cafed54df0 100644 --- a/src/python/pants/backend/go/lint/vet/rules_integration_test.py +++ b/src/python/pants/backend/go/lint/vet/rules_integration_test.py @@ -21,6 +21,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -47,6 +48,7 @@ def rule_runner() -> RuleRunner: *third_party_pkg.rules(), *sdk.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *import_analysis.rules(), *link.rules(), *build_pkg.rules(), diff --git a/src/python/pants/backend/go/target_type_rules.py b/src/python/pants/backend/go/target_type_rules.py index e352336d884..088f587fe8f 100644 --- a/src/python/pants/backend/go/target_type_rules.py +++ b/src/python/pants/backend/go/target_type_rules.py @@ -30,7 +30,7 @@ from pants.backend.go.util_rules.third_party_pkg import ( AllThirdPartyPackages, AllThirdPartyPackagesRequest, - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, ThirdPartyPkgInfoRequest, ) from pants.base.exceptions import ResolveError @@ -163,7 +163,7 @@ async def inject_go_third_party_package_dependencies( ) tgt = wrapped_target.target pkg_info = await Get( - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, ThirdPartyPkgInfoRequest( tgt[GoImportPathField].value, go_mod_info.digest, go_mod_info.mod_path ), @@ -212,7 +212,7 @@ async def generate_targets_from_go_mod( AllThirdPartyPackagesRequest(go_mod_info.digest, go_mod_info.mod_path), ) - def create_tgt(pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyPackageTarget: + def create_tgt(pkg_info: ThirdPartyPkgAnalysis) -> GoThirdPartyPackageTarget: return GoThirdPartyPackageTarget( {GoImportPathField.alias: pkg_info.import_path}, # E.g. `src/go:mod#github.com/google/uuid`. diff --git a/src/python/pants/backend/go/target_type_rules_test.py b/src/python/pants/backend/go/target_type_rules_test.py index e5f0c9c99dc..a6b034c2f79 100644 --- a/src/python/pants/backend/go/target_type_rules_test.py +++ b/src/python/pants/backend/go/target_type_rules_test.py @@ -29,6 +29,7 @@ first_party_pkg, go_mod, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -57,6 +58,7 @@ def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *go_mod.rules(), + *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *sdk.rules(), diff --git a/src/python/pants/backend/go/util_rules/assembly_integration_test.py b/src/python/pants/backend/go/util_rules/assembly_integration_test.py index 6357c92d0e0..b6c99fd8951 100644 --- a/src/python/pants/backend/go/util_rules/assembly_integration_test.py +++ b/src/python/pants/backend/go/util_rules/assembly_integration_test.py @@ -21,6 +21,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -43,6 +44,7 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *link.rules(), *target_type_rules.rules(), *third_party_pkg.rules(), diff --git a/src/python/pants/backend/go/util_rules/build_pkg_target.py b/src/python/pants/backend/go/util_rules/build_pkg_target.py index 37d51b18284..1b8769f2b27 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_target.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_target.py @@ -23,7 +23,10 @@ FirstPartyPkgDigestRequest, ) from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest -from pants.backend.go.util_rules.third_party_pkg import ThirdPartyPkgInfo, ThirdPartyPkgInfoRequest +from pants.backend.go.util_rules.third_party_pkg import ( + ThirdPartyPkgAnalysis, + ThirdPartyPkgInfoRequest, +) from pants.build_graph.address import Address from pants.engine.engine_aware import EngineAwareParameter from pants.engine.internals.selectors import Get, MultiGet @@ -103,7 +106,7 @@ async def setup_build_go_package_target_request( _go_mod_address = target.address.maybe_convert_to_target_generator() _go_mod_info = await Get(GoModInfo, GoModInfoRequest(_go_mod_address)) _third_party_pkg_info = await Get( - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, ThirdPartyPkgInfoRequest(import_path, _go_mod_info.digest, _go_mod_info.mod_path), ) diff --git a/src/python/pants/backend/go/util_rules/build_pkg_target_test.py b/src/python/pants/backend/go/util_rules/build_pkg_target_test.py index 783fdc30555..d0974c50787 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_target_test.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_target_test.py @@ -18,6 +18,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -46,6 +47,7 @@ def rule_runner() -> RuleRunner: *import_analysis.rules(), *link.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *target_type_rules.rules(), @@ -160,7 +162,7 @@ def test_build_third_party_pkg_target(rule_runner: RuleRunner) -> None: rule_runner, Address("", target_name="mod", generated_name=import_path), expected_import_path=import_path, - expected_dir_path="github.com/google/uuid@v1.3.0", + expected_dir_path="gopath/pkg/mod/github.com/google/uuid@v1.3.0", expected_go_file_names=[ "dce.go", "doc.go", @@ -247,7 +249,7 @@ def test_build_target_with_dependencies(rule_runner: RuleRunner) -> None: rule_runner, Address("", target_name="mod", generated_name=xerrors_internal_import_path), expected_import_path=xerrors_internal_import_path, - expected_dir_path="golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/internal", + expected_dir_path="gopath/pkg/mod/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/internal", expected_go_file_names=["internal.go"], expected_direct_dependency_import_paths=[], expected_transitive_dependency_import_paths=[], @@ -258,7 +260,7 @@ def test_build_target_with_dependencies(rule_runner: RuleRunner) -> None: rule_runner, Address("", target_name="mod", generated_name=xerrors_import_path), expected_import_path=xerrors_import_path, - expected_dir_path="golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543", + expected_dir_path="gopath/pkg/mod/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543", expected_go_file_names=[ "adaptor.go", "doc.go", diff --git a/src/python/pants/backend/go/util_rules/build_pkg_test.py b/src/python/pants/backend/go/util_rules/build_pkg_test.py index a3116973c27..dbc7a1cd023 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_test.py @@ -17,6 +17,7 @@ go_mod, import_analysis, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -42,6 +43,7 @@ def rule_runner() -> RuleRunner: *go_mod.rules(), *first_party_pkg.rules(), *link.rules(), + *pkg_analyzer.rules(), *third_party_pkg.rules(), *target_type_rules.rules(), QueryRule(BuiltGoPackage, [BuildGoPackageRequest]), diff --git a/src/python/pants/backend/go/util_rules/embed_integration_test.py b/src/python/pants/backend/go/util_rules/embed_integration_test.py index 181037daa46..27ac8c73e1a 100644 --- a/src/python/pants/backend/go/util_rules/embed_integration_test.py +++ b/src/python/pants/backend/go/util_rules/embed_integration_test.py @@ -18,6 +18,7 @@ first_party_pkg, go_mod, link, + pkg_analyzer, sdk, tests_analysis, third_party_pkg, @@ -40,6 +41,7 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), + *pkg_analyzer.rules(), *link.rules(), *sdk.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg.py b/src/python/pants/backend/go/util_rules/first_party_pkg.py index 282d6680ab1..de677a0e566 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg.py @@ -17,6 +17,7 @@ OwningGoMod, OwningGoModRequest, ) +from pants.backend.go.util_rules.pkg_analyzer import PackageAnalyzerSetup from pants.build_graph.address import Address from pants.core.target_types import ResourceSourceField from pants.core.util_rules import source_files @@ -157,12 +158,9 @@ async def compute_first_party_package_import_path( @rule async def analyze_first_party_package( request: FirstPartyPkgAnalysisRequest, + analyzer: PackageAnalyzerSetup, ) -> FallibleFirstPartyPkgAnalysis: - analyzer, wrapped_target, import_path_info, owning_go_mod = await MultiGet( - Get( - LoadedGoBinary, - LoadedGoBinaryRequest("analyze_package", ("main.go", "read.go"), "./package_analyzer"), - ), + wrapped_target, import_path_info, owning_go_mod = await MultiGet( Get(WrappedTarget, Address, request.address), Get(FirstPartyPkgImportPath, FirstPartyPkgImportPathRequest(request.address)), Get(OwningGoMod, OwningGoModRequest(request.address)), @@ -178,7 +176,7 @@ async def analyze_first_party_package( result = await Get( FallibleProcessResult, Process( - ("./package_analyzer", request.address.spec_path or "."), + (analyzer.path, request.address.spec_path or "."), input_digest=input_digest, description=f"Determine metadata for {request.address}", level=LogLevel.DEBUG, diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py index a8b575b3448..1ff9ffda394 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py @@ -16,6 +16,7 @@ first_party_pkg, go_mod, link, + pkg_analyzer, sdk, third_party_pkg, ) @@ -53,6 +54,7 @@ def rule_runner() -> RuleRunner: *build_pkg.rules(), *link.rules(), *assembly.rules(), + *pkg_analyzer.rules(), generate_targets_from_resources, UnionRule(GenerateTargetsRequest, GenerateTargetsFromResources), QueryRule(FallibleFirstPartyPkgAnalysis, [FirstPartyPkgAnalysisRequest]), diff --git a/src/python/pants/backend/go/util_rules/pkg_analyzer.py b/src/python/pants/backend/go/util_rules/pkg_analyzer.py new file mode 100644 index 00000000000..b520e9ef0e7 --- /dev/null +++ b/src/python/pants/backend/go/util_rules/pkg_analyzer.py @@ -0,0 +1,36 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass + +from pants.backend.go.go_sources import load_go_binary +from pants.backend.go.go_sources.load_go_binary import LoadedGoBinary, LoadedGoBinaryRequest +from pants.engine.fs import Digest +from pants.engine.internals.selectors import Get +from pants.engine.rules import collect_rules, rule + + +@dataclass(frozen=True) +class PackageAnalyzerSetup: + digest: Digest + path: str + + +@rule +async def setup_go_package_analyzer() -> PackageAnalyzerSetup: + binary_path = "./package_analyzer" + binary = await Get( + LoadedGoBinary, + LoadedGoBinaryRequest("analyze_package", ("main.go", "read.go"), binary_path), + ) + return PackageAnalyzerSetup( + digest=binary.digest, + path=binary_path, + ) + + +def rules(): + return ( + *collect_rules(), + *load_go_binary.rules(), + ) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index 4335b9ac7ff..4414ba9f531 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -3,28 +3,34 @@ from __future__ import annotations +import json import logging -import os.path +import os +import posixpath from dataclasses import dataclass +from typing import Any import ijson +from pants.backend.go.util_rules.pkg_analyzer import PackageAnalyzerSetup from pants.backend.go.util_rules.sdk import GoSdkProcess +from pants.core.goals.tailor import group_by_dir +from pants.engine.collection import DeduplicatedCollection from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( EMPTY_DIGEST, - AddPrefix, Digest, DigestSubset, + GlobExpansionConjunction, GlobMatchErrorBehavior, PathGlobs, - RemovePrefix, + Snapshot, ) -from pants.engine.process import ProcessResult +from pants.engine.process import Process, ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel -from pants.util.strutil import strip_prefix, strip_v2_chroot_path +from pants.util.ordered_set import FrozenOrderedSet logger = logging.getLogger(__name__) @@ -34,7 +40,7 @@ class GoThirdPartyPkgError(Exception): @dataclass(frozen=True) -class ThirdPartyPkgInfo: +class ThirdPartyPkgAnalysis: """All the info and files needed to build a third-party package. The digest only contains the files for the package, with all prefixes stripped. @@ -72,7 +78,7 @@ def debug_hint(self) -> str: @dataclass(frozen=True) -class AllThirdPartyPackages(FrozenDict[str, ThirdPartyPkgInfo]): +class AllThirdPartyPackages(FrozenDict[str, ThirdPartyPkgAnalysis]): """All the packages downloaded from a go.mod, along with a digest of the downloaded files. The digest has files in the format `gopath/pkg/mod`, which is what `GoSdkProcess` sets `GOPATH` @@ -81,7 +87,7 @@ class AllThirdPartyPackages(FrozenDict[str, ThirdPartyPkgInfo]): """ digest: Digest - import_paths_to_pkg_info: FrozenDict[str, ThirdPartyPkgInfo] + import_paths_to_pkg_info: FrozenDict[str, ThirdPartyPkgAnalysis] @dataclass(frozen=True) @@ -90,118 +96,360 @@ class AllThirdPartyPackagesRequest: go_mod_path: str -@rule(desc="Download and analyze all third-party Go packages", level=LogLevel.DEBUG) -async def download_and_analyze_third_party_packages( - request: AllThirdPartyPackagesRequest, -) -> AllThirdPartyPackages: - # NB: We download all modules to GOPATH={chroot}/gopath. Running `go list ...` from {chroot} - # would naively try analyzing the contents of the GOPATH like they were first-party packages. - # This results in errors like this: - # - # package /gopath/pkg/mod/golang.org/x/text@v0.3.0/unicode: can only use - # path@version syntax with 'go get' and 'go install' in module-aware mode - # - # Instead, we make sure we run `go list` from a subdirectory of the chroot. It can still - # access the contents of `GOPATH`, but won't incorrectly treat its contents as - # first-party packages. - go_mod_dir = os.path.dirname(request.go_mod_path) - if not go_mod_dir: - go_mod_dir = "go_mod_prefix" - go_mod_digest = await Get(Digest, AddPrefix(request.go_mod_digest, go_mod_dir)) - else: - go_mod_digest = request.go_mod_digest - - list_argv = ( - "list", - # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace. - # Instead, we expect them to be well-formed already. - # - # It would be convenient to set `-mod=mod` to allow edits, and then compare the resulting - # files to the input so that we could print a diff for the user to know how to update. But - # `-mod=mod` results in more packages being downloaded and added to `go.mod` than is - # actually necessary. - # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a - # message from Go and won't be intuitive for Pants users what to do. - "-mod=readonly", - # There may be some packages in the transitive closure that cannot be built, but we should - # not blow up Pants. - # - # For example, a package that sets the special value `package documentation` and has no - # source files would naively error due to `build constraints exclude all Go files`, even - # though we should not error on that package. - "-e", - "-json", - # This matches all packages. `all` only matches first-party packages and complains that - # there are no `.go` files. - "...", +@dataclass(frozen=True) +class ModuleDescriptorsRequest: + digest: Digest + path: str + + +@dataclass(frozen=True) +class ModuleDescriptor: + import_path: str + name: str + version: str + indirect: bool + minimum_go_version: str | None + sum: str | None + + +class ModuleDescriptors(DeduplicatedCollection[ModuleDescriptor]): + """An ordered list of `ModuleDescriptor`s.""" + + +@dataclass(frozen=True) +class AnalyzeThirdPartyModuleRequest: + go_mod_digest: Digest + go_mod_path: str + import_path: str + name: str + version: str + minimum_go_version: str | None + + +@dataclass(frozen=True) +class AnalyzedThirdPartyModule: + packages: FrozenOrderedSet[ThirdPartyPkgAnalysis] + + +@dataclass(frozen=True) +class AnalyzeThirdPartyPackageRequest: + pkg_json: FrozenDict[str, Any] + module_sources_digest: Digest + module_sources_path: str + module_import_path: str + package_path: str + minimum_go_version: str | None + + +@dataclass(frozen=True) +class FallibleThirdPartyPkgAnalysis: + """Metadata for a third-party Go package, but fallible if our analysis failed.""" + + analysis: ThirdPartyPkgAnalysis | None + import_path: str + exit_code: int = 0 + stderr: str | None = None + + +@rule +async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> ModuleDescriptors: + mod_list_result = await Get( + ProcessResult, + GoSdkProcess( + command=["list", "-mod=readonly", "-e", "-m", "-json", "all"], + input_digest=request.digest, + working_dir=request.path if request.path else None, + allow_downloads=True, + description="Analyze Go module dependencies.", + ), ) - list_result = await Get( + + if len(mod_list_result.stdout) == 0: + return ModuleDescriptors([]) + + descriptors: dict[tuple[str, str], ModuleDescriptor] = {} + + for mod_json in ijson.items(mod_list_result.stdout, "", multiple_values=True): + # Skip the first-party module being analyzed. + if "Main" in mod_json and mod_json["Main"]: + continue + + if "Replace" in mod_json: + # TODO: Reject local file path replacements? Gazelle does. + name = mod_json["Replace"]["Path"] + version = mod_json["Replace"]["Version"] + else: + name = mod_json["Path"] + version = mod_json["Version"] + + descriptors[(name, version)] = ModuleDescriptor( + import_path=mod_json["Path"], + name=name, + version=version, + indirect=mod_json.get("Indirect", False), + minimum_go_version=mod_json.get("GoVersion"), + sum=None, + ) + + # TODO: Augment the modules with go.sum entries? + # Gazelle does this, mainly to store the sum on the go_repository rule. We could store it (or its + # absence) to be able to download sums automatically. + + return ModuleDescriptors(descriptors.values()) + + +def strip_sandbox_prefix(path: str, marker: str) -> str: + """Strip a path prefix from a path using a marker string to find the start of the portion to not + strip. This is used to strip absolute paths used in the execution sandbox by `go`. + + Note: The marker string is required because we cannot assume how the prefix will be formed since it + will differ depending on which execution environment is used (e.g, local or remote). + """ + marker_pos = path.find(marker) + if marker_pos != -1: + return path[marker_pos:] + else: + return path + + +def _freeze_json_dict(d: dict[Any, Any]) -> FrozenDict[str, Any]: + result = {} + for k, v in d.items(): + if not isinstance(k, str): + raise AssertionError("Got non-`str` key for _freeze_json_dict.") + + f: Any = None + if isinstance(v, list): + f = tuple(v) + elif isinstance(v, dict): + f = _freeze_json_dict(v) + elif isinstance(v, str) or isinstance(v, int): + f = v + else: + raise AssertionError(f"Unsupported value type for _freeze_json_dict: {type(v)}") + result[k] = f + return FrozenDict(result) + + +@rule +async def analyze_go_third_party_module( + request: AnalyzeThirdPartyModuleRequest, + analyzer: PackageAnalyzerSetup, +) -> AnalyzedThirdPartyModule: + # Download the module. + # TODO: Download all at once and split? Would not allow caching already downloaded items though. + # TODO: Should gopath be an append-only cache or should we reassemble GOPATH cache when trying to run + # module operations that could benefit from the cache? + download_result = await Get( ProcessResult, GoSdkProcess( - command=list_argv, - description=f"Run `go list` to download {request.go_mod_path}", - input_digest=go_mod_digest, - output_directories=("gopath/pkg/mod",), - working_dir=go_mod_dir, + ("mod", "download", "-json", f"{request.name}@{request.version}"), + input_digest=request.go_mod_digest, # for go.sum + working_dir=request.go_mod_path if request.go_mod_path else None, allow_downloads=True, + output_directories=("gopath",), + description=f"Download Go module {request.name}@{request.version}.", ), ) - stripped_result_digest = await Get( - Digest, RemovePrefix(list_result.output_digest, "gopath/pkg/mod") + + if len(download_result.stdout) == 0: + raise AssertionError( + f"Expected output from `go mod download` for {request.name}@{request.version}." + ) + + module_metadata = json.loads(download_result.stdout) + module_sources_relpath = strip_sandbox_prefix(module_metadata["Dir"], "gopath/") + go_mod_relpath = strip_sandbox_prefix(module_metadata["GoMod"], "gopath/") + + # Subset the output directory to just the module sources and go.mod (which may be generated). + module_sources_snapshot = await Get( + Snapshot, + DigestSubset( + download_result.output_digest, + PathGlobs( + [f"{module_sources_relpath}/**", go_mod_relpath], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + conjunction=GlobExpansionConjunction.all_match, + description_of_origin=f"the download of Go module {request.name}@{request.version}", + ), + ), ) - all_digest_subset_gets = [] - all_pkg_info_kwargs = [] - all_failed_pkg_info = [] - for pkg_json in ijson.items(list_result.stdout, "", multiple_values=True): - if "Standard" in pkg_json: + # Determine directories with potential Go packages in them. + candidate_package_dirs = [] + files_by_dir = group_by_dir( + p for p in module_sources_snapshot.files if p.startswith(module_sources_relpath) + ) + for maybe_pkg_dir, files in files_by_dir.items(): + # Skip directories where "testdata" would end up in the import path. + # See https://github.com/golang/go/blob/f005df8b582658d54e63d59953201299d6fee880/src/go/build/build.go#L580-L585 + if "testdata" in maybe_pkg_dir.split("/"): continue - import_path = pkg_json["ImportPath"] - maybe_error, maybe_failed_pkg_info = maybe_raise_or_create_error_or_create_failed_pkg_info( - pkg_json, import_path + # Consider directories with at least one `.go` file as package candidates. + if any(f for f in files if f.endswith(".go")): + candidate_package_dirs.append(maybe_pkg_dir) + candidate_package_dirs.sort() + + # Analyze all of the packages in this module. + analyzer_relpath = "__analyzer" + analysis_result = await Get( + ProcessResult, + Process( + [os.path.join(analyzer_relpath, analyzer.path), *candidate_package_dirs], + input_digest=module_sources_snapshot.digest, + immutable_input_digests={ + analyzer_relpath: analyzer.digest, + }, + description=f"Analyze metadata for Go third-party module: {request.name}@{request.version}", + level=LogLevel.DEBUG, + ), + ) + + if len(analysis_result.stdout) == 0: + return AnalyzedThirdPartyModule(FrozenOrderedSet()) + + package_analysis_gets = [] + for pkg_path, pkg_json in zip( + candidate_package_dirs, ijson.items(analysis_result.stdout, "", multiple_values=True) + ): + package_analysis_gets.append( + Get( + FallibleThirdPartyPkgAnalysis, + AnalyzeThirdPartyPackageRequest( + pkg_json=_freeze_json_dict(pkg_json), + module_sources_digest=module_sources_snapshot.digest, + module_sources_path=module_sources_relpath, + module_import_path=request.name, + package_path=pkg_path, + minimum_go_version=request.minimum_go_version, + ), + ) ) - if maybe_failed_pkg_info: - all_failed_pkg_info.append(maybe_failed_pkg_info) - continue + analyzed_packages_fallible = await MultiGet(package_analysis_gets) + analyzed_packages = [ + pkg.analysis for pkg in analyzed_packages_fallible if pkg.analysis and pkg.exit_code == 0 + ] + return AnalyzedThirdPartyModule(FrozenOrderedSet(analyzed_packages)) + + +@rule +async def analyze_go_third_party_package( + request: AnalyzeThirdPartyPackageRequest, +) -> FallibleThirdPartyPkgAnalysis: + import_path_tail = posixpath.relpath(request.package_path, start=request.module_sources_path) + import_path = ( + posixpath.join(request.module_import_path, import_path_tail) + if import_path_tail != "." + else request.module_import_path + ) - dir_path = strip_prefix(strip_v2_chroot_path(pkg_json["Dir"]), "gopath/pkg/mod/") - all_pkg_info_kwargs.append( - dict( - import_path=import_path, - dir_path=dir_path, - imports=tuple(pkg_json.get("Imports", ())), - go_files=tuple(pkg_json.get("GoFiles", ())), - s_files=tuple(pkg_json.get("SFiles", ())), - minimum_go_version=pkg_json.get("Module", {}).get("GoVersion"), - error=maybe_error, + if "Error" in request.pkg_json or "InvalidGoFiles" in request.pkg_json: + error = request.pkg_json.get("Error", "") + if error: + error += "\n" + if "InvalidGoFiles" in request.pkg_json: + error += "\n".join( + f"{filename}: {error}" + for filename, error in request.pkg_json.get("InvalidGoFiles", {}).items() ) + error += "\n" + return FallibleThirdPartyPkgAnalysis( + analysis=None, import_path=import_path, exit_code=1, stderr=error ) - all_digest_subset_gets.append( - Get( - Digest, - DigestSubset( - stripped_result_digest, - PathGlobs( - [os.path.join(dir_path, "*")], - glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin=f"downloading {import_path}", - ), - ), + + maybe_error: GoThirdPartyPkgError | None = None + + for key in ( + "CgoFiles", + "CompiledGoFiles", + "CFiles", + "CXXFiles", + "MFiles", + "HFiles", + "FFiles", + "SwigFiles", + "SwigCXXFiles", + "SysoFiles", + ): + if key in request.pkg_json: + maybe_error = GoThirdPartyPkgError( + f"The third-party package {import_path} includes `{key}`, which Pants does " + "not yet support. Please open a feature request at " + "https://github.com/pantsbuild/pants/issues/new/choose so that we know to " + "prioritize adding support. Please include this error message and the version of " + "the third-party module." ) + + package_digest = await Get( + Digest, + DigestSubset( + request.module_sources_digest, + PathGlobs( + [os.path.join(request.package_path, "*")], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=f"the analysis of Go package {import_path}", + ), + ), + ) + + analysis = ThirdPartyPkgAnalysis( + digest=package_digest, + import_path=import_path, + dir_path=request.package_path, + imports=tuple(request.pkg_json.get("Imports", ())), + go_files=tuple(request.pkg_json.get("GoFiles", ())), + s_files=tuple(request.pkg_json.get("SFiles", ())), + minimum_go_version=request.minimum_go_version, + error=maybe_error, + ) + + return FallibleThirdPartyPkgAnalysis( + analysis=analysis, + import_path=import_path, + exit_code=0, + stderr=None, + ) + + +@rule(desc="Download and analyze all third-party Go packages", level=LogLevel.DEBUG) +async def download_and_analyze_third_party_packages( + request: AllThirdPartyPackagesRequest, +) -> AllThirdPartyPackages: + modules = await Get( + ModuleDescriptors, + ModuleDescriptorsRequest( + digest=request.go_mod_digest, + path=os.path.dirname(request.go_mod_path), + ), + ) + + analyzed_modules = await MultiGet( + Get( + AnalyzedThirdPartyModule, + AnalyzeThirdPartyModuleRequest( + go_mod_digest=request.go_mod_digest, + go_mod_path=request.go_mod_path, + import_path=mod.name, + name=mod.name, + version=mod.version, + minimum_go_version=mod.minimum_go_version, + ), ) + for mod in modules + ) - all_digest_subsets = await MultiGet(all_digest_subset_gets) import_path_to_info = { - pkg_info_kwargs["import_path"]: ThirdPartyPkgInfo(digest=digest_subset, **pkg_info_kwargs) - for pkg_info_kwargs, digest_subset in zip(all_pkg_info_kwargs, all_digest_subsets) + pkg.import_path: pkg + for analyzed_module in analyzed_modules + for pkg in analyzed_module.packages } - import_path_to_info.update((pkg_info.import_path, pkg_info) for pkg_info in all_failed_pkg_info) - return AllThirdPartyPackages(list_result.output_digest, FrozenDict(import_path_to_info)) + + return AllThirdPartyPackages(EMPTY_DIGEST, FrozenDict(import_path_to_info)) @rule -async def extract_package_info(request: ThirdPartyPkgInfoRequest) -> ThirdPartyPkgInfo: +async def extract_package_info(request: ThirdPartyPkgInfoRequest) -> ThirdPartyPkgAnalysis: all_packages = await Get( AllThirdPartyPackages, AllThirdPartyPackagesRequest(request.go_mod_digest, request.go_mod_path), @@ -218,7 +466,7 @@ async def extract_package_info(request: ThirdPartyPkgInfoRequest) -> ThirdPartyP def maybe_raise_or_create_error_or_create_failed_pkg_info( go_list_json: dict, import_path: str -) -> tuple[GoThirdPartyPkgError | None, ThirdPartyPkgInfo | None]: +) -> tuple[GoThirdPartyPkgError | None, ThirdPartyPkgAnalysis | None]: """Error for unrecoverable errors, otherwise lazily create an error or `ThirdPartyPkgInfo` for recoverable errors. @@ -247,7 +495,7 @@ def maybe_raise_or_create_error_or_create_failed_pkg_info( f"that we can figure out how to support this:" f"\n\n{go_list_json}" ) - return None, ThirdPartyPkgInfo( + return None, ThirdPartyPkgAnalysis( import_path=import_path, dir_path="", digest=EMPTY_DIGEST, @@ -270,29 +518,6 @@ def maybe_raise_or_create_error_or_create_failed_pkg_info( None, ) - for key in ( - "CgoFiles", - "CompiledGoFiles", - "CFiles", - "CXXFiles", - "MFiles", - "HFiles", - "FFiles", - "SwigFiles", - "SwigCXXFiles", - "SysoFiles", - ): - if key in go_list_json: - return ( - GoThirdPartyPkgError( - f"The third-party package {import_path} includes `{key}`, which Pants does " - "not yet support. Please open a feature request at " - "https://github.com/pantsbuild/pants/issues/new/choose so that we know to " - "prioritize adding support. Please include this error message and the version of " - "the third-party module." - ), - None, - ) return None, None diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py index 5c6ffc7bca5..048ee05def0 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py @@ -8,15 +8,27 @@ import pytest +from pants.backend.go import target_type_rules +from pants.backend.go.go_sources import load_go_binary from pants.backend.go.target_types import GoModTarget -from pants.backend.go.util_rules import sdk, third_party_pkg +from pants.backend.go.util_rules import ( + assembly, + build_pkg, + first_party_pkg, + go_mod, + import_analysis, + link, + pkg_analyzer, + sdk, + third_party_pkg, +) from pants.backend.go.util_rules.third_party_pkg import ( AllThirdPartyPackages, AllThirdPartyPackagesRequest, - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, ThirdPartyPkgInfoRequest, ) -from pants.engine.fs import EMPTY_DIGEST, Digest, Snapshot +from pants.engine.fs import Digest, Snapshot from pants.engine.process import ProcessExecutionFailure from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner, engine_error @@ -28,8 +40,17 @@ def rule_runner() -> RuleRunner: rules=[ *sdk.rules(), *third_party_pkg.rules(), + *first_party_pkg.rules(), + *pkg_analyzer.rules(), + *load_go_binary.rules(), + *build_pkg.rules(), + *import_analysis.rules(), + *link.rules(), + *assembly.rules(), + *target_type_rules.rules(), + *go_mod.rules(), QueryRule(AllThirdPartyPackages, [AllThirdPartyPackagesRequest]), - QueryRule(ThirdPartyPkgInfo, [ThirdPartyPkgInfoRequest]), + QueryRule(ThirdPartyPkgAnalysis, [ThirdPartyPkgInfoRequest]), ], target_types=[GoModTarget], ) @@ -156,7 +177,7 @@ def assert_pkg_info( assert_pkg_info( import_path="github.com/google/uuid", - dir_path="github.com/google/uuid@v1.3.0", + dir_path="gopath/pkg/mod/github.com/google/uuid@v1.3.0", imports=( "bytes", "crypto/md5", @@ -209,7 +230,7 @@ def assert_pkg_info( ) assert_pkg_info( import_path="golang.org/x/text/unicode/bidi", - dir_path="golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c/unicode/bidi", + dir_path="gopath/pkg/mod/golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c/unicode/bidi", imports=("container/list", "fmt", "log", "sort", "unicode/utf8"), go_files=("bidi.go", "bracket.go", "core.go", "prop.go", "tables.go", "trieval.go"), extra_files=( @@ -298,7 +319,7 @@ def test_pkg_missing(rule_runner: RuleRunner) -> None: AssertionError, contains="The package `another_project.org/foo` was not downloaded" ): rule_runner.request( - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, [ThirdPartyPkgInfoRequest("another_project.org/foo", digest, "go.mod")], ) @@ -393,7 +414,7 @@ def test_unsupported_sources(rule_runner: RuleRunner) -> None: ), ) pkg_info = rule_runner.request( - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, [ThirdPartyPkgInfoRequest("golang.org/x/mobile/bind/objc", digest, "go.mod")], ) assert pkg_info.error is not None @@ -514,39 +535,8 @@ def test_determine_pkg_info_module_with_replace_directive(rule_runner: RuleRunne ), ) pkg_info = rule_runner.request( - ThirdPartyPkgInfo, + ThirdPartyPkgAnalysis, [ThirdPartyPkgInfoRequest("github.com/hashicorp/consul/api", digest, "go.mod")], ) - assert pkg_info.dir_path == "github.com/hashicorp/consul/api@v1.3.0" + assert pkg_info.dir_path == "gopath/pkg/mod/github.com/hashicorp/consul/api@v1.3.0" assert "raw.go" in pkg_info.go_files - - -def test_ambiguous_package(rule_runner: RuleRunner) -> None: - digest = set_up_go_mod( - rule_runner, - dedent( - """\ - module example.com/third-party-module - go 1.16 - require github.com/ugorji/go v1.1.4 - require github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 - """ - ), - dedent( - """\ - github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= - github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= - github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 h1:3SVOIvH7Ae1KRYyQWRjXWJEA9sS/c/pjvH++55Gr648= - github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= - """ - ), - ) - pkg_info = rule_runner.request( - ThirdPartyPkgInfo, - [ThirdPartyPkgInfoRequest("github.com/ugorji/go/codec", digest, "go.mod")], - ) - assert pkg_info.error is not None - # This particular error is tricky because `Dir` will not have been set, which we need to - # determine the dir_path and the digest. - assert pkg_info.dir_path == "" - assert pkg_info.digest == EMPTY_DIGEST From 52807301e955fef24f3bee3d41d06e2d31a94473 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 13 Jan 2022 11:52:38 +0100 Subject: [PATCH 02/11] rename ThirdPartyPkgInfoRequest -> ThirdPartyPkgAnalysisRequest [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/go/target_type_rules.py | 4 ++-- .../pants/backend/go/util_rules/build_pkg_target.py | 4 ++-- .../pants/backend/go/util_rules/third_party_pkg.py | 4 ++-- .../backend/go/util_rules/third_party_pkg_test.py | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/python/pants/backend/go/target_type_rules.py b/src/python/pants/backend/go/target_type_rules.py index 088f587fe8f..224e23b4ff1 100644 --- a/src/python/pants/backend/go/target_type_rules.py +++ b/src/python/pants/backend/go/target_type_rules.py @@ -31,7 +31,7 @@ AllThirdPartyPackages, AllThirdPartyPackagesRequest, ThirdPartyPkgAnalysis, - ThirdPartyPkgInfoRequest, + ThirdPartyPkgAnalysisRequest, ) from pants.base.exceptions import ResolveError from pants.base.specs import AddressSpecs, SiblingAddresses @@ -164,7 +164,7 @@ async def inject_go_third_party_package_dependencies( tgt = wrapped_target.target pkg_info = await Get( ThirdPartyPkgAnalysis, - ThirdPartyPkgInfoRequest( + ThirdPartyPkgAnalysisRequest( tgt[GoImportPathField].value, go_mod_info.digest, go_mod_info.mod_path ), ) diff --git a/src/python/pants/backend/go/util_rules/build_pkg_target.py b/src/python/pants/backend/go/util_rules/build_pkg_target.py index 1b8769f2b27..83cadec2dd2 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_target.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_target.py @@ -25,7 +25,7 @@ from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest from pants.backend.go.util_rules.third_party_pkg import ( ThirdPartyPkgAnalysis, - ThirdPartyPkgInfoRequest, + ThirdPartyPkgAnalysisRequest, ) from pants.build_graph.address import Address from pants.engine.engine_aware import EngineAwareParameter @@ -107,7 +107,7 @@ async def setup_build_go_package_target_request( _go_mod_info = await Get(GoModInfo, GoModInfoRequest(_go_mod_address)) _third_party_pkg_info = await Get( ThirdPartyPkgAnalysis, - ThirdPartyPkgInfoRequest(import_path, _go_mod_info.digest, _go_mod_info.mod_path), + ThirdPartyPkgAnalysisRequest(import_path, _go_mod_info.digest, _go_mod_info.mod_path), ) # We error if trying to _build_ a package with issues (vs. only generating the target and diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index 4414ba9f531..f33a9da7b79 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -63,7 +63,7 @@ class ThirdPartyPkgAnalysis: @dataclass(frozen=True) -class ThirdPartyPkgInfoRequest(EngineAwareParameter): +class ThirdPartyPkgAnalysisRequest(EngineAwareParameter): """Request the info and digest needed to build a third-party package. The package's module must be included in the input `go.mod`/`go.sum`. @@ -449,7 +449,7 @@ async def download_and_analyze_third_party_packages( @rule -async def extract_package_info(request: ThirdPartyPkgInfoRequest) -> ThirdPartyPkgAnalysis: +async def extract_package_info(request: ThirdPartyPkgAnalysisRequest) -> ThirdPartyPkgAnalysis: all_packages = await Get( AllThirdPartyPackages, AllThirdPartyPackagesRequest(request.go_mod_digest, request.go_mod_path), diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py index 048ee05def0..5c0a7519e3a 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py @@ -26,7 +26,7 @@ AllThirdPartyPackages, AllThirdPartyPackagesRequest, ThirdPartyPkgAnalysis, - ThirdPartyPkgInfoRequest, + ThirdPartyPkgAnalysisRequest, ) from pants.engine.fs import Digest, Snapshot from pants.engine.process import ProcessExecutionFailure @@ -50,7 +50,7 @@ def rule_runner() -> RuleRunner: *target_type_rules.rules(), *go_mod.rules(), QueryRule(AllThirdPartyPackages, [AllThirdPartyPackagesRequest]), - QueryRule(ThirdPartyPkgAnalysis, [ThirdPartyPkgInfoRequest]), + QueryRule(ThirdPartyPkgAnalysis, [ThirdPartyPkgAnalysisRequest]), ], target_types=[GoModTarget], ) @@ -320,7 +320,7 @@ def test_pkg_missing(rule_runner: RuleRunner) -> None: ): rule_runner.request( ThirdPartyPkgAnalysis, - [ThirdPartyPkgInfoRequest("another_project.org/foo", digest, "go.mod")], + [ThirdPartyPkgAnalysisRequest("another_project.org/foo", digest, "go.mod")], ) @@ -415,7 +415,7 @@ def test_unsupported_sources(rule_runner: RuleRunner) -> None: ) pkg_info = rule_runner.request( ThirdPartyPkgAnalysis, - [ThirdPartyPkgInfoRequest("golang.org/x/mobile/bind/objc", digest, "go.mod")], + [ThirdPartyPkgAnalysisRequest("golang.org/x/mobile/bind/objc", digest, "go.mod")], ) assert pkg_info.error is not None @@ -536,7 +536,7 @@ def test_determine_pkg_info_module_with_replace_directive(rule_runner: RuleRunne ) pkg_info = rule_runner.request( ThirdPartyPkgAnalysis, - [ThirdPartyPkgInfoRequest("github.com/hashicorp/consul/api", digest, "go.mod")], + [ThirdPartyPkgAnalysisRequest("github.com/hashicorp/consul/api", digest, "go.mod")], ) assert pkg_info.dir_path == "gopath/pkg/mod/github.com/hashicorp/consul/api@v1.3.0" assert "raw.go" in pkg_info.go_files From 25219cee360ef280bee27c1e2d98e900e9b604b5 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 13:48:44 +0100 Subject: [PATCH 03/11] put back test [ci skip-rust] [ci skip-build-wheels] --- .../go/util_rules/third_party_pkg_test.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py index 5c0a7519e3a..29b703e50c4 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py @@ -540,3 +540,36 @@ def test_determine_pkg_info_module_with_replace_directive(rule_runner: RuleRunne ) assert pkg_info.dir_path == "gopath/pkg/mod/github.com/hashicorp/consul/api@v1.3.0" assert "raw.go" in pkg_info.go_files + + +def test_ambiguous_package(rule_runner: RuleRunner) -> None: + digest = set_up_go_mod( + rule_runner, + dedent( + """\ + module example.com/third-party-module + go 1.16 + require github.com/ugorji/go v1.1.4 + require github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 + """ + ), + dedent( + """\ + github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= + github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= + github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 h1:3SVOIvH7Ae1KRYyQWRjXWJEA9sS/c/pjvH++55Gr648= + github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= + """ + ), + ) + pkg_info = rule_runner.request( + ThirdPartyPkgAnalysis, + [ThirdPartyPkgAnalysisRequest("github.com/ugorji/go/codec", digest, "go.mod")], + ) + print(f"pkg_info = {pkg_info}") + assert pkg_info.error is None + assert ( + pkg_info.dir_path + == "gopath/pkg/mod/github.com/ugorji/go/codec@v0.0.0-20181204163529-d75b2dcb6bc8" + ) + assert "encode.go" in pkg_info.go_files From aa5684fd4d2de70dab578e10b7200ea1a7785ca6 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 15:41:18 +0100 Subject: [PATCH 04/11] avoid incorrect use of posixpath [ci skip-rust] --- .../backend/go/util_rules/third_party_pkg.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index f33a9da7b79..2933ca9dcad 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -6,7 +6,6 @@ import json import logging import os -import posixpath from dataclasses import dataclass from typing import Any @@ -337,12 +336,22 @@ async def analyze_go_third_party_module( async def analyze_go_third_party_package( request: AnalyzeThirdPartyPackageRequest, ) -> FallibleThirdPartyPkgAnalysis: - import_path_tail = posixpath.relpath(request.package_path, start=request.module_sources_path) - import_path = ( - posixpath.join(request.module_import_path, import_path_tail) - if import_path_tail != "." - else request.module_import_path - ) + if not request.package_path.startswith(request.module_sources_path): + raise AssertionError( + "The path within GOPATH for a package in a module must always be prefixed by the path " + "to the applicable module's root directory. " + f"This was not the case however for module {request.module_import_path}.\n\n" + "This may be a bug in Pants. Please report this issue at " + "https://github.com/pantsbuild/pants/issues/new/choose and include the following data: " + f"package_path: {request.package_path}; module_sources_path: {request.module_sources_path}; " + f"module_import_path: {request.module_import_path}" + ) + import_path_tail = request.package_path[len(request.module_sources_path) :].strip(os.sep) + if import_path_tail != "": + parts = import_path_tail.split(os.sep) + import_path = "/".join([request.module_import_path, *parts]) + else: + import_path = request.module_import_path if "Error" in request.pkg_json or "InvalidGoFiles" in request.pkg_json: error = request.pkg_json.get("Error", "") From 4f2f61b1acaaa37c099ec08ccb1fd75e8b0f5952 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 16:13:40 +0100 Subject: [PATCH 05/11] pass downloaded go.mod's to download step [ci skip-rust] [ci skip-build-wheels] --- .../backend/go/util_rules/third_party_pkg.py | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index 2933ca9dcad..e1f0c387f03 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -14,7 +14,6 @@ from pants.backend.go.util_rules.pkg_analyzer import PackageAnalyzerSetup from pants.backend.go.util_rules.sdk import GoSdkProcess from pants.core.goals.tailor import group_by_dir -from pants.engine.collection import DeduplicatedCollection from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( EMPTY_DIGEST, @@ -22,6 +21,7 @@ DigestSubset, GlobExpansionConjunction, GlobMatchErrorBehavior, + MergeDigests, PathGlobs, Snapshot, ) @@ -111,8 +111,10 @@ class ModuleDescriptor: sum: str | None -class ModuleDescriptors(DeduplicatedCollection[ModuleDescriptor]): - """An ordered list of `ModuleDescriptor`s.""" +@dataclass(frozen=True) +class ModuleDescriptors: + modules: FrozenOrderedSet[ModuleDescriptor] + go_mods_digest: Digest @dataclass(frozen=True) @@ -152,11 +154,28 @@ class FallibleThirdPartyPkgAnalysis: @rule async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> ModuleDescriptors: + # List the modules used directly and indirectly by this module. + # + # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace. + # Instead, we expect them to be well-formed already. + # + # Options used: + # - `-mod=readonly': It would be convenient to set `-mod=mod` to allow edits, and then compare the + # resulting files to the input so that we could print a diff for the user to know how to update. But + # `-mod=mod` results in more packages being downloaded and added to `go.mod` than is + # actually necessary. + # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a + # message from Go and won't be intuitive for Pants users what to do. + # - `-e` is used to not fail if one of the modules is problematic. There may be some packages in the transitive + # closure that cannot be built, but we should not blow up Pants. For example, a package that sets the + # special value `package documentation` and has no source files would naively error due to + # `build constraints exclude all Go files`, even though we should not error on that package. mod_list_result = await Get( ProcessResult, GoSdkProcess( command=["list", "-mod=readonly", "-e", "-m", "-json", "all"], input_digest=request.digest, + output_directories=("gopath",), working_dir=request.path if request.path else None, allow_downloads=True, description="Analyze Go module dependencies.", @@ -164,7 +183,7 @@ async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> Modu ) if len(mod_list_result.stdout) == 0: - return ModuleDescriptors([]) + return ModuleDescriptors(FrozenOrderedSet(), EMPTY_DIGEST) descriptors: dict[tuple[str, str], ModuleDescriptor] = {} @@ -194,7 +213,7 @@ async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> Modu # Gazelle does this, mainly to store the sum on the go_repository rule. We could store it (or its # absence) to be able to download sums automatically. - return ModuleDescriptors(descriptors.values()) + return ModuleDescriptors(FrozenOrderedSet(descriptors.values()), mod_list_result.output_digest) def strip_sandbox_prefix(path: str, marker: str) -> str: @@ -236,9 +255,6 @@ async def analyze_go_third_party_module( analyzer: PackageAnalyzerSetup, ) -> AnalyzedThirdPartyModule: # Download the module. - # TODO: Download all at once and split? Would not allow caching already downloaded items though. - # TODO: Should gopath be an append-only cache or should we reassemble GOPATH cache when trying to run - # module operations that could benefit from the cache? download_result = await Get( ProcessResult, GoSdkProcess( @@ -425,7 +441,7 @@ async def analyze_go_third_party_package( async def download_and_analyze_third_party_packages( request: AllThirdPartyPackagesRequest, ) -> AllThirdPartyPackages: - modules = await Get( + module_analysis = await Get( ModuleDescriptors, ModuleDescriptorsRequest( digest=request.go_mod_digest, @@ -433,11 +449,15 @@ async def download_and_analyze_third_party_packages( ), ) + go_mod_digest = await Get( + Digest, MergeDigests([request.go_mod_digest, module_analysis.go_mods_digest]) + ) + analyzed_modules = await MultiGet( Get( AnalyzedThirdPartyModule, AnalyzeThirdPartyModuleRequest( - go_mod_digest=request.go_mod_digest, + go_mod_digest=go_mod_digest, go_mod_path=request.go_mod_path, import_path=mod.name, name=mod.name, @@ -445,7 +465,7 @@ async def download_and_analyze_third_party_packages( minimum_go_version=mod.minimum_go_version, ), ) - for mod in modules + for mod in module_analysis.modules ) import_path_to_info = { From 8e6392853708fa2b70fe1356df01d5b3f4dd7613 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 16:18:02 +0100 Subject: [PATCH 06/11] avoid spamming pkg_analyzer.rules() all over the place [ci skip-rust] [ci skip-build-wheels] --- .../backend/go/goals/package_binary_integration_test.py | 2 -- src/python/pants/backend/go/goals/tailor_test.py | 2 -- src/python/pants/backend/go/goals/test_test.py | 2 -- .../pants/backend/go/lint/gofmt/rules_integration_test.py | 2 -- .../pants/backend/go/lint/vet/rules_integration_test.py | 2 -- .../backend/go/util_rules/assembly_integration_test.py | 2 -- .../pants/backend/go/util_rules/build_pkg_target_test.py | 2 -- .../pants/backend/go/util_rules/embed_integration_test.py | 2 -- src/python/pants/backend/go/util_rules/first_party_pkg.py | 3 ++- .../pants/backend/go/util_rules/first_party_pkg_test.py | 2 -- src/python/pants/backend/go/util_rules/third_party_pkg.py | 6 +++++- .../pants/backend/go/util_rules/third_party_pkg_test.py | 2 -- 12 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/python/pants/backend/go/goals/package_binary_integration_test.py b/src/python/pants/backend/go/goals/package_binary_integration_test.py index 0cd9f9daa1d..ad5e8779046 100644 --- a/src/python/pants/backend/go/goals/package_binary_integration_test.py +++ b/src/python/pants/backend/go/goals/package_binary_integration_test.py @@ -21,7 +21,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -43,7 +42,6 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *link.rules(), *target_type_rules.rules(), *third_party_pkg.rules(), diff --git a/src/python/pants/backend/go/goals/tailor_test.py b/src/python/pants/backend/go/goals/tailor_test.py index 62d13d195b8..9a4a6ce439a 100644 --- a/src/python/pants/backend/go/goals/tailor_test.py +++ b/src/python/pants/backend/go/goals/tailor_test.py @@ -16,7 +16,6 @@ first_party_pkg, go_mod, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -36,7 +35,6 @@ def rule_runner() -> RuleRunner: rules=[ *go_tailor_rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *sdk.rules(), diff --git a/src/python/pants/backend/go/goals/test_test.py b/src/python/pants/backend/go/goals/test_test.py index 581dea99b04..4d60dcc44c7 100644 --- a/src/python/pants/backend/go/goals/test_test.py +++ b/src/python/pants/backend/go/goals/test_test.py @@ -19,7 +19,6 @@ first_party_pkg, go_mod, link, - pkg_analyzer, sdk, tests_analysis, third_party_pkg, @@ -41,7 +40,6 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *link.rules(), *sdk.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py b/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py index 62de818bc8d..6278b508e15 100644 --- a/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py +++ b/src/python/pants/backend/go/lint/gofmt/rules_integration_test.py @@ -18,7 +18,6 @@ first_party_pkg, go_mod, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -45,7 +44,6 @@ def rule_runner() -> RuleRunner: *third_party_pkg.rules(), *sdk.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *build_pkg.rules(), *link.rules(), *assembly.rules(), diff --git a/src/python/pants/backend/go/lint/vet/rules_integration_test.py b/src/python/pants/backend/go/lint/vet/rules_integration_test.py index 3cafed54df0..9f51b0e6a4a 100644 --- a/src/python/pants/backend/go/lint/vet/rules_integration_test.py +++ b/src/python/pants/backend/go/lint/vet/rules_integration_test.py @@ -21,7 +21,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -48,7 +47,6 @@ def rule_runner() -> RuleRunner: *third_party_pkg.rules(), *sdk.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *import_analysis.rules(), *link.rules(), *build_pkg.rules(), diff --git a/src/python/pants/backend/go/util_rules/assembly_integration_test.py b/src/python/pants/backend/go/util_rules/assembly_integration_test.py index b6c99fd8951..6357c92d0e0 100644 --- a/src/python/pants/backend/go/util_rules/assembly_integration_test.py +++ b/src/python/pants/backend/go/util_rules/assembly_integration_test.py @@ -21,7 +21,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -44,7 +43,6 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *link.rules(), *target_type_rules.rules(), *third_party_pkg.rules(), diff --git a/src/python/pants/backend/go/util_rules/build_pkg_target_test.py b/src/python/pants/backend/go/util_rules/build_pkg_target_test.py index d0974c50787..5a4f741476d 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_target_test.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_target_test.py @@ -18,7 +18,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -47,7 +46,6 @@ def rule_runner() -> RuleRunner: *import_analysis.rules(), *link.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/util_rules/embed_integration_test.py b/src/python/pants/backend/go/util_rules/embed_integration_test.py index 27ac8c73e1a..181037daa46 100644 --- a/src/python/pants/backend/go/util_rules/embed_integration_test.py +++ b/src/python/pants/backend/go/util_rules/embed_integration_test.py @@ -18,7 +18,6 @@ first_party_pkg, go_mod, link, - pkg_analyzer, sdk, tests_analysis, third_party_pkg, @@ -41,7 +40,6 @@ def rule_runner() -> RuleRunner: *build_pkg_target.rules(), *first_party_pkg.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *link.rules(), *sdk.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg.py b/src/python/pants/backend/go/util_rules/first_party_pkg.py index de677a0e566..46568cb7700 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg.py @@ -10,6 +10,7 @@ from pants.backend.go.go_sources import load_go_binary from pants.backend.go.go_sources.load_go_binary import LoadedGoBinary, LoadedGoBinaryRequest from pants.backend.go.target_types import GoPackageSourcesField +from pants.backend.go.util_rules import pkg_analyzer from pants.backend.go.util_rules.embedcfg import EmbedConfig from pants.backend.go.util_rules.go_mod import ( GoModInfo, @@ -323,4 +324,4 @@ async def setup_first_party_pkg_digest( def rules(): - return (*collect_rules(), *source_files.rules(), *load_go_binary.rules()) + return (*collect_rules(), *source_files.rules(), *load_go_binary.rules(), *pkg_analyzer.rules()) diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py index 1ff9ffda394..a8b575b3448 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py @@ -16,7 +16,6 @@ first_party_pkg, go_mod, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -54,7 +53,6 @@ def rule_runner() -> RuleRunner: *build_pkg.rules(), *link.rules(), *assembly.rules(), - *pkg_analyzer.rules(), generate_targets_from_resources, UnionRule(GenerateTargetsRequest, GenerateTargetsFromResources), QueryRule(FallibleFirstPartyPkgAnalysis, [FirstPartyPkgAnalysisRequest]), diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index e1f0c387f03..ad2e4913e27 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -11,6 +11,7 @@ import ijson +from pants.backend.go.util_rules import pkg_analyzer from pants.backend.go.util_rules.pkg_analyzer import PackageAnalyzerSetup from pants.backend.go.util_rules.sdk import GoSdkProcess from pants.core.goals.tailor import group_by_dir @@ -551,4 +552,7 @@ def maybe_raise_or_create_error_or_create_failed_pkg_info( def rules(): - return collect_rules() + return ( + *collect_rules(), + *pkg_analyzer.rules(), + ) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py index 29b703e50c4..9245f418ede 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py @@ -18,7 +18,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -41,7 +40,6 @@ def rule_runner() -> RuleRunner: *sdk.rules(), *third_party_pkg.rules(), *first_party_pkg.rules(), - *pkg_analyzer.rules(), *load_go_binary.rules(), *build_pkg.rules(), *import_analysis.rules(), From d3589f62ba7abf9342611713326bb257d49add40 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 21:24:48 +0100 Subject: [PATCH 07/11] add comments explaining allow_downloads=True [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/go/util_rules/third_party_pkg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index ad2e4913e27..288de7912f4 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -178,6 +178,7 @@ async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> Modu input_digest=request.digest, output_directories=("gopath",), working_dir=request.path if request.path else None, + # Allow downloads of the module metadata (i.e., go.mod files). allow_downloads=True, description="Analyze Go module dependencies.", ), @@ -262,6 +263,7 @@ async def analyze_go_third_party_module( ("mod", "download", "-json", f"{request.name}@{request.version}"), input_digest=request.go_mod_digest, # for go.sum working_dir=request.go_mod_path if request.go_mod_path else None, + # Allow downloads of the module sources. allow_downloads=True, output_directories=("gopath",), description=f"Download Go module {request.name}@{request.version}.", From 15ce45d5c4a17cd0761fda37aed1161c7070b1b4 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 21:30:00 +0100 Subject: [PATCH 08/11] cleanup more pkg_analyzer strays [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/go/goals/check_test.py | 2 -- src/python/pants/backend/go/target_type_rules_test.py | 2 -- src/python/pants/backend/go/util_rules/build_pkg_test.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/src/python/pants/backend/go/goals/check_test.py b/src/python/pants/backend/go/goals/check_test.py index 48f0a70d4a3..33c3d3e1885 100644 --- a/src/python/pants/backend/go/goals/check_test.py +++ b/src/python/pants/backend/go/goals/check_test.py @@ -19,7 +19,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -40,7 +39,6 @@ def rule_runner() -> RuleRunner: *import_analysis.rules(), *link.rules(), *go_mod.rules(), - *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/target_type_rules_test.py b/src/python/pants/backend/go/target_type_rules_test.py index a6b034c2f79..e5f0c9c99dc 100644 --- a/src/python/pants/backend/go/target_type_rules_test.py +++ b/src/python/pants/backend/go/target_type_rules_test.py @@ -29,7 +29,6 @@ first_party_pkg, go_mod, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -58,7 +57,6 @@ def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *go_mod.rules(), - *pkg_analyzer.rules(), *first_party_pkg.rules(), *third_party_pkg.rules(), *sdk.rules(), diff --git a/src/python/pants/backend/go/util_rules/build_pkg_test.py b/src/python/pants/backend/go/util_rules/build_pkg_test.py index dbc7a1cd023..a3116973c27 100644 --- a/src/python/pants/backend/go/util_rules/build_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/build_pkg_test.py @@ -17,7 +17,6 @@ go_mod, import_analysis, link, - pkg_analyzer, sdk, third_party_pkg, ) @@ -43,7 +42,6 @@ def rule_runner() -> RuleRunner: *go_mod.rules(), *first_party_pkg.rules(), *link.rules(), - *pkg_analyzer.rules(), *third_party_pkg.rules(), *target_type_rules.rules(), QueryRule(BuiltGoPackage, [BuildGoPackageRequest]), From db7172a89e71dd8cb2a344eac02a4e8afe6bf2b1 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 21:30:40 +0100 Subject: [PATCH 09/11] remove stray print [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/go/util_rules/third_party_pkg_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py index 9245f418ede..2f8459a65f7 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg_test.py @@ -564,7 +564,6 @@ def test_ambiguous_package(rule_runner: RuleRunner) -> None: ThirdPartyPkgAnalysis, [ThirdPartyPkgAnalysisRequest("github.com/ugorji/go/codec", digest, "go.mod")], ) - print(f"pkg_info = {pkg_info}") assert pkg_info.error is None assert ( pkg_info.dir_path From d145eca599fb50a770014571c8ae69acbbabb36c Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 21:34:03 +0100 Subject: [PATCH 10/11] remove unused (yet) `sum` field [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/go/util_rules/third_party_pkg.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index 288de7912f4..059ea4fc5de 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -109,7 +109,6 @@ class ModuleDescriptor: version: str indirect: bool minimum_go_version: str | None - sum: str | None @dataclass(frozen=True) @@ -208,7 +207,6 @@ async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> Modu version=version, indirect=mod_json.get("Indirect", False), minimum_go_version=mod_json.get("GoVersion"), - sum=None, ) # TODO: Augment the modules with go.sum entries? From 606f69359c388656acf47809d09618b789fb9d6c Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Fri, 14 Jan 2022 21:48:14 +0100 Subject: [PATCH 11/11] fix working_dir [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/go/util_rules/third_party_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/go/util_rules/third_party_pkg.py b/src/python/pants/backend/go/util_rules/third_party_pkg.py index 059ea4fc5de..3eada420126 100644 --- a/src/python/pants/backend/go/util_rules/third_party_pkg.py +++ b/src/python/pants/backend/go/util_rules/third_party_pkg.py @@ -260,7 +260,7 @@ async def analyze_go_third_party_module( GoSdkProcess( ("mod", "download", "-json", f"{request.name}@{request.version}"), input_digest=request.go_mod_digest, # for go.sum - working_dir=request.go_mod_path if request.go_mod_path else None, + working_dir=os.path.dirname(request.go_mod_path) if request.go_mod_path else None, # Allow downloads of the module sources. allow_downloads=True, output_directories=("gopath",),