Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[internal] go: rewrite third party package analysis to not use go list #14157

Merged
merged 11 commits into from
Jan 14, 2022
2 changes: 2 additions & 0 deletions src/python/pants/backend/experimental/go/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
go_mod,
import_analysis,
link,
pkg_analyzer,
sdk,
tests_analysis,
third_party_pkg,
Expand All @@ -44,6 +45,7 @@ def rules():
*go_mod.rules(),
*first_party_pkg.rules(),
*link.rules(),
*pkg_analyzer.rules(),
*sdk.rules(),
*tests_analysis.rules(),
*tailor.rules(),
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/backend/go/goals/check_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
go_mod,
import_analysis,
link,
pkg_analyzer,
tdyas marked this conversation as resolved.
Show resolved Hide resolved
sdk,
third_party_pkg,
)
Expand All @@ -39,6 +40,7 @@ def rule_runner() -> RuleRunner:
*import_analysis.rules(),
*link.rules(),
*go_mod.rules(),
*pkg_analyzer.rules(),
*first_party_pkg.rules(),
*third_party_pkg.rules(),
*target_type_rules.rules(),
Expand Down
10 changes: 5 additions & 5 deletions src/python/pants/backend/go/target_type_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
from pants.backend.go.util_rules.third_party_pkg import (
AllThirdPartyPackages,
AllThirdPartyPackagesRequest,
ThirdPartyPkgInfo,
ThirdPartyPkgInfoRequest,
ThirdPartyPkgAnalysis,
ThirdPartyPkgAnalysisRequest,
)
from pants.base.exceptions import ResolveError
from pants.base.specs import AddressSpecs, SiblingAddresses
Expand Down Expand Up @@ -163,8 +163,8 @@ async def inject_go_third_party_package_dependencies(
)
tgt = wrapped_target.target
pkg_info = await Get(
ThirdPartyPkgInfo,
ThirdPartyPkgInfoRequest(
ThirdPartyPkgAnalysis,
ThirdPartyPkgAnalysisRequest(
tgt[GoImportPathField].value, go_mod_info.digest, go_mod_info.mod_path
),
)
Expand Down Expand Up @@ -212,7 +212,7 @@ async def generate_targets_from_go_mod(
AllThirdPartyPackagesRequest(go_mod_info.digest, go_mod_info.mod_path),
)

def create_tgt(pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyPackageTarget:
def create_tgt(pkg_info: ThirdPartyPkgAnalysis) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{GoImportPathField.alias: pkg_info.import_path},
# E.g. `src/go:mod#github.com/google/uuid`.
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/backend/go/target_type_rules_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
first_party_pkg,
go_mod,
link,
pkg_analyzer,
sdk,
third_party_pkg,
)
Expand Down Expand Up @@ -57,6 +58,7 @@ def rule_runner() -> RuleRunner:
rule_runner = RuleRunner(
rules=[
*go_mod.rules(),
*pkg_analyzer.rules(),
tdyas marked this conversation as resolved.
Show resolved Hide resolved
*first_party_pkg.rules(),
*third_party_pkg.rules(),
*sdk.rules(),
Expand Down
9 changes: 6 additions & 3 deletions src/python/pants/backend/go/util_rules/build_pkg_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
FirstPartyPkgDigestRequest,
)
from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest
from pants.backend.go.util_rules.third_party_pkg import ThirdPartyPkgInfo, ThirdPartyPkgInfoRequest
from pants.backend.go.util_rules.third_party_pkg import (
ThirdPartyPkgAnalysis,
ThirdPartyPkgAnalysisRequest,
)
from pants.build_graph.address import Address
from pants.engine.engine_aware import EngineAwareParameter
from pants.engine.internals.selectors import Get, MultiGet
Expand Down Expand Up @@ -103,8 +106,8 @@ async def setup_build_go_package_target_request(
_go_mod_address = target.address.maybe_convert_to_target_generator()
_go_mod_info = await Get(GoModInfo, GoModInfoRequest(_go_mod_address))
_third_party_pkg_info = await Get(
ThirdPartyPkgInfo,
ThirdPartyPkgInfoRequest(import_path, _go_mod_info.digest, _go_mod_info.mod_path),
ThirdPartyPkgAnalysis,
ThirdPartyPkgAnalysisRequest(import_path, _go_mod_info.digest, _go_mod_info.mod_path),
)

# We error if trying to _build_ a package with issues (vs. only generating the target and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def test_build_third_party_pkg_target(rule_runner: RuleRunner) -> None:
rule_runner,
Address("", target_name="mod", generated_name=import_path),
expected_import_path=import_path,
expected_dir_path="github.com/google/uuid@v1.3.0",
expected_dir_path="gopath/pkg/mod/github.com/google/uuid@v1.3.0",
expected_go_file_names=[
"dce.go",
"doc.go",
Expand Down Expand Up @@ -247,7 +247,7 @@ def test_build_target_with_dependencies(rule_runner: RuleRunner) -> None:
rule_runner,
Address("", target_name="mod", generated_name=xerrors_internal_import_path),
expected_import_path=xerrors_internal_import_path,
expected_dir_path="golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/internal",
expected_dir_path="gopath/pkg/mod/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/internal",
expected_go_file_names=["internal.go"],
expected_direct_dependency_import_paths=[],
expected_transitive_dependency_import_paths=[],
Expand All @@ -258,7 +258,7 @@ def test_build_target_with_dependencies(rule_runner: RuleRunner) -> None:
rule_runner,
Address("", target_name="mod", generated_name=xerrors_import_path),
expected_import_path=xerrors_import_path,
expected_dir_path="golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543",
expected_dir_path="gopath/pkg/mod/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543",
expected_go_file_names=[
"adaptor.go",
"doc.go",
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/backend/go/util_rules/build_pkg_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
go_mod,
import_analysis,
link,
pkg_analyzer,
sdk,
third_party_pkg,
)
Expand All @@ -42,6 +43,7 @@ def rule_runner() -> RuleRunner:
*go_mod.rules(),
*first_party_pkg.rules(),
*link.rules(),
*pkg_analyzer.rules(),
tdyas marked this conversation as resolved.
Show resolved Hide resolved
*third_party_pkg.rules(),
*target_type_rules.rules(),
QueryRule(BuiltGoPackage, [BuildGoPackageRequest]),
Expand Down
13 changes: 6 additions & 7 deletions src/python/pants/backend/go/util_rules/first_party_pkg.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@
from pants.backend.go.go_sources import load_go_binary
from pants.backend.go.go_sources.load_go_binary import LoadedGoBinary, LoadedGoBinaryRequest
from pants.backend.go.target_types import GoPackageSourcesField
from pants.backend.go.util_rules import pkg_analyzer
from pants.backend.go.util_rules.embedcfg import EmbedConfig
from pants.backend.go.util_rules.go_mod import (
GoModInfo,
GoModInfoRequest,
OwningGoMod,
OwningGoModRequest,
)
from pants.backend.go.util_rules.pkg_analyzer import PackageAnalyzerSetup
from pants.build_graph.address import Address
from pants.core.target_types import ResourceSourceField
from pants.core.util_rules import source_files
Expand Down Expand Up @@ -157,12 +159,9 @@ async def compute_first_party_package_import_path(
@rule
async def analyze_first_party_package(
request: FirstPartyPkgAnalysisRequest,
analyzer: PackageAnalyzerSetup,
) -> FallibleFirstPartyPkgAnalysis:
analyzer, wrapped_target, import_path_info, owning_go_mod = await MultiGet(
Get(
LoadedGoBinary,
LoadedGoBinaryRequest("analyze_package", ("main.go", "read.go"), "./package_analyzer"),
),
wrapped_target, import_path_info, owning_go_mod = await MultiGet(
Get(WrappedTarget, Address, request.address),
Get(FirstPartyPkgImportPath, FirstPartyPkgImportPathRequest(request.address)),
Get(OwningGoMod, OwningGoModRequest(request.address)),
Expand All @@ -178,7 +177,7 @@ async def analyze_first_party_package(
result = await Get(
FallibleProcessResult,
Process(
("./package_analyzer", request.address.spec_path or "."),
(analyzer.path, request.address.spec_path or "."),
input_digest=input_digest,
description=f"Determine metadata for {request.address}",
level=LogLevel.DEBUG,
Expand Down Expand Up @@ -325,4 +324,4 @@ async def setup_first_party_pkg_digest(


def rules():
return (*collect_rules(), *source_files.rules(), *load_go_binary.rules())
return (*collect_rules(), *source_files.rules(), *load_go_binary.rules(), *pkg_analyzer.rules())
36 changes: 36 additions & 0 deletions src/python/pants/backend/go/util_rules/pkg_analyzer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from dataclasses import dataclass

from pants.backend.go.go_sources import load_go_binary
from pants.backend.go.go_sources.load_go_binary import LoadedGoBinary, LoadedGoBinaryRequest
from pants.engine.fs import Digest
from pants.engine.internals.selectors import Get
from pants.engine.rules import collect_rules, rule


@dataclass(frozen=True)
class PackageAnalyzerSetup:
digest: Digest
path: str


@rule
async def setup_go_package_analyzer() -> PackageAnalyzerSetup:
binary_path = "./package_analyzer"
binary = await Get(
LoadedGoBinary,
LoadedGoBinaryRequest("analyze_package", ("main.go", "read.go"), binary_path),
)
return PackageAnalyzerSetup(
digest=binary.digest,
path=binary_path,
)


def rules():
return (
*collect_rules(),
*load_go_binary.rules(),
)
Loading