From c3c587ad194de48978c38a4ed984c9613e2ecfc0 Mon Sep 17 00:00:00 2001 From: Tianyu Geng Date: Mon, 11 Mar 2024 14:57:25 -0700 Subject: [PATCH] Make fineGrainedHashExternalRepos work with BzlMod With Bzlmod, the location of external repository is no longer `/external/`. Hence, a devoted repo resolver is added and it resolves bzlmod repo location by doing a `bazel query`. The result is cached to mitigate performance impact of such a call. --- .../main/kotlin/com/bazel_diff/di/Modules.kt | 61 +++++++++--------- .../bazel_diff/hash/ExternalRepoResolver.kt | 47 ++++++++++++++ .../com/bazel_diff/hash/SourceFileHasher.kt | 13 ++-- cli/src/test/kotlin/com/bazel_diff/Modules.kt | 12 ++-- .../test/kotlin/com/bazel_diff/e2e/E2ETest.kt | 56 ++++++++++++++++ .../bazel_diff/hash/SourceFileHasherTest.kt | 23 +++---- .../fine-grained-hash-bzlmod-test-1.zip | Bin 0 -> 8168 bytes .../fine-grained-hash-bzlmod-test-2.zip | Bin 0 -> 8216 bytes ...ined-hash-bzlmod-test-impacted-targets.txt | 23 +++++++ 9 files changed, 180 insertions(+), 55 deletions(-) create mode 100644 cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt create mode 100644 cli/src/test/resources/fixture/fine-grained-hash-bzlmod-test-1.zip create mode 100644 cli/src/test/resources/fixture/fine-grained-hash-bzlmod-test-2.zip create mode 100644 cli/src/test/resources/fixture/fine-grained-hash-bzlmod-test-impacted-targets.txt diff --git a/cli/src/main/kotlin/com/bazel_diff/di/Modules.kt b/cli/src/main/kotlin/com/bazel_diff/di/Modules.kt index c9837fb2..af10e373 100644 --- a/cli/src/main/kotlin/com/bazel_diff/di/Modules.kt +++ b/cli/src/main/kotlin/com/bazel_diff/di/Modules.kt @@ -2,10 +2,7 @@ package com.bazel_diff.di import com.bazel_diff.bazel.BazelClient import com.bazel_diff.bazel.BazelQueryService -import com.bazel_diff.hash.BuildGraphHasher -import com.bazel_diff.hash.RuleHasher -import com.bazel_diff.hash.SourceFileHasher -import com.bazel_diff.hash.TargetHasher +import com.bazel_diff.hash.* import com.bazel_diff.io.ContentHashProvider import com.bazel_diff.log.Logger import com.bazel_diff.log.StderrLogger @@ -23,26 +20,36 @@ import java.nio.file.Paths @OptIn(ExperimentalCoroutinesApi::class) fun hasherModule( - workingDirectory: Path, - bazelPath: Path, - contentHashPath: File?, - startupOptions: List, - commandOptions: List, - cqueryOptions: List, - useCquery: Boolean, - keepGoing: Boolean, - fineGrainedHashExternalRepos: Set, + workingDirectory: Path, + bazelPath: Path, + contentHashPath: File?, + startupOptions: List, + commandOptions: List, + cqueryOptions: List, + useCquery: Boolean, + keepGoing: Boolean, + fineGrainedHashExternalRepos: Set, ): Module = module { + val result = runBlocking { + process( + bazelPath.toString(), "info", "output_base", + stdout = Redirect.CAPTURE, + workingDirectory = workingDirectory.toFile(), + stderr = Redirect.PRINT, + destroyForcibly = true, + ) + } + val outputPath = Paths.get(result.output.single()) val debug = System.getProperty("DEBUG", "false").equals("true") single { BazelQueryService( - workingDirectory, - bazelPath, - startupOptions, - commandOptions, - cqueryOptions, - keepGoing, - debug + workingDirectory, + bazelPath, + startupOptions, + commandOptions, + cqueryOptions, + keepGoing, + debug ) } single { BazelClient(useCquery, fineGrainedHashExternalRepos) } @@ -50,19 +57,9 @@ fun hasherModule( single { TargetHasher() } single { RuleHasher(useCquery, fineGrainedHashExternalRepos) } single { SourceFileHasher(fineGrainedHashExternalRepos) } + single { ExternalRepoResolver(workingDirectory, bazelPath, outputPath) } single(named("working-directory")) { workingDirectory } - single(named("output-base")) { - val result = runBlocking { - process( - bazelPath.toString(), "info", "output_base", - stdout = Redirect.CAPTURE, - workingDirectory = workingDirectory.toFile(), - stderr = Redirect.PRINT, - destroyForcibly = true, - ) - } - Paths.get(result.output.single()) - } + single(named("output-base")) { outputPath } single { ContentHashProvider(contentHashPath) } } diff --git a/cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt b/cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt new file mode 100644 index 00000000..376df046 --- /dev/null +++ b/cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt @@ -0,0 +1,47 @@ +package com.bazel_diff.hash + +import com.google.common.cache.CacheBuilder +import com.google.common.cache.CacheLoader +import org.koin.core.component.KoinComponent +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +class ExternalRepoResolver( + private val workingDirectory: Path, + private val bazelPath: Path, + private val outputBase: Path, +) : KoinComponent { + private val externalRoot: Path by lazy { + outputBase.resolve("external") + } + + private val cache = CacheBuilder.newBuilder().build(CacheLoader.from { repoName: String -> + val externalRepoRoot = externalRoot.resolve(repoName) + if (Files.exists(externalRepoRoot)) { + return@from externalRepoRoot + } + resolveBzlModPath(repoName) + }) + + fun resolveExternalRepoRoot(repoName: String): Path { + return cache.get(repoName) + } + + private fun resolveBzlModPath(repoName: String): Path { + // Query result line should look something like "/external//some/bazel/target: