Make fineGrainedHashExternalRepos work with BzlMod

cli/src/main/kotlin/com/bazel_diff/di/Modules.kt

@@ -2,10 +2,7 @@ package com.bazel_diff.di
 
 import com.bazel_diff.bazel.BazelClient
 import com.bazel_diff.bazel.BazelQueryService
-import com.bazel_diff.hash.BuildGraphHasher
-import com.bazel_diff.hash.RuleHasher
-import com.bazel_diff.hash.SourceFileHasher
-import com.bazel_diff.hash.TargetHasher
+import com.bazel_diff.hash.*
 import com.bazel_diff.io.ContentHashProvider
 import com.bazel_diff.log.Logger
 import com.bazel_diff.log.StderrLogger
@@ -23,46 +20,46 @@ import java.nio.file.Paths
 
 @OptIn(ExperimentalCoroutinesApi::class)
 fun hasherModule(
-    workingDirectory: Path,
-    bazelPath: Path,
-    contentHashPath: File?,
-    startupOptions: List<String>,
-    commandOptions: List<String>,
-    cqueryOptions: List<String>,
-    useCquery: Boolean,
-    keepGoing: Boolean,
-    fineGrainedHashExternalRepos: Set<String>,
+        workingDirectory: Path,
+        bazelPath: Path,
+        contentHashPath: File?,
+        startupOptions: List<String>,
+        commandOptions: List<String>,
+        cqueryOptions: List<String>,
+        useCquery: Boolean,
+        keepGoing: Boolean,
+        fineGrainedHashExternalRepos: Set<String>,
 ): Module = module {
+    val result = runBlocking {
+        process(
+                bazelPath.toString(), "info", "output_base",
+                stdout = Redirect.CAPTURE,
+                workingDirectory = workingDirectory.toFile(),
+                stderr = Redirect.PRINT,
+                destroyForcibly = true,
+        )
+    }
+    val outputPath = Paths.get(result.output.single())
     val debug = System.getProperty("DEBUG", "false").equals("true")
     single {
         BazelQueryService(
-            workingDirectory,
-            bazelPath,
-            startupOptions,
-            commandOptions,
-            cqueryOptions,
-            keepGoing,
-            debug
+                workingDirectory,
+                bazelPath,
+                startupOptions,
+                commandOptions,
+                cqueryOptions,
+                keepGoing,
+                debug
         )
     }
     single { BazelClient(useCquery, fineGrainedHashExternalRepos) }
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
     single { RuleHasher(useCquery, fineGrainedHashExternalRepos) }
     single { SourceFileHasher(fineGrainedHashExternalRepos) }
+    single { ExternalRepoResolver(workingDirectory, bazelPath, outputPath) }
     single(named("working-directory")) { workingDirectory }
-    single(named("output-base")) {
-        val result = runBlocking {
-            process(
-                bazelPath.toString(), "info", "output_base",
-                stdout = Redirect.CAPTURE,
-                workingDirectory = workingDirectory.toFile(),
-                stderr = Redirect.PRINT,
-                destroyForcibly = true,
-            )
-        }
-        Paths.get(result.output.single())
-    }
+    single(named("output-base")) { outputPath }
     single { ContentHashProvider(contentHashPath) }
 }
 

cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt

@@ -0,0 +1,47 @@
+package com.bazel_diff.hash
+
+import com.google.common.cache.CacheBuilder
+import com.google.common.cache.CacheLoader
+import org.koin.core.component.KoinComponent
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+
+class ExternalRepoResolver(
+        private val workingDirectory: Path,
+        private val bazelPath: Path,
+        private val outputBase: Path,
+) : KoinComponent {
+    private val externalRoot: Path by lazy {
+        outputBase.resolve("external")
+    }
+
+    private val cache = CacheBuilder.newBuilder().build(CacheLoader.from { repoName: String ->
+        val externalRepoRoot = externalRoot.resolve(repoName)
+        if (Files.exists(externalRepoRoot)) {
+            return@from externalRepoRoot
+        }
+        resolveBzlModPath(repoName)
+    })
+
+    fun resolveExternalRepoRoot(repoName: String): Path {
+        return cache.get(repoName)
+    }
+
+    private fun resolveBzlModPath(repoName: String): Path {
+        // Query result line should look something like "<exec root>/external/<canonical repo name>/some/bazel/target: <kind> <label>"
+        val queryResultLine = runProcessAndCaptureFirstLine(bazelPath.toString(), "query", "@$repoName//...", "--output", "location")
+        val path = Paths.get(queryResultLine.split(": ", limit = 2)[0])
+        val bzlModRelativePath = path.relativize(externalRoot).first()
+        return externalRoot.resolve(bzlModRelativePath)
+    }
+
+    private fun runProcessAndCaptureFirstLine(vararg command: String): String {
+        val process = ProcessBuilder(*command).directory(workingDirectory.toFile()).start()
+        process.inputStream.bufferedReader().use {
+            // read the first line and close the stream so that Bazel doesn't need to continue
+            // output all the query result.
+            return it.readLine()
+        }
+    }
+}

cli/src/main/kotlin/com/bazel_diff/hash/SourceFileHasher.kt

@@ -13,8 +13,8 @@ class SourceFileHasher : KoinComponent {
     private val workingDirectory: Path
     private val logger: Logger
     private val relativeFilenameToContentHash: Map<String, String>?
-    private val outputBase: Path
     private val fineGrainedHashExternalRepos: Set<String>
+    private val externalRepoResolver: ExternalRepoResolver
 
     init {
         val logger: Logger by inject()
@@ -26,16 +26,16 @@ class SourceFileHasher : KoinComponent {
         this.workingDirectory = workingDirectory
         val contentHashProvider: ContentHashProvider by inject()
         relativeFilenameToContentHash = contentHashProvider.filenameToHash
-        val outputBase: Path by inject(qualifier = named("output-base"))
-        this.outputBase = outputBase
         this.fineGrainedHashExternalRepos = fineGrainedHashExternalRepos
+        val externalRepoResolver: ExternalRepoResolver by inject()
+        this.externalRepoResolver = externalRepoResolver
     }
 
-    constructor(workingDirectory: Path, outputBase: Path, relativeFilenameToContentHash: Map<String, String>?, fineGrainedHashExternalRepos: Set<String> = emptySet()) {
+    constructor(workingDirectory: Path, relativeFilenameToContentHash: Map<String, String>?, externalRepoResolver: ExternalRepoResolver, fineGrainedHashExternalRepos: Set<String> = emptySet()) {
         this.workingDirectory = workingDirectory
-        this.outputBase = outputBase
         this.relativeFilenameToContentHash = relativeFilenameToContentHash
         this.fineGrainedHashExternalRepos = fineGrainedHashExternalRepos
+        this.externalRepoResolver = externalRepoResolver
     }
 
     fun digest(sourceFileTarget: BazelSourceFileTarget): ByteArray {
@@ -55,7 +55,8 @@ class SourceFileHasher : KoinComponent {
                     return@sha256
                 }
                 val relativePath = Paths.get(parts[1].removePrefix(":").replace(':', '/'))
-                outputBase.resolve("external/$repoName").resolve(relativePath)
+                val externalRepoRoot = externalRepoResolver.resolveExternalRepoRoot(repoName)
+                externalRepoRoot.resolve(relativePath)
             } else {
                 return@sha256
             }

cli/src/test/kotlin/com/bazel_diff/Modules.kt

@@ -1,10 +1,7 @@
 package com.bazel_diff
 
 import com.bazel_diff.bazel.BazelClient
-import com.bazel_diff.hash.BuildGraphHasher
-import com.bazel_diff.hash.RuleHasher
-import com.bazel_diff.hash.SourceFileHasher
-import com.bazel_diff.hash.TargetHasher
+import com.bazel_diff.hash.*
 import com.bazel_diff.io.ContentHashProvider
 import com.bazel_diff.log.Logger
 import com.google.gson.GsonBuilder
@@ -14,15 +11,18 @@ import org.koin.dsl.module
 import java.nio.file.Paths
 
 fun testModule(): Module = module {
+    val outputBase = Paths.get("output-base")
+    val workingDirectory = Paths.get("working-directory")
     single<Logger> { SilentLogger }
     single { BazelClient(false, emptySet()) }
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
     single { RuleHasher(false, emptySet()) }
+    single { ExternalRepoResolver(workingDirectory, Paths.get("bazel"), outputBase) }
     single { SourceFileHasher() }
     single { GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create() }
-    single(named("working-directory")) { Paths.get("working-directory") }
-    single(named("output-base")) { Paths.get("output-base") }
+    single(named("working-directory")) { workingDirectory }
+    single(named("output-base")) { outputBase }
     single { ContentHashProvider(null) }
 }
 

cli/src/test/kotlin/com/bazel_diff/e2e/E2ETest.kt

@@ -123,6 +123,62 @@ class E2ETest {
         assertThat(actual).isEqualTo(expected)
     }
 
+    @Test
+    fun testFineGrainedHashBzlMod() {
+        // The difference between these two snapshots is simply upgrading the Guava version.
+        // Following is the diff. (The diff on maven_install.json is omitted)
+        //
+        // diff --git a/MODULE.bazel b/MODULE.bazel
+        //         index 9a58823..3ffded3 100644
+        // --- a/MODULE.bazel
+        // +++ b/MODULE.bazel
+        // @@ -4,7 +4,7 @@ maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
+        // maven.install(
+        //         artifacts = [
+        //             "junit:junit:4.12",
+        //             -        "com.google.guava:guava:31.1-jre",
+        //             +        "com.google.guava:guava:32.0.0-jre",
+        //         ],
+        //         lock_file = "//:maven_install.json",
+        //         repositories = [
+        //
+        // The project contains a single target that depends on Guava:
+        // //src/main/java/com/integration:guava-user
+        //
+        // So this target, its derived targets, and all other changed external targets should be
+        // the only impacted targets.
+        val projectA = extractFixtureProject("/fixture/fine-grained-hash-bzlmod-test-1.zip")
+        val projectB = extractFixtureProject("/fixture/fine-grained-hash-bzlmod-test-2.zip")
+
+        val workingDirectoryA = projectA
+        val workingDirectoryB = projectB
+        val bazelPath = "bazel"
+        val outputDir = temp.newFolder()
+        val from = File(outputDir, "starting_hashes.json")
+        val to = File(outputDir, "final_hashes.json")
+        val impactedTargetsOutput = File(outputDir, "impacted_targets.txt")
+
+        val cli = CommandLine(BazelDiff())
+        //From
+        cli.execute(
+                "generate-hashes", "-w", workingDirectoryA.absolutePath, "-b", bazelPath, "--fineGrainedHashExternalRepos", "bazel_diff_maven", from.absolutePath
+        )
+        //To
+        cli.execute(
+                "generate-hashes", "-w", workingDirectoryB.absolutePath, "-b", bazelPath, "--fineGrainedHashExternalRepos", "bazel_diff_maven", to.absolutePath
+        )
+        //Impacted targets
+        cli.execute(
+                "get-impacted-targets", "-sh", from.absolutePath, "-fh", to.absolutePath, "-o", impactedTargetsOutput.absolutePath
+        )
+
+        val actual: Set<String> = impactedTargetsOutput.readLines().filter { it.isNotBlank() }.toSet()
+        val expected: Set<String> =
+                javaClass.getResourceAsStream("/fixture/fine-grained-hash-bzlmod-test-impacted-targets.txt").use { it.bufferedReader().readLines().filter { it.isNotBlank() }.toSet() }
+
+        assertThat(actual).isEqualTo(expected)
+    }
+
     // TODO: re-enable the test after https://github.com/bazelbuild/bazel/issues/21010 is fixed
     @Ignore("cquery mode is broken with Bazel 7 because --transition=lite is crashes due to https://github.com/bazelbuild/bazel/issues/21010")
     @Test

cli/src/test/kotlin/com/bazel_diff/hash/SourceFileHasherTest.kt

@@ -15,12 +15,13 @@ import java.nio.file.Files
 import java.nio.file.Paths
 
 
-internal class SourceFileHasherTest: KoinTest {
+internal class SourceFileHasherTest : KoinTest {
     private val repoAbsolutePath = Paths.get("").toAbsolutePath()
     private val outputBasePath = Files.createTempDirectory("SourceFileHasherTest")
     private val fixtureFileTarget = "//cli/src/test/kotlin/com/bazel_diff/hash/fixture:foo.ts"
     private val fixtureFileContent: ByteArray
     private val seed = "seed".toByteArray()
+    private val externalRepoResolver = ExternalRepoResolver(repoAbsolutePath, Paths.get("bazel"), outputBasePath)
 
     init {
         val path = Paths.get("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts")
@@ -35,7 +36,7 @@ internal class SourceFileHasherTest: KoinTest {
 
     @Test
     fun testHashConcreteFile() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -48,7 +49,7 @@ internal class SourceFileHasherTest: KoinTest {
 
     @Test
     fun testHashConcreteFileInExternalRepo() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null, setOf("external_repo"))
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver, setOf("external_repo"))
         val externalRepoFilePath = outputBasePath.resolve("external/external_repo/path/to/my_file.txt")
         Files.createDirectories(externalRepoFilePath.parent)
         val externalRepoFileTarget = "@external_repo//path/to:my_file.txt"
@@ -66,7 +67,7 @@ internal class SourceFileHasherTest: KoinTest {
 
     @Test
     fun testSoftHashConcreteFile() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.softDigest(bazelSourceFileTarget)?.toHexString()
         val expected = sha256 {
@@ -79,7 +80,7 @@ internal class SourceFileHasherTest: KoinTest {
 
     @Test
     fun testSoftHashNonExistedFile() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget("//i/do/not/exist", seed)
         val actual = hasher.softDigest(bazelSourceFileTarget)
         assertThat(actual).isNull()
@@ -88,7 +89,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testSoftHashExternalTarget() = runBlocking {
         val target = "@bazel-diff//some:file"
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
         val actual = hasher.softDigest(bazelSourceFileTarget)
         assertThat(actual).isNull()
@@ -97,7 +98,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashNonExistedFile() = runBlocking {
         val target = "//i/do/not/exist"
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -110,7 +111,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashExternalTarget() = runBlocking {
         val target = "@bazel-diff//some:file"
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {}.toHexString()
@@ -120,7 +121,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashWithProvidedContentHash() = runBlocking {
         val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts" to "foo-content-hash")
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
+        val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -134,7 +135,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashWithProvidedContentHashButNotInKey() = runBlocking {
         val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
+        val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -149,7 +150,7 @@ internal class SourceFileHasherTest: KoinTest {
     fun testHashWithProvidedContentHashWithLeadingColon() = runBlocking {
         val targetName = "//:cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts"
         val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
-        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
+        val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash, externalRepoResolver)
         val bazelSourceFileTarget = BazelSourceFileTarget(targetName, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {