Add --fineGrainedHashExternalRepos

cli/src/main/kotlin/com/bazel_diff/bazel/BazelClient.kt

@@ -4,16 +4,17 @@ import com.bazel_diff.log.Logger
 import com.google.devtools.build.lib.query2.proto.proto2api.Build
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
-import java.util.concurrent.ConcurrentMap
-import java.util.Calendar
+import java.util.*
 
-class BazelClient : KoinComponent {
+class BazelClient(private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
     private val logger: Logger by inject()
     private val queryService: BazelQueryService by inject()
 
     suspend fun queryAllTargets(): List<BazelTarget> {
         val queryEpoch = Calendar.getInstance().getTimeInMillis()
-        val targets = queryService.query("'//external:all-targets' + '//...:all-targets'")
+
+        val query = listOf("//external:all-targets", "//...:all-targets") + fineGrainedHashExternalRepos.map { "@$it//...:all-targets" }
+        val targets = queryService.query(query.joinToString(" + ") { "'$it'" })
         val queryDuration = Calendar.getInstance().getTimeInMillis() - queryEpoch
         logger.i { "All targets queried in $queryDuration" }
         return targets.mapNotNull { target: Build.Target ->
@@ -22,9 +23,11 @@ class BazelClient : KoinComponent {
                 Build.Target.Discriminator.SOURCE_FILE -> BazelTarget.SourceFile(
                     target
                 )
+
                 Build.Target.Discriminator.GENERATED_FILE -> BazelTarget.GeneratedFile(
                     target
                 )
+
                 else -> {
                     logger.w { "Unsupported target type in the build graph: ${target.type.name}" }
                     null

cli/src/main/kotlin/com/bazel_diff/bazel/BazelRule.kt

@@ -21,13 +21,15 @@ class BazelRule(private val rule: Build.Rule) {
             }
         }
     }
-    val ruleInputList: List<String>
-        get() = rule.ruleInputList.map { ruleInput: String -> transformRuleInput(ruleInput) }
+
+    fun ruleInputList(fineGrainedHashExternalRepos: Set<String>): List<String> {
+        return rule.ruleInputList.map { ruleInput: String -> transformRuleInput(fineGrainedHashExternalRepos, ruleInput) }
+    }
 
     val name: String = rule.name
 
-    private fun transformRuleInput(ruleInput: String): String {
-        if (ruleInput.startsWith("@")) {
+    private fun transformRuleInput(fineGrainedHashExternalRepos: Set<String>, ruleInput: String): String {
+        if (ruleInput.startsWith("@") && fineGrainedHashExternalRepos.none { ruleInput.startsWith("@$it") }) {
             val splitRule = ruleInput.split("//".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
             if (splitRule.size == 2) {
                 var externalRule = splitRule[0]

cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt

@@ -1,5 +1,6 @@
 package com.bazel_diff.cli
 
+import com.bazel_diff.cli.converter.CommaSeparatedValueConverter
 import com.bazel_diff.cli.converter.NormalisingPathConverter
 import com.bazel_diff.cli.converter.OptionsConverter
 import com.bazel_diff.di.hasherModule
@@ -64,6 +65,14 @@ class GenerateHashesCommand : Callable<Int> {
     )
     var bazelCommandOptions: List<String> = emptyList()
 
+    @CommandLine.Option(
+        names = ["--fineGrainedHashExternalRepos"],
+        description = ["Comma separate list of external repos in which fine-grained hashes are computed for the targets. By default, external repos are treated as an opaque blob. If an external repo is specified here, bazel-diff instead computes the hash for individual targets. For example, one wants to specify `maven` here if they user rules_jvm_external so that individual third party dependency change won't invalidate all targets in the mono repo."],
+        scope = CommandLine.ScopeType.INHERIT,
+        converter = [CommaSeparatedValueConverter::class],
+    )
+    var fineGrainedHashExternalRepos: Set<String> = emptySet()
+
     @CommandLine.Option(
         names = ["-k", "--keep_going"],
         negatable = true,
@@ -89,7 +98,7 @@ class GenerateHashesCommand : Callable<Int> {
     lateinit var spec: CommandLine.Model.CommandSpec
 
     override fun call(): Int {
-        validate(contentHashPath=contentHashPath)
+        validate(contentHashPath = contentHashPath)
 
         startKoin {
             modules(
@@ -100,6 +109,7 @@ class GenerateHashesCommand : Callable<Int> {
                     bazelStartupOptions,
                     bazelCommandOptions,
                     keepGoing,
+                    fineGrainedHashExternalRepos,
                 ),
                 loggingModule(parent.verbose),
                 serialisationModule(),

cli/src/main/kotlin/com/bazel_diff/cli/converter/CommaSeparatedValueConverter.kt

@@ -0,0 +1,9 @@
+package com.bazel_diff.cli.converter
+
+import picocli.CommandLine.ITypeConverter
+
+class CommaSeparatedValueConverter : ITypeConverter<Set<String>> {
+    override fun convert(value: String): Set<String> {
+        return value.split(",").dropLastWhile { it.isEmpty() }.toSet()
+    }
+}

cli/src/main/kotlin/com/bazel_diff/di/Modules.kt

@@ -23,6 +23,7 @@ fun hasherModule(
     startupOptions: List<String>,
     commandOptions: List<String>,
     keepGoing: Boolean?,
+    fineGrainedHashExternalRepos: Set<String>,
 ): Module = module {
     val debug = System.getProperty("DEBUG", "false").equals("true")
     single {
@@ -35,10 +36,10 @@ fun hasherModule(
             debug
         )
     }
-    single { BazelClient() }
+    single { BazelClient(fineGrainedHashExternalRepos) }
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
-    single { RuleHasher() }
+    single { RuleHasher(fineGrainedHashExternalRepos) }
     single { SourceFileHasher() }
     single(named("working-directory")) { workingDirectory }
     single { ContentHashProvider(contentHashPath) }

cli/src/main/kotlin/com/bazel_diff/hash/RuleHasher.kt

@@ -8,9 +8,10 @@ import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
 import java.util.concurrent.ConcurrentMap
 
-class RuleHasher : KoinComponent {
+class RuleHasher(private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
     private val logger: Logger by inject()
     private val sourceFileHasher: SourceFileHasher by inject()
+
     @VisibleForTesting
     class CircularDependencyException(message: String) : Exception(message)
 
@@ -42,14 +43,15 @@ class RuleHasher : KoinComponent {
             safePutBytes(rule.digest)
             safePutBytes(seedHash)
 
-            for (ruleInput in rule.ruleInputList) {
+            for (ruleInput in rule.ruleInputList(fineGrainedHashExternalRepos)) {
                 safePutBytes(ruleInput.toByteArray())
 
                 val inputRule = allRulesMap[ruleInput]
                 when {
                     inputRule == null && sourceDigests.containsKey(ruleInput) -> {
                         safePutBytes(sourceDigests[ruleInput])
                     }
+
                     inputRule?.name != null && inputRule.name != rule.name -> {
                         val ruleInputHash = digest(
                             inputRule,
@@ -61,6 +63,7 @@ class RuleHasher : KoinComponent {
                         )
                         safePutBytes(ruleInputHash)
                     }
+
                     else -> {
                         val heuristicDigest = sourceFileHasher.softDigest(BazelSourceFileTarget(ruleInput, ByteArray(0)))
                         when {
@@ -69,6 +72,7 @@ class RuleHasher : KoinComponent {
                                 sourceDigests[ruleInput] = heuristicDigest
                                 safePutBytes(heuristicDigest)
                             }
+
                             else -> logger.w { "Unable to calculate digest for input $ruleInput for rule ${rule.name}" }
                         }
                     }

cli/src/test/kotlin/com/bazel_diff/Modules.kt

@@ -15,10 +15,10 @@ import java.nio.file.Paths
 
 fun testModule(): Module = module {
     single<Logger> { SilentLogger }
-    single { BazelClient() }
+    single { BazelClient(emptySet()) }
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
-    single { RuleHasher() }
+    single { RuleHasher(emptySet()) }
     single { SourceFileHasher() }
     single { GsonBuilder().setPrettyPrinting().create() }
     single(named("working-directory")) { Paths.get("working-directory") }

cli/src/test/kotlin/com/bazel_diff/hash/BuildGraphHasherTest.kt

@@ -1,10 +1,8 @@
 package com.bazel_diff.hash
 
 import assertk.all
-import assertk.assertAll
 import assertk.assertThat
 import assertk.assertions.*
-import assertk.assertions.any
 import com.bazel_diff.bazel.BazelClient
 import com.bazel_diff.bazel.BazelRule
 import com.bazel_diff.bazel.BazelTarget
@@ -21,10 +19,8 @@ import org.koin.test.mock.MockProviderRule
 import org.koin.test.mock.declareMock
 import org.mockito.Mockito
 import org.mockito.junit.MockitoJUnit
-import org.mockito.kotlin.any
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.whenever
-import java.util.*
 
 
 class BuildGraphHasherTest : KoinTest {
@@ -148,7 +144,7 @@ class BuildGraphHasherTest : KoinTest {
             // they are run in parallel, so we don't know whether rule3 or rule4 will be processed first
             message().matchesPredicate {
                 it!!.contains("\\brule3 -> rule4 -> rule3\\b".toRegex()) ||
-                it.contains("\\brule4 -> rule3 -> rule4\\b".toRegex())
+                    it.contains("\\brule4 -> rule3 -> rule4\\b".toRegex())
             }
         }
     }
@@ -178,7 +174,7 @@ class BuildGraphHasherTest : KoinTest {
         val target = mock<BazelTarget.Rule>()
         val rule = mock<BazelRule>()
         whenever(rule.name).thenReturn(name)
-        whenever(rule.ruleInputList).thenReturn(inputs)
+        whenever(rule.ruleInputList(emptySet())).thenReturn(inputs)
         whenever(rule.digest).thenReturn(digest.toByteArray())
         whenever(target.rule).thenReturn(rule)
         whenever(target.name).thenReturn(name)