Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions cli/src/main/kotlin/com/bazel_diff/bazel/BazelModService.kt
Comment thread
lukasmi93 marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
package com.bazel_diff.bazel

import com.bazel_diff.extensions.toHexString
import com.bazel_diff.hash.sha256
import com.bazel_diff.log.Logger
import com.bazel_diff.process.Redirect
import com.bazel_diff.process.process
import java.io.File
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.runBlocking
Expand Down Expand Up @@ -109,6 +113,133 @@ class BazelModService(
}
}

/**
* Computes a stable fingerprint of the currently resolved external dependency state.
*
* The hash includes:
* - bzlmod mode marker + `bazel mod graph --output=json`
* - repository-definition bytes from `bazel mod show_repo` (streamed proto when available)
*/
suspend fun getDependencyFingerprint(): String? {
if (!isBzlmodEnabled) {
return sha256 { putBytes("mode:legacy".toByteArray(StandardCharsets.UTF_8)) }.toHexString()
}

val moduleGraphJson = getModuleGraphJson() ?: ""
val canonicalRepos = discoverCanonicalBzlmodRepos()

val streamedShowRepo = showRepoStreamedProto(canonicalRepos)
if (streamedShowRepo != null && streamedShowRepo.exitCode == 0) {
return sha256 {
putBytes("mode:bzlmod\n".toByteArray(StandardCharsets.UTF_8))
putBytes("moduleGraphJson:".toByteArray(StandardCharsets.UTF_8))
putBytes(moduleGraphJson.toByteArray(StandardCharsets.UTF_8))
putBytes("\nshowRepo:\n".toByteArray(StandardCharsets.UTF_8))
putBytes(streamedShowRepo.stdout)
}
.toHexString()
}

val showRepoText = resolveShowRepoTextFallback(canonicalRepos) ?: return null
return sha256 {
putBytes("mode:bzlmod\n".toByteArray(StandardCharsets.UTF_8))
putBytes("moduleGraphJson:".toByteArray(StandardCharsets.UTF_8))
putBytes(moduleGraphJson.toByteArray(StandardCharsets.UTF_8))
putBytes("\nshowRepoText:\n".toByteArray(StandardCharsets.UTF_8))
putBytes(showRepoText.toByteArray(StandardCharsets.UTF_8))
}
.toHexString()
}

/**
* Returns canonical bzlmod repo names in @@<canonical> form, discovered from
* `bazel mod dump_repo_mapping ""`.
*/
private fun discoverCanonicalBzlmodRepos(): List<String> {
val output = runBazelRaw(listOf("mod", "dump_repo_mapping", "")) ?: return emptyList()
if (output.exitCode != 0) {
return emptyList()
}
return String(output.stdout, StandardCharsets.UTF_8)
.lineSequence()
.mapNotNull { line -> parseCanonicalRepoNames(line) }
.flatten()
.filter { it.contains('+') || it.contains('~') }
.map { "@@$it" }
.toSet()
.sorted()
}

private fun parseCanonicalRepoNames(line: String): List<String>? {
val parsed = runCatching {
@Suppress("UNCHECKED_CAST")
com.google.gson.Gson().fromJson(line.trim(), Map::class.java) as Map<String, Any?>
}
if (parsed.isFailure) return null
return parsed.getOrNull()?.values?.mapNotNull { it as? String }
}

private fun showRepoStreamedProto(canonicalRepos: List<String>): RawCommandResult? {
val args = mutableListOf("mod", "show_repo")
if (canonicalRepos.isNotEmpty()) {
args.addAll(canonicalRepos)
}
args.add("--output=streamed_proto")
return runBazelRaw(args)
}

private fun resolveShowRepoTextFallback(canonicalRepos: List<String>): String? {
val allVisible = runBazelRaw(listOf("mod", "show_repo", "--all_visible_repos", "--output=text"))
if (allVisible != null && allVisible.exitCode == 0) {
return String(allVisible.stdout, StandardCharsets.UTF_8)
}

val args = mutableListOf("mod", "show_repo")
if (canonicalRepos.isNotEmpty()) {
args.addAll(canonicalRepos)
}
args.add("--output=text")
val fallback = runBazelRaw(args) ?: return null
if (fallback.exitCode != 0) {
return null
}
return String(fallback.stdout, StandardCharsets.UTF_8)
}

private data class RawCommandResult(
val exitCode: Int,
val stdout: ByteArray,
)

private fun runBazelRaw(args: List<String>): RawCommandResult? {
val command =
mutableListOf<String>().apply {
add(bazelPath.toString())
if (noBazelrc) {
add("--bazelrc=/dev/null")
}
addAll(startupOptions)
addAll(args)
}
return try {
val nullDevice = if (System.getProperty("os.name").startsWith("Windows")) "NUL" else "/dev/null"
val process =
ProcessBuilder(command)
.directory(workingDirectory.toFile())
.redirectError(ProcessBuilder.Redirect.to(File(nullDevice)))
.start()
val stdout = process.inputStream.readBytes()
val exitCode = process.waitFor()
if (exitCode != 0) {
logger.w { "Command failed (exit=$exitCode): ${command.joinToString(" ")}" }
}
RawCommandResult(exitCode = exitCode, stdout = stdout)
} catch (e: Exception) {
logger.w { "Failed to execute ${command.joinToString(" ")}: ${e.message}" }
null
}
}

@OptIn(ExperimentalCoroutinesApi::class)
private suspend fun checkBzlmodEnabled(): Boolean {
val cmd =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ import org.koin.core.component.inject
data class HashFileData(
val hashes: Map<String, TargetHash>,
val moduleGraphJson: String?,
val depEdges: Map<String, List<String>> = emptyMap()
val depEdges: Map<String, List<String>> = emptyMap(),
val dependencyFingerprint: String? = null,
)

class DeserialiseHashesInteractor : KoinComponent {
Expand Down Expand Up @@ -46,6 +47,7 @@ class DeserialiseHashesInteractor : KoinComponent {

val metadata = jsonObject.getAsJsonObject("metadata")
val moduleGraphJson = metadata?.get("moduleGraphJson")?.asString
val dependencyFingerprint = metadata?.get("dependencyFingerprint")?.asString

// The query service persists the dependency-edge adjacency list (label -> direct dep labels)
// under metadata.depEdges when started with --trackDeps, so build-graph distance metrics can
Expand All @@ -56,7 +58,7 @@ class DeserialiseHashesInteractor : KoinComponent {
gson.fromJson<Map<String, List<String>>>(it, depShape)
} ?: emptyMap()

return HashFileData(hashes, moduleGraphJson, depEdges)
return HashFileData(hashes, moduleGraphJson, depEdges, dependencyFingerprint)
} else {
// Legacy format - just a flat map of hashes
val shape = object : TypeToken<Map<String, String>>() {}.type
Expand Down
69 changes: 48 additions & 21 deletions cli/src/main/kotlin/com/bazel_diff/server/HashService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ class HashService(
val generation: HashGenerationBreakdown? = null,
)

private data class GuardState(
var currentDependencyFingerprint: String? = null,
var computed: Boolean = false,
)

override fun getHashes(
sha: String,
modifiedFilepaths: Set<Path>,
Expand All @@ -135,20 +140,11 @@ class HashService(

/**
* Returns the hash data plus whether it was served from the cache. "Hit" includes the
* waited-behind-another-generation case (the after-lock re-check): this request itself ran no
* checkout/query, though its duration then includes the lock wait.
* waited-behind-another-generation case (the after-lock re-check). Cache hits may still do a
* checkout to validate the dependency fingerprint, but they do not rerun the hasher.
*/
private fun retrieve(sha: String, modifiedFilepaths: Set<Path>): Retrieval {
val key = cacheKey(sha, modifiedFilepaths)
val readStartNanos = System.nanoTime()
storage.get(key)?.let { bytes ->
val data =
deserialiser.executeTargetHashWithMetadataFromString(
String(bytes, StandardCharsets.UTF_8))
val readMillis = elapsedMillis(readStartNanos)
logger.i { "Hash cache hit for $sha (read+deserialize ${readMillis}ms)" }
return Retrieval(data, cacheHit = true, cacheReadMillis = readMillis)
}
return generate(sha, modifiedFilepaths, key)
}

Expand All @@ -160,6 +156,7 @@ class HashService(

private fun generate(sha: String, modifiedFilepaths: Set<Path>, key: String): Retrieval {
val lockStartNanos = System.nanoTime()
val guard = GuardState()
synchronized(generationLock) {
val lockWaitMillis = elapsedMillis(lockStartNanos)
// Re-check under the lock: another thread may have generated this revision while we waited.
Expand All @@ -168,17 +165,19 @@ class HashService(
val data =
deserialiser.executeTargetHashWithMetadataFromString(String(it, StandardCharsets.UTF_8))
val readMillis = elapsedMillis(readStartNanos)
logger.i {
"Hash cache hit for $sha (after ${lockWaitMillis}ms lock wait, " +
"read+deserialize ${readMillis}ms)"
if (cacheEntryMatchesDependencyFingerprint(sha, data, guard)) {
logger.i {
"Hash cache hit for $sha (after ${lockWaitMillis}ms lock wait, " +
"read+deserialize ${readMillis}ms)"
}
return Retrieval(
data, cacheHit = true, lockWaitMillis = lockWaitMillis, cacheReadMillis = readMillis)
}
return Retrieval(
data, cacheHit = true, lockWaitMillis = lockWaitMillis, cacheReadMillis = readMillis)
}
logger.i { "Hash cache miss for $sha - generating hashes" }

val checkoutStartNanos = System.nanoTime()
gitClient.checkout(sha)
ensureWorkspaceAndFingerprintForSha(sha, guard)
val checkoutMillis = elapsedMillis(checkoutStartNanos)

val hasherTimings = HasherPhaseTimings()
Expand All @@ -192,8 +191,12 @@ class HashService(

val writeStartNanos = System.nanoTime()
val depEdges = depEdgesOf(hashes)
val dependencyFingerprint =
guard.currentDependencyFingerprint ?: runBlocking { bazelModService.getDependencyFingerprint() }
storage.put(
key, serialize(hashes, moduleGraphJson, depEdges).toByteArray(StandardCharsets.UTF_8))
key,
serialize(hashes, moduleGraphJson, depEdges, dependencyFingerprint)
.toByteArray(StandardCharsets.UTF_8))
val cacheWriteMillis = elapsedMillis(writeStartNanos)

val breakdown =
Expand All @@ -218,13 +221,35 @@ class HashService(
"targets=${hashes.size}"
}
return Retrieval(
HashFileData(hashes, moduleGraphJson, depEdges),
HashFileData(hashes, moduleGraphJson, depEdges, dependencyFingerprint),
cacheHit = false,
lockWaitMillis = lockWaitMillis,
generation = breakdown)
}
}

private fun cacheEntryMatchesDependencyFingerprint(
sha: String,
data: HashFileData,
guard: GuardState,
): Boolean {
val cachedFingerprint = data.dependencyFingerprint
if (cachedFingerprint == null) {
return false
}
ensureWorkspaceAndFingerprintForSha(sha, guard)
val current = guard.currentDependencyFingerprint
return current != null && current == cachedFingerprint
}

private fun ensureWorkspaceAndFingerprintForSha(sha: String, guard: GuardState) {
if (guard.computed) return
gitClient.checkout(sha)
guard.currentDependencyFingerprint = runBlocking { bazelModService.getDependencyFingerprint() }
guard.computed = true
}


private fun elapsedMillis(startNanos: Long): Long = (System.nanoTime() - startNanos) / 1_000_000

/**
Expand All @@ -245,14 +270,16 @@ class HashService(
private fun serialize(
hashes: Map<String, TargetHash>,
moduleGraphJson: String?,
depEdges: Map<String, List<String>>
depEdges: Map<String, List<String>>,
dependencyFingerprint: String?,
): String {
val serializedHashes = hashes.mapValues { it.value.toJson(true) }
val output =
if (moduleGraphJson != null || depEdges.isNotEmpty()) {
if (moduleGraphJson != null || depEdges.isNotEmpty() || dependencyFingerprint != null) {
val metadata = mutableMapOf<String, Any>()
if (moduleGraphJson != null) metadata["moduleGraphJson"] = moduleGraphJson
if (depEdges.isNotEmpty()) metadata["depEdges"] = depEdges
if (dependencyFingerprint != null) metadata["dependencyFingerprint"] = dependencyFingerprint
mapOf("hashes" to serializedHashes, "metadata" to metadata)
} else {
serializedHashes
Expand Down
20 changes: 16 additions & 4 deletions cli/src/test/kotlin/com/bazel_diff/server/HashServiceTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ class HashServiceTest : KoinTest {
private fun newService(git: GitClient, storage: HashCacheStorage, trackDeps: Boolean = false) =
HashService(git, storage, "fp", emptySet(), emptySet(), trackDeps)

private fun stubDependencyFingerprint(value: String = "dep-fp") {
runBlocking { whenever(bazelModService.getDependencyFingerprint()).thenReturn(value) }
}

@Test
fun cacheMissGeneratesAndStores() {
whenever(buildGraphHasher.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull()))
Expand Down Expand Up @@ -144,6 +148,7 @@ class HashServiceTest : KoinTest {
whenever(buildGraphHasher.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull()))
.thenReturn(sampleHashes)
runBlocking { whenever(bazelModService.getModuleGraphJson()).thenReturn(null) }
stubDependencyFingerprint()
val git = RecordingGitClient()
val storage = InMemoryStorage()
val service = newService(git, storage)
Expand All @@ -152,8 +157,9 @@ class HashServiceTest : KoinTest {
val second = service.getHashes("sha1")

assertThat(second.hashes).isEqualTo(sampleHashes)
// Only the first call touches the workspace / runs the hasher.
assertThat(git.checkouts).isEqualTo(listOf("sha1"))
// Cache hits still checkout to validate the dependency fingerprint, but they do not rerun the
// hasher.
assertThat(git.checkouts).isEqualTo(listOf("sha1", "sha1"))
verify(buildGraphHasher, times(1))
.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull())
}
Expand All @@ -163,6 +169,7 @@ class HashServiceTest : KoinTest {
whenever(buildGraphHasher.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull()))
.thenReturn(sampleHashes)
runBlocking { whenever(bazelModService.getModuleGraphJson()).thenReturn(null) }
stubDependencyFingerprint()
val service = newService(RecordingGitClient(), InMemoryStorage())

val missProfiler = QueryProfiler()
Expand Down Expand Up @@ -204,6 +211,7 @@ class HashServiceTest : KoinTest {
whenever(buildGraphHasher.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull()))
.thenReturn(sampleHashes)
runBlocking { whenever(bazelModService.getModuleGraphJson()).thenReturn("""{"graph":1}""") }
stubDependencyFingerprint()
val storage = InMemoryStorage()

// Generate once, then read back through a fresh service over the same storage (cache hit path).
Expand All @@ -219,6 +227,7 @@ class HashServiceTest : KoinTest {
whenever(buildGraphHasher.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull()))
.thenReturn(sampleHashesWithDeps)
runBlocking { whenever(bazelModService.getModuleGraphJson()).thenReturn(null) }
stubDependencyFingerprint()
val storage = InMemoryStorage()

val generated = newService(RecordingGitClient(), storage, trackDeps = true).getHashes("sha1")
Expand Down Expand Up @@ -281,6 +290,8 @@ class HashServiceTest : KoinTest {

@Test
fun deserializeLegacyFlatCacheEntry() {
whenever(buildGraphHasher.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull()))
.thenReturn(sampleHashes)
val storage = InMemoryStorage()
storage.entries["sha1.fp"] = """{"//:a":"Rule#h~d"}""".toByteArray(StandardCharsets.UTF_8)

Expand All @@ -289,8 +300,8 @@ class HashServiceTest : KoinTest {
assertThat(data.hashes).isEqualTo(sampleHashes)
assertThat(data.moduleGraphJson).isNull()
assertThat(data.depEdges).isEqualTo(emptyMap())
// No generation on a pure cache hit.
verify(buildGraphHasher, times(0))
// Legacy cache entries without dependencyFingerprint are now recomputed.
verify(buildGraphHasher, times(1))
.hashAllBazelTargetsAndSourcefiles(any(), any(), any(), anyOrNull())
}

Expand All @@ -307,6 +318,7 @@ class HashServiceTest : KoinTest {
sampleHashes
}
runBlocking { whenever(bazelModService.getModuleGraphJson()).thenReturn(null) }
stubDependencyFingerprint()

val service = newService(RecordingGitClient(), InMemoryStorage())
val missDone = CountDownLatch(1)
Expand Down
Loading
Loading