Improve stats querying performance (insertions/deletions/files change…

…d). Fixes immense lags for repos with very large files in its commits

Now first only the "files changed" amount is queried and the additions/deletions fixed after, with an upper limit on how many will be calculated in parallel.

Reason being that the insertions/deletions aren't cached by Git so it needs to read through the entire commit files.

src/global.d.ts

@@ -70,10 +70,13 @@ interface Commit {
 	refs: GitRef[]
 	subject: string
 	merge?: boolean
+	/** undefined means not yet queried, an empty object signifies a loading state,
+	only files_changed present means it went through name stats already and
+	full properties come after full retrieval  */
 	stats?: {
-		files_changed: number
-		insertions: number
-		deletions: number
+		files_changed?: number
+		insertions?: number
+		deletions?: number
 	}
 }
 

web/src/bridge.js

@@ -39,7 +39,7 @@ export let exchange_message = async (/** @type {string} */ command, /** @type {a
 	let resp = await new Promise((ok) => {
 		response_handlers[message_id_counter] = ok
 	})
-	console.info('exchange_message', command, data) // , resp
+	// console.info('exchange_message', command, data) // , resp
 
 	if (resp.error) {
 		let error = new Error(JSON.stringify({ error_response: resp.error, request }))

web/src/state/store.js

@@ -137,21 +137,56 @@ export let refresh_main_view = ({ before_execute } = {}) => {
 let is_updating_commit_stats = false
 /** @type {Commit[]} */
 let queued_commits_for_update_stats = []
-export let update_commit_stats = async (/** @type {Commit[]} */ commits_) => {
+export let update_commit_stats = async (/** @type {Commit[]} */ commits_, level = 0) => {
+	if (! commits_.length || level === 0 && /* probably heavily overloaded */queued_commits_for_update_stats.length > 120)
+		return
+	commits_.forEach(commit => commit.stats = {}) // Prevent from running them twice
+	update_commit_stats_fast(commits_) // async
 	if (is_updating_commit_stats)
 		return queued_commits_for_update_stats.push(...commits_)
 	is_updating_commit_stats = true
-	await work_update_commit_stats(commits_)
+	await update_commit_stats_full(commits_)
 	is_updating_commit_stats = false
 	if (queued_commits_for_update_stats.length) {
-		update_commit_stats(queued_commits_for_update_stats.filter(c => ! c.stats))
+		update_commit_stats(queued_commits_for_update_stats.filter(c => c.stats?.insertions == null), level + 1)
 		queued_commits_for_update_stats = []
 	}
 }
-/** Can be *very* slow in very big repos so it's important to keep it to a minimum */
-async function work_update_commit_stats(/** @type {Commit[]} */ commits_) {
-	if (! commits_.length)
+/** Only the `files_changed` properties are filled */
+async function update_commit_stats_fast(/** @type {Commit[]} */ commits_) {
+	// console.time('update_commit_stats_fast')
+	// --name-status prints all changed files below one another with M / D etc modifiers in front.
+	// Couldn't find any other way to just get the amount of changed files as fast as this.
+	let data = await git('show --format="%h" --name-status ' + commits_.map((c) => c.hash).join(' '))
+	if (! data)
 		return
+	let hash = ''
+	let files_changed = 0
+	for (let line of data.split('\n').filter(Boolean)/* ensure cleanup of last commit */.concat('pseudo-hash')) {
+		let [hash_or_file_stati, file_name] = line.split('\t')
+		if (hash_or_file_stati && ! file_name) {
+			if (hash) {
+				let commit = commits_[commits_.findIndex((cp) => cp.hash === hash)]
+				if (! commit) {
+					console.warn('Details for below error: Hash:', hash, 'Commits:', commits_, 'returned data:', data)
+					throw new Error(`Tried to retrieve fast commit stats but the returned hash '${hash}' can't be found in the commit listing`)
+				}
+				if (commit.stats?.files_changed) {
+					// update_commit_stats_full() finished earlier
+				} else
+					commit.stats = { files_changed }
+				files_changed = 0
+			}
+			hash = hash_or_file_stati
+			continue
+		}
+		files_changed++
+	}
+	// console.timeEnd('update_commit_stats_fast')
+}
+/** Can be *very* slow with commits with big files so it's important to keep it to a minimum */
+async function update_commit_stats_full(/** @type {Commit[]} */ commits_) {
+	// console.time('update_commit_stats_full')
 	let data = await git('show --format="%h" --shortstat ' + commits_.map((c) => c.hash).join(' '))
 	if (! data)
 		return
@@ -175,11 +210,16 @@ async function work_update_commit_stats(/** @type {Commit[]} */ commits_) {
 		let commit = commits_[commits_.findIndex((cp) => cp.hash === hash)]
 		if (! commit) {
 			// Happened once, but no idea why
-			console.warn('Failed to retrieve commit stats. Hash:', hash, 'Commits:', commits_, 'returned data:', data)
-			throw new Error(`Tried to retrieve commit stats but the returned hash '${hash}' can't be found in the commit listing`)
+			console.warn('Details for below error: Hash:', hash, 'Commits:', commits_, 'returned data:', data)
+			throw new Error(`Tried to retrieve full commit stats but the returned hash '${hash}' can't be found in the commit listing`)
 		}
+		if (! commit.merge && commit.stats?.files_changed !== stat.files_changed)
+			// Happens rarely for some reason
+			console.warn('Commit stats files_changed mismatch between fast and full mode. Fast: ', commit.stats?.files_changed, ', full:', stat.files_changed, ', commit: ', commit)
+
 		commit.stats = stat
 	}
+	// console.timeEnd('update_commit_stats_full')
 }
 
 /** @type {Vue.Ref<GitAction|null>} */
@@ -306,7 +346,7 @@ export let init = () => {
 		{ fetch_stash_refs: false, fetch_branches: false }).then((parsed) =>
 		commits.value = parsed.commits
 			.concat({ subject: '..........Loading more..........', author_email: '', hash: '-', index_in_graph_output: -1, vis_lines: [{ y0: 0.5, yn: 0.5, x0: 0, xn: 2000, branch: { color: 'yellow', type: 'branch', name: '', id: '' } }], author_name: '', hash_long: '', refs: [] })
-			.map(c => ({ ...c, stats: /* to prevent loading them */ { files_changed: 0, insertions: 0, deletions: 0 } })))
+			.map(c => ({ ...c, stats: /* to prevent loading them */ {} })))
 
 	add_push_listener('config-change', async () => {
 		await refresh_config()

web/src/views/CommitRow.vue

@@ -17,7 +17,7 @@
 			<div class="stats flex-noshrink row align-center justify-flex-end gap-5">
 				<template v-if="commit.stats?.files_changed">
 					<div class="changes" title="Changed lines in amount of files">
-						<span>
+						<span v-if="commit.stats.insertions != null">
 							<strong>{{ commit.stats.insertions + commit.stats.deletions }}</strong>
 						</span>
 						<span class="grey"> in </span>