From fa4bea8122c0546f947c450cf24895ab7663d99e Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sun, 5 Mar 2023 01:46:18 +0900 Subject: [PATCH 01/20] PoC: visualization for how far behind the lastest commit --- client/src/translations/differences/index.tsx | 22 ++++++++++ server/translations.ts | 44 ++++++++++++++----- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index fe48c97ca6d2..dda3355715de 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -35,6 +35,8 @@ interface DocumentEdits { parentModified: string; commitURL: string; parentCommitURL: string; + sourceCommitsBehindCount: number; + sourceCommitURL?: string; } interface Document { @@ -606,6 +608,7 @@ function DocumentsTable({ + @@ -658,6 +661,14 @@ function DocumentsTable({ {doc.differences.total.toLocaleString()} + + + ); })} @@ -680,6 +691,17 @@ function DocumentsTable({ ); } +function L10nSourceCommitModified({ + sourceCommitsBehindCount, + sourceCommitURL, +}: Pick) { + if (!sourceCommitURL) return null; + + return ( + {`${sourceCommitsBehindCount} commits behind`} + ); +} + function LastModified({ edits }: { edits: DocumentEdits }) { const modified = dayjs(edits.modified); const parentModified = dayjs(edits.parentModified); diff --git a/server/translations.ts b/server/translations.ts index c6c8190985f7..fb34f7e69ab0 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -3,6 +3,7 @@ import path from "node:path"; import express from "express"; import { fdir } from "fdir"; +import { execSync } from "node:child_process"; import { getPopularities, Document, Translation } from "../content/index.js"; import { @@ -141,22 +142,45 @@ function getDocument(filePath) { }; } + function getCommitBehindFromLatest(filename, commitHash) { + const commitHashes = execSync( + `cd ${CONTENT_ROOT} && git log --pretty=format:%H -- ${filename}` + ) + .toString() + .split("\n"); + + return commitHashes.indexOf(commitHash); + } + function packageEdits(document, parentDocument) { - const commitURL = getLastCommitURL( - document.fileInfo.root, - document.metadata.hash - ); - const parentCommitURL = getLastCommitURL( - parentDocument.fileInfo.root, - parentDocument.metadata.hash - ); - const modified = document.metadata.modified; - const parentModified = parentDocument.metadata.modified; + const { + fileInfo: { root: fileRoot }, + metadata: { hash: fileHash, modified, l10n }, + } = document; + const { + fileInfo: { root: parentFileRoot, path: parentFilePath }, + metadata: { hash: parentFileHash, parentModified }, + } = parentDocument; + + const commitURL = getLastCommitURL(fileRoot, fileHash); + const parentCommitURL = getLastCommitURL(parentFileRoot, parentFileHash); + let sourceCommitURL; + let sourceCommitsBehindCount; + if (l10n?.sourceCommit) { + sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit); + sourceCommitsBehindCount = getCommitBehindFromLatest( + parentFilePath, + l10n.sourceCommit + ); + } + return { commitURL, parentCommitURL, modified, parentModified, + sourceCommitURL, + sourceCommitsBehindCount, }; } From 6541ebc2931dce9e858f3ea7e33516a4d598da79 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sun, 5 Mar 2023 02:01:29 +0900 Subject: [PATCH 02/20] feat: tag for Metadata does not exist. --- client/src/translations/differences/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index dda3355715de..66e46317ee4b 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -695,7 +695,7 @@ function L10nSourceCommitModified({ sourceCommitsBehindCount, sourceCommitURL, }: Pick) { - if (!sourceCommitURL) return null; + if (!sourceCommitURL) return

Metadata does not exist.

; return ( {`${sourceCommitsBehindCount} commits behind`} From 19b57336415156883dd727273634e75562bf8037 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Mon, 6 Mar 2023 19:20:25 +0900 Subject: [PATCH 03/20] fix: filter available --- client/src/translations/differences/index.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index 66e46317ee4b..4abe478a1143 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -576,6 +576,10 @@ function DocumentsTable({ const a = A.mdn_url; const b = B.mdn_url; return reverse * a.localeCompare(b); + } else if (sort === "sourceCommit") { + const a = A.edits.sourceCommitsBehindCount || -1; + const b = B.edits.sourceCommitsBehindCount || -1; + return reverse * (b - a); } else { throw new Error(`Unrecognized sort '${sort}'`); } From 6c9af9b159a28b1159736f748f787c28558e4fbd Mon Sep 17 00:00:00 2001 From: hochan222 Date: Mon, 6 Mar 2023 19:22:43 +0900 Subject: [PATCH 04/20] refactor: execSync cwd option --- server/translations.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index fb34f7e69ab0..b13759f94767 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -143,9 +143,9 @@ function getDocument(filePath) { } function getCommitBehindFromLatest(filename, commitHash) { - const commitHashes = execSync( - `cd ${CONTENT_ROOT} && git log --pretty=format:%H -- ${filename}` - ) + const commitHashes = execSync(`git log --pretty=format:%H -- ${filename}`, { + cwd: CONTENT_ROOT, + }) .toString() .split("\n"); From 1f508709068de27d5e2f335fcc8770b6670cf9eb Mon Sep 17 00:00:00 2001 From: hochan222 Date: Mon, 6 Mar 2023 20:34:17 +0900 Subject: [PATCH 05/20] fix: filter priority --- client/src/translations/differences/index.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index 4abe478a1143..d93a57b81149 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -577,8 +577,11 @@ function DocumentsTable({ const b = B.mdn_url; return reverse * a.localeCompare(b); } else if (sort === "sourceCommit") { - const a = A.edits.sourceCommitsBehindCount || -1; - const b = B.edits.sourceCommitsBehindCount || -1; + const a = A.edits.sourceCommitsBehindCount; + const b = B.edits.sourceCommitsBehindCount; + if (b === undefined) { + return -1; + } return reverse * (b - a); } else { throw new Error(`Unrecognized sort '${sort}'`); From 36e4628d6b1ff24c1952011b9319fd5340f51b2b Mon Sep 17 00:00:00 2001 From: hochan222 Date: Mon, 6 Mar 2023 21:28:30 +0900 Subject: [PATCH 06/20] Optimization through cache --- server/translations.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index b13759f94767..85db48fa17c6 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -122,6 +122,8 @@ export function findDocuments({ locale }) { }; } +const commitHashCache = {}; + function getDocument(filePath) { function packagePopularity(document, parentDocument) { return { @@ -143,13 +145,17 @@ function getDocument(filePath) { } function getCommitBehindFromLatest(filename, commitHash) { - const commitHashes = execSync(`git log --pretty=format:%H -- ${filename}`, { - cwd: CONTENT_ROOT, - }) - .toString() - .split("\n"); - - return commitHashes.indexOf(commitHash); + if (commitHashCache[filename] === undefined) { + commitHashCache[filename] = execSync( + `git log --pretty=format:%H -- ${filename}`, + { + cwd: CONTENT_ROOT, + } + ) + .toString() + .split("\n"); + } + return commitHashCache[filename].indexOf(commitHash); } function packageEdits(document, parentDocument) { From 9ebd1a6185e918e29060b4598bb8efa49240804f Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 11 Mar 2023 20:49:27 +0900 Subject: [PATCH 07/20] reduce ja time to 214s with git rev-list --- server/translations.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index 85db48fa17c6..dba74b4e70a1 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -146,16 +146,19 @@ function getDocument(filePath) { function getCommitBehindFromLatest(filename, commitHash) { if (commitHashCache[filename] === undefined) { - commitHashCache[filename] = execSync( - `git log --pretty=format:%H -- ${filename}`, - { - cwd: CONTENT_ROOT, - } - ) - .toString() - .split("\n"); + try { + commitHashCache[filename] = execSync( + `git rev-list --count ${commitHash}..HEAD -- ${filename}`, + { + cwd: CONTENT_ROOT, + } + ).toString(); + } catch (err) { + // console.log(err) + } } - return commitHashCache[filename].indexOf(commitHash); + // console.log(`${filename}: ${commitHashCache[filename]}`) + return commitHashCache[filename]; } function packageEdits(document, parentDocument) { From 9d3d8f742d9e386502c7b2b24d4d4e7a5a7d2b07 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 11 Mar 2023 21:49:16 +0900 Subject: [PATCH 08/20] feat: source commit error report --- .gitignore | 1 + server/translations.ts | 33 +++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index c9444914f4f1..ce64f2c26264 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ testing/content/files/en-us/markdown/tool/m2h/index.html # eslintcache client/.eslintcache popularities.json +source-commit-report.txt client/src/.linaria_cache ssr/.linaria-cache/ diff --git a/server/translations.ts b/server/translations.ts index dba74b4e70a1..279794e6e372 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -144,26 +144,42 @@ function getDocument(filePath) { }; } - function getCommitBehindFromLatest(filename, commitHash) { - if (commitHashCache[filename] === undefined) { + function recordInvalidSourceCommit(str: string) { + const filePath = "./source-commit-report.txt"; + if (!fs.existsSync(filePath)) { + fs.writeFileSync(filePath, ""); + console.log(`File created!`); + } + + fs.appendFile(filePath, `${str}\n`, function (err) { + if (err) throw err; + }); + } + + function getCommitBehindFromLatest( + filePath: string, + parentFilePath: string, + commitHash: string + ) { + if (commitHashCache[filePath] === undefined) { try { - commitHashCache[filename] = execSync( - `git rev-list --count ${commitHash}..HEAD -- ${filename}`, + commitHashCache[filePath] = execSync( + `git rev-list --count ${commitHash}..HEAD -- ${parentFilePath}`, { cwd: CONTENT_ROOT, } ).toString(); } catch (err) { - // console.log(err) + recordInvalidSourceCommit(`${filePath}: ${commitHash}`); } } - // console.log(`${filename}: ${commitHashCache[filename]}`) - return commitHashCache[filename]; + + return commitHashCache[filePath]; } function packageEdits(document, parentDocument) { const { - fileInfo: { root: fileRoot }, + fileInfo: { root: fileRoot, path: filePath }, metadata: { hash: fileHash, modified, l10n }, } = document; const { @@ -178,6 +194,7 @@ function getDocument(filePath) { if (l10n?.sourceCommit) { sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit); sourceCommitsBehindCount = getCommitBehindFromLatest( + filePath, parentFilePath, l10n.sourceCommit ); From b04917e4b8ac45234500b6fc02dfed54988043c4 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 11 Mar 2023 23:46:54 +0900 Subject: [PATCH 09/20] Optimize source commit logic Generate source commit error report --- .gitignore | 1 + server/translations.ts | 46 ++++++++++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index ce64f2c26264..7dad957d6a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ testing/content/files/en-us/markdown/tool/m2h/index.html # eslintcache client/.eslintcache popularities.json +source-commit.json source-commit-report.txt client/src/.linaria_cache diff --git a/server/translations.ts b/server/translations.ts index 279794e6e372..e3a0f0df1c34 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -61,6 +61,13 @@ function packageTranslationDifferences(translationDifferences) { } const _foundDocumentsCache = new Map(); +const commitHashCache = fs.existsSync("./source-commit.json") + ? new Map( + Object.entries( + JSON.parse(fs.readFileSync("./source-commit.json", "utf8")) + ) + ) + : new Map(); export function findDocuments({ locale }) { const counts = { // Number of documents found that aren't skipped @@ -115,6 +122,12 @@ export function findDocuments({ locale }) { took, }; + fs.writeFileSync( + "./source-commit.json", + JSON.stringify(Object.fromEntries(commitHashCache)), + "utf8" + ); + return { counts, times, @@ -122,8 +135,6 @@ export function findDocuments({ locale }) { }; } -const commitHashCache = {}; - function getDocument(filePath) { function packagePopularity(document, parentDocument) { return { @@ -148,7 +159,6 @@ function getDocument(filePath) { const filePath = "./source-commit-report.txt"; if (!fs.existsSync(filePath)) { fs.writeFileSync(filePath, ""); - console.log(`File created!`); } fs.appendFile(filePath, `${str}\n`, function (err) { @@ -157,29 +167,34 @@ function getDocument(filePath) { } function getCommitBehindFromLatest( - filePath: string, + fileFolder: string, parentFilePath: string, commitHash: string ) { - if (commitHashCache[filePath] === undefined) { + if (!commitHashCache.has(fileFolder)) { try { - commitHashCache[filePath] = execSync( - `git rev-list --count ${commitHash}..HEAD -- ${parentFilePath}`, - { - cwd: CONTENT_ROOT, - } - ).toString(); + commitHashCache.set( + fileFolder, + execSync( + `git rev-list --count ${commitHash}..HEAD -- ${parentFilePath}`, + { + cwd: CONTENT_ROOT, + } + ) + .toString() + .trimEnd() + ); } catch (err) { - recordInvalidSourceCommit(`${filePath}: ${commitHash}`); + recordInvalidSourceCommit(`${fileFolder}: ${commitHash}`); } } - return commitHashCache[filePath]; + return commitHashCache.get(fileFolder); } function packageEdits(document, parentDocument) { const { - fileInfo: { root: fileRoot, path: filePath }, + fileInfo: { root: fileRoot, folder: fileFolder }, metadata: { hash: fileHash, modified, l10n }, } = document; const { @@ -191,10 +206,11 @@ function getDocument(filePath) { const parentCommitURL = getLastCommitURL(parentFileRoot, parentFileHash); let sourceCommitURL; let sourceCommitsBehindCount; + if (l10n?.sourceCommit) { sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit); sourceCommitsBehindCount = getCommitBehindFromLatest( - filePath, + fileFolder, parentFilePath, l10n.sourceCommit ); From 06e74e38f5ec1f0a3fea5e84c8fa3d2499c01d0d Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sun, 12 Mar 2023 00:44:55 +0900 Subject: [PATCH 10/20] feat: source commit importance color --- client/src/translations/differences/index.tsx | 9 ++++++++- server/translations.ts | 16 +++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index d93a57b81149..2144962b2400 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -704,8 +704,15 @@ function L10nSourceCommitModified({ }: Pick) { if (!sourceCommitURL) return

Metadata does not exist.

; + const getImportanceColor = () => { + if (sourceCommitsBehindCount === 0) return "🟢"; + return sourceCommitsBehindCount < 10 ? "🟠" : "🔴"; + }; + return ( - {`${sourceCommitsBehindCount} commits behind`} + {`${getImportanceColor()} ${sourceCommitsBehindCount} commits behind`} ); } diff --git a/server/translations.ts b/server/translations.ts index 9d63fa73a906..0519df316a0d 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -175,14 +175,16 @@ function getDocument(filePath) { try { commitHashCache.set( fileFolder, - execSync( - `git rev-list --count ${commitHash}..HEAD -- ${parentFilePath}`, - { - cwd: CONTENT_ROOT, - } + Number( + execSync( + `git rev-list --count ${commitHash}..HEAD -- ${parentFilePath}`, + { + cwd: CONTENT_ROOT, + } + ) + .toString() + .trimEnd() ) - .toString() - .trimEnd() ); } catch (err) { recordInvalidSourceCommit(`${fileFolder}: ${commitHash}`); From ad5246de5b8b09c84a2143d532f6ff869338759f Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 25 Mar 2023 18:46:36 +0900 Subject: [PATCH 11/20] new approach for traversing git commit graph Co-authored-by: LeoMcA --- server/translations.ts | 54 +++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index 0519df316a0d..45357bb7d1ab 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -68,6 +68,8 @@ const commitHashCache = fs.existsSync("./source-commit.json") ) ) : new Map(); +const memCommitStore = new Map(); +let memCommitStoreOldest = "HEAD"; export async function findDocuments({ locale }) { const counts = { // Number of documents found that aren't skipped @@ -155,8 +157,9 @@ function getDocument(filePath) { }; } - function recordInvalidSourceCommit(str: string) { + function recordInvalidSourceCommit(isValid: boolean, str: string) { const filePath = "./source-commit-report.txt"; + if (isValid) return; if (!fs.existsSync(filePath)) { fs.writeFileSync(filePath, ""); } @@ -171,24 +174,37 @@ function getDocument(filePath) { parentFilePath: string, commitHash: string ) { - if (!commitHashCache.has(fileFolder)) { - try { - commitHashCache.set( - fileFolder, - Number( - execSync( - `git rev-list --count ${commitHash}..HEAD -- ${parentFilePath}`, - { - cwd: CONTENT_ROOT, - } - ) - .toString() - .trimEnd() - ) - ); - } catch (err) { - recordInvalidSourceCommit(`${fileFolder}: ${commitHash}`); + try { + let count = 0; + if (!memCommitStore.has(commitHash)) { + execSync( + `git log --pretty=format:'%H' --name-only ${commitHash}..${memCommitStoreOldest}`, + { + cwd: CONTENT_ROOT, + } + ) + .toString() + .trimEnd() + .split("\n\n") + .forEach((commit) => { + const [hash, ...files] = commit.split("\n"); + memCommitStore.set(hash, files); + }); + memCommitStoreOldest = commitHash; + } + for (const [hash, files] of memCommitStore.entries()) { + if (hash === commitHash) { + recordInvalidSourceCommit( + files.includes(parentFilePath), + `${fileFolder}: ${commitHash}` + ); + break; + } + if (files.includes(parentFilePath)) count++; } + commitHashCache.set(fileFolder, count); + } catch (err) { + recordInvalidSourceCommit(false, `${fileFolder}: ${commitHash}`); } return commitHashCache.get(fileFolder); @@ -213,7 +229,7 @@ function getDocument(filePath) { sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit); sourceCommitsBehindCount = getCommitBehindFromLatest( fileFolder, - parentFilePath, + parentFilePath.replace(parentFileRoot, "files"), l10n.sourceCommit ); } From a71eae9fc7b8dc1981637863ba9e08135ef83e83 Mon Sep 17 00:00:00 2001 From: hochan Lee Date: Sat, 25 Mar 2023 18:52:35 +0900 Subject: [PATCH 12/20] remove p tag in table tag Co-authored-by: Leo McArdle --- client/src/translations/differences/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index 2144962b2400..5912a75d0861 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -702,7 +702,7 @@ function L10nSourceCommitModified({ sourceCommitsBehindCount, sourceCommitURL, }: Pick) { - if (!sourceCommitURL) return

Metadata does not exist.

; + if (!sourceCommitURL) return <>Metadata does not exist.; const getImportanceColor = () => { if (sourceCommitsBehindCount === 0) return "🟢"; From 153d68117c6cbf097d7585c5f0800f333b481d6b Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 25 Mar 2023 19:16:17 +0900 Subject: [PATCH 13/20] add details to report message and file name Co-authored-by: LeoMcA --- .gitignore | 2 +- server/translations.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7dad957d6a0b..29b2e3c9078f 100644 --- a/.gitignore +++ b/.gitignore @@ -104,7 +104,7 @@ testing/content/files/en-us/markdown/tool/m2h/index.html client/.eslintcache popularities.json source-commit.json -source-commit-report.txt +source-commit-invalid-report.txt client/src/.linaria_cache ssr/.linaria-cache/ diff --git a/server/translations.ts b/server/translations.ts index 45357bb7d1ab..bdb20fbe74ac 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -157,14 +157,19 @@ function getDocument(filePath) { }; } - function recordInvalidSourceCommit(isValid: boolean, str: string) { - const filePath = "./source-commit-report.txt"; + function recordInvalidSourceCommit( + isValid: boolean, + fileFolder: string, + commitHash: string + ) { + const filePath = "./source-commit-invalid-report.txt"; + const errorMessage = `- ${commitHash} commit hash is invalid in ${fileFolder}`; if (isValid) return; if (!fs.existsSync(filePath)) { fs.writeFileSync(filePath, ""); } - fs.appendFile(filePath, `${str}\n`, function (err) { + fs.appendFile(filePath, `${errorMessage}\n`, function (err) { if (err) throw err; }); } @@ -196,7 +201,8 @@ function getDocument(filePath) { if (hash === commitHash) { recordInvalidSourceCommit( files.includes(parentFilePath), - `${fileFolder}: ${commitHash}` + fileFolder, + commitHash ); break; } @@ -204,7 +210,7 @@ function getDocument(filePath) { } commitHashCache.set(fileFolder, count); } catch (err) { - recordInvalidSourceCommit(false, `${fileFolder}: ${commitHash}`); + recordInvalidSourceCommit(false, fileFolder, commitHash); } return commitHashCache.get(fileFolder); From 94eee57f1c8f2584e2b663e338560ea9011d7481 Mon Sep 17 00:00:00 2001 From: hochan Lee Date: Sat, 25 Mar 2023 19:36:43 +0900 Subject: [PATCH 14/20] modify source commit filter condition Co-authored-by: Leo McArdle --- client/src/translations/differences/index.tsx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index 5912a75d0861..673a208726f5 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -577,11 +577,8 @@ function DocumentsTable({ const b = B.mdn_url; return reverse * a.localeCompare(b); } else if (sort === "sourceCommit") { - const a = A.edits.sourceCommitsBehindCount; - const b = B.edits.sourceCommitsBehindCount; - if (b === undefined) { - return -1; - } + const a = A.edits.sourceCommitsBehindCount ?? -1; + const b = B.edits.sourceCommitsBehindCount ?? -1; return reverse * (b - a); } else { throw new Error(`Unrecognized sort '${sort}'`); From 2d2dba7f7ef2a11b1e34f9d06f9d3da20f5edd9f Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 25 Mar 2023 19:54:11 +0900 Subject: [PATCH 15/20] fix: consider sourceCommitsBehindCount undefined --- client/src/translations/differences/index.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/src/translations/differences/index.tsx b/client/src/translations/differences/index.tsx index 673a208726f5..bdd97d2654f6 100644 --- a/client/src/translations/differences/index.tsx +++ b/client/src/translations/differences/index.tsx @@ -35,7 +35,7 @@ interface DocumentEdits { parentModified: string; commitURL: string; parentCommitURL: string; - sourceCommitsBehindCount: number; + sourceCommitsBehindCount?: number; sourceCommitURL?: string; } @@ -699,7 +699,12 @@ function L10nSourceCommitModified({ sourceCommitsBehindCount, sourceCommitURL, }: Pick) { - if (!sourceCommitURL) return <>Metadata does not exist.; + if ( + !sourceCommitURL || + (!sourceCommitsBehindCount && sourceCommitsBehindCount !== 0) + ) { + return <>Metadata does not exist.; + } const getImportanceColor = () => { if (sourceCommitsBehindCount === 0) return "🟢"; From 032db13fb5ac6a1accf61bf7e01a9d639772eda9 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Mon, 27 Mar 2023 01:59:30 +0900 Subject: [PATCH 16/20] check automatic cache validation --- server/translations.ts | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index bdb20fbe74ac..829be5c4cf90 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -61,7 +61,7 @@ function packageTranslationDifferences(translationDifferences) { } const _foundDocumentsCache = new Map(); -const commitHashCache = fs.existsSync("./source-commit.json") +const sourceCommitCache = fs.existsSync("./source-commit.json") ? new Map( Object.entries( JSON.parse(fs.readFileSync("./source-commit.json", "utf8")) @@ -71,6 +71,32 @@ const commitHashCache = fs.existsSync("./source-commit.json") const memCommitStore = new Map(); let memCommitStoreOldest = "HEAD"; export async function findDocuments({ locale }) { + function checkCacheValidation(prevCache: Map): void { + const contentHash = getRecentRepoHash(CONTENT_ROOT); + const translatedContentHash = getRecentRepoHash(CONTENT_TRANSLATED_ROOT); + + function getRecentRepoHash(cwd: string): string { + return execSync("git rev-parse HEAD", { cwd }).toString().trimEnd(); + } + function isValidSourceCommitCache(): boolean { + return ( + prevCache.has(CONTENT_ROOT) && + prevCache.has(CONTENT_TRANSLATED_ROOT) && + prevCache.get(CONTENT_ROOT) === contentHash && + prevCache.get(CONTENT_TRANSLATED_ROOT) === translatedContentHash + ); + } + + if (!isValidSourceCommitCache()) { + prevCache.clear(); + prevCache.set(CONTENT_ROOT, contentHash); + prevCache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash); + sourceCommitCache.clear(); + memCommitStore.clear(); + memCommitStoreOldest = "HEAD"; + } + } + const counts = { // Number of documents found that aren't skipped found: 0, @@ -91,6 +117,7 @@ export async function findDocuments({ locale }) { }); counts.total = documentsFound.count; + checkCacheValidation(_foundDocumentsCache); if (!_foundDocumentsCache.has(locale)) { _foundDocumentsCache.set(locale, new Map()); } @@ -126,7 +153,7 @@ export async function findDocuments({ locale }) { fs.writeFileSync( "./source-commit.json", - JSON.stringify(Object.fromEntries(commitHashCache)), + JSON.stringify(Object.fromEntries(sourceCommitCache)), "utf8" ); @@ -208,12 +235,12 @@ function getDocument(filePath) { } if (files.includes(parentFilePath)) count++; } - commitHashCache.set(fileFolder, count); + sourceCommitCache.set(fileFolder, count); } catch (err) { recordInvalidSourceCommit(false, fileFolder, commitHash); } - return commitHashCache.get(fileFolder); + return sourceCommitCache.get(fileFolder); } function packageEdits(document, parentDocument) { From c799cec181216fda1f00f79673e1580c5daa881e Mon Sep 17 00:00:00 2001 From: hochan222 Date: Mon, 27 Mar 2023 02:23:02 +0900 Subject: [PATCH 17/20] refactor: types and naming --- server/translations.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index 829be5c4cf90..d4eb79b7ce17 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -62,12 +62,12 @@ function packageTranslationDifferences(translationDifferences) { const _foundDocumentsCache = new Map(); const sourceCommitCache = fs.existsSync("./source-commit.json") - ? new Map( + ? new Map( Object.entries( JSON.parse(fs.readFileSync("./source-commit.json", "utf8")) ) ) - : new Map(); + : new Map(); const memCommitStore = new Map(); let memCommitStoreOldest = "HEAD"; export async function findDocuments({ locale }) { @@ -78,7 +78,7 @@ export async function findDocuments({ locale }) { function getRecentRepoHash(cwd: string): string { return execSync("git rev-parse HEAD", { cwd }).toString().trimEnd(); } - function isValidSourceCommitCache(): boolean { + function isValidCache(): boolean { return ( prevCache.has(CONTENT_ROOT) && prevCache.has(CONTENT_TRANSLATED_ROOT) && @@ -87,7 +87,7 @@ export async function findDocuments({ locale }) { ); } - if (!isValidSourceCommitCache()) { + if (!isValidCache()) { prevCache.clear(); prevCache.set(CONTENT_ROOT, contentHash); prevCache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash); @@ -205,7 +205,7 @@ function getDocument(filePath) { fileFolder: string, parentFilePath: string, commitHash: string - ) { + ): number { try { let count = 0; if (!memCommitStore.has(commitHash)) { From 08701ad6230f99d2d65a1529fcccf48ea6623090 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sat, 22 Apr 2023 20:38:54 +0900 Subject: [PATCH 18/20] execSync -> spawn, various fixes for optimization Co-authored-by: LeoMcA --- server/translations.ts | 138 +++++++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 40 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index d4eb79b7ce17..e23cae47a4f1 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -3,7 +3,7 @@ import path from "node:path"; import express from "express"; import { fdir } from "fdir"; -import { execSync } from "node:child_process"; +import { execSync, spawn } from "node:child_process"; import { getPopularities, Document, Translation } from "../content/index.js"; import { @@ -68,8 +68,8 @@ const sourceCommitCache = fs.existsSync("./source-commit.json") ) ) : new Map(); -const memCommitStore = new Map(); -let memCommitStoreOldest = "HEAD"; +const commitFiles = new Map(); +let commitFilesOldest = "HEAD"; export async function findDocuments({ locale }) { function checkCacheValidation(prevCache: Map): void { const contentHash = getRecentRepoHash(CONTENT_ROOT); @@ -92,8 +92,8 @@ export async function findDocuments({ locale }) { prevCache.set(CONTENT_ROOT, contentHash); prevCache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash); sourceCommitCache.clear(); - memCommitStore.clear(); - memCommitStoreOldest = "HEAD"; + commitFiles.clear(); + commitFilesOldest = "HEAD"; } } @@ -128,7 +128,7 @@ export async function findDocuments({ locale }) { if (!cache.has(filePath) || cache.get(filePath).mtime < mtime) { counts.cacheMisses++; - const document = getDocument(filePath); + const document = await getDocument(filePath); cache.set(filePath, { document, mtime, @@ -164,7 +164,7 @@ export async function findDocuments({ locale }) { }; } -function getDocument(filePath) { +async function getDocument(filePath) { function packagePopularity(document, parentDocument) { return { value: document.metadata.popularity, @@ -185,13 +185,15 @@ function getDocument(filePath) { } function recordInvalidSourceCommit( - isValid: boolean, fileFolder: string, - commitHash: string + commitHash: string, + message: string ) { const filePath = "./source-commit-invalid-report.txt"; - const errorMessage = `- ${commitHash} commit hash is invalid in ${fileFolder}`; - if (isValid) return; + const errorMessage = `- ${commitHash} commit hash is invalid in ${fileFolder}: ${message.replace( + /\n/g, + " " + )}`; if (!fs.existsSync(filePath)) { fs.writeFileSync(filePath, ""); } @@ -201,49 +203,101 @@ function getDocument(filePath) { }); } - function getCommitBehindFromLatest( + class GitError extends Error { + constructor(stderr: string) { + super(stderr); + this.name = "GitError"; + } + } + + function fillMemStore(commitHash: string) { + return new Promise((resolve, reject) => { + commitHash = "13e249a7051d637deb3c8fd3a25c397c401edf13"; // oldest commit in mdn/content + const memBefore = process.memoryUsage.rss(); + console.log(`rss before: ${memBefore} bytes`); + + const git = spawn( + "git", + [ + "log", + "--pretty=format:%x00%x00%H", + "--name-only", + "-z", + `${commitHash}..${commitFilesOldest}`, + ], + { + cwd: CONTENT_ROOT, + } + ); + + let stdoutBuffer = ""; + + git.stdout.on("data", (data) => { + stdoutBuffer += data.toString(); + const commits = stdoutBuffer.split("\0\0"); + const partial = commits.pop(); + stdoutBuffer = partial; + commits.forEach((commit) => { + const [dirtyHash, files] = commit.split("\n"); + // necessary for commits following those with no changes: + const hash = dirtyHash.replace(/\0/g, ""); + commitFiles.set(hash, files ? files.split("\0") : []); + }); + }); + + let stderr = ""; + + git.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + git.on("close", (code) => { + const memAfter = process.memoryUsage.rss(); + console.log(`rss after: ${memAfter} bytes`); + console.log(`rss diff: ${memAfter - memBefore} bytes`); + + commitFilesOldest = commitHash; + code ? reject(new GitError(stderr)) : resolve(null); + }); + }); + } + + async function getCommitBehindFromLatest( fileFolder: string, parentFilePath: string, commitHash: string - ): number { + ): Promise { try { let count = 0; - if (!memCommitStore.has(commitHash)) { - execSync( - `git log --pretty=format:'%H' --name-only ${commitHash}..${memCommitStoreOldest}`, - { - cwd: CONTENT_ROOT, - } - ) - .toString() - .trimEnd() - .split("\n\n") - .forEach((commit) => { - const [hash, ...files] = commit.split("\n"); - memCommitStore.set(hash, files); - }); - memCommitStoreOldest = commitHash; + if (!commitFiles.has(commitHash)) { + await fillMemStore(commitHash); } - for (const [hash, files] of memCommitStore.entries()) { + for (const [hash, files] of commitFiles.entries()) { if (hash === commitHash) { - recordInvalidSourceCommit( - files.includes(parentFilePath), - fileFolder, - commitHash - ); + if (!files.includes(parentFilePath)) { + recordInvalidSourceCommit( + fileFolder, + commitHash, + "file isn't changed in this commit" + ); + } break; } if (files.includes(parentFilePath)) count++; } sourceCommitCache.set(fileFolder, count); } catch (err) { - recordInvalidSourceCommit(false, fileFolder, commitHash); + if (err instanceof GitError) { + recordInvalidSourceCommit(fileFolder, commitHash, err.message); + } else { + throw err; + } } return sourceCommitCache.get(fileFolder); } - function packageEdits(document, parentDocument) { + async function packageEdits(document, parentDocument) { const { fileInfo: { root: fileRoot, folder: fileFolder }, metadata: { hash: fileHash, modified, l10n }, @@ -260,7 +314,7 @@ function getDocument(filePath) { if (l10n?.sourceCommit) { sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit); - sourceCommitsBehindCount = getCommitBehindFromLatest( + sourceCommitsBehindCount = await getCommitBehindFromLatest( fileFolder, parentFilePath.replace(parentFileRoot, "files"), l10n.sourceCommit @@ -280,12 +334,16 @@ function getDocument(filePath) { // We can't just open the `index.json` and return it like that in the XHR // payload. It's too much stuff and some values need to be repackaged/ // serialized or some other transformation computation. - function packageDocument(document, englishDocument, translationDifferences) { + async function packageDocument( + document, + englishDocument, + translationDifferences + ) { const mdn_url = document.url; const { title } = document.metadata; const popularity = packagePopularity(document, englishDocument); const differences = packageTranslationDifferences(translationDifferences); - const edits = packageEdits(document, englishDocument); + const edits = await packageEdits(document, englishDocument); return { popularity, differences, edits, mdn_url, title }; } @@ -308,7 +366,7 @@ function getDocument(filePath) { )) { differences.push(difference); } - return packageDocument(document, englishDocument, differences); + return await packageDocument(document, englishDocument, differences); } const _defaultLocaleDocumentsCache = new Map(); From afcee51449565915cbb2faf67fe30ed1b05465ad Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sun, 23 Apr 2023 01:19:59 +0900 Subject: [PATCH 19/20] fix: make sourceCommitCache work correctly --- server/translations.ts | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index e23cae47a4f1..fe8e2919205b 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -60,9 +60,11 @@ function packageTranslationDifferences(translationDifferences) { return { total, countByType }; } +type RecentRepoHashType = string; + const _foundDocumentsCache = new Map(); const sourceCommitCache = fs.existsSync("./source-commit.json") - ? new Map( + ? new Map( Object.entries( JSON.parse(fs.readFileSync("./source-commit.json", "utf8")) ) @@ -78,22 +80,29 @@ export async function findDocuments({ locale }) { function getRecentRepoHash(cwd: string): string { return execSync("git rev-parse HEAD", { cwd }).toString().trimEnd(); } - function isValidCache(): boolean { + function updateRecentRepoHash(cache: Map): void { + cache.set(CONTENT_ROOT, contentHash); + cache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash); + } + function isValidCache(cache: Map): boolean { return ( - prevCache.has(CONTENT_ROOT) && - prevCache.has(CONTENT_TRANSLATED_ROOT) && - prevCache.get(CONTENT_ROOT) === contentHash && - prevCache.get(CONTENT_TRANSLATED_ROOT) === translatedContentHash + cache.has(CONTENT_ROOT) && + cache.has(CONTENT_TRANSLATED_ROOT) && + cache.get(CONTENT_ROOT) === contentHash && + cache.get(CONTENT_TRANSLATED_ROOT) === translatedContentHash ); } - if (!isValidCache()) { + if (isValidCache(sourceCommitCache)) { + return; + } + if (!isValidCache(prevCache)) { prevCache.clear(); - prevCache.set(CONTENT_ROOT, contentHash); - prevCache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash); sourceCommitCache.clear(); commitFiles.clear(); commitFilesOldest = "HEAD"; + updateRecentRepoHash(prevCache); + updateRecentRepoHash(sourceCommitCache); } } @@ -267,6 +276,10 @@ async function getDocument(filePath) { parentFilePath: string, commitHash: string ): Promise { + if (sourceCommitCache.has(fileFolder)) { + return sourceCommitCache.get(fileFolder) as number; + } + try { let count = 0; if (!commitFiles.has(commitHash)) { @@ -294,7 +307,7 @@ async function getDocument(filePath) { } } - return sourceCommitCache.get(fileFolder); + return sourceCommitCache.get(fileFolder) as number; } async function packageEdits(document, parentDocument) { From cf36eb5e09e6067d49b5d4e3a8e269f0bd4b9e53 Mon Sep 17 00:00:00 2001 From: hochan222 Date: Sun, 23 Apr 2023 02:16:23 +0900 Subject: [PATCH 20/20] remove log --- server/translations.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/translations.ts b/server/translations.ts index fe8e2919205b..e0206b3a53e9 100644 --- a/server/translations.ts +++ b/server/translations.ts @@ -221,10 +221,6 @@ async function getDocument(filePath) { function fillMemStore(commitHash: string) { return new Promise((resolve, reject) => { - commitHash = "13e249a7051d637deb3c8fd3a25c397c401edf13"; // oldest commit in mdn/content - const memBefore = process.memoryUsage.rss(); - console.log(`rss before: ${memBefore} bytes`); - const git = spawn( "git", [ @@ -261,10 +257,6 @@ async function getDocument(filePath) { }); git.on("close", (code) => { - const memAfter = process.memoryUsage.rss(); - console.log(`rss after: ${memAfter} bytes`); - console.log(`rss diff: ${memAfter - memBefore} bytes`); - commitFilesOldest = commitHash; code ? reject(new GitError(stderr)) : resolve(null); });