diff --git a/.github/workflows/build-crowdin.yml b/.github/workflows/build-crowdin.yml deleted file mode 100644 index 8680079f692..00000000000 --- a/.github/workflows/build-crowdin.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Build Crowdin project - -on: - schedule: - - cron: "20 04 1 * *" # Runs at 4:20 AM on the first day of every month - workflow_dispatch: - -jobs: - trigger_crowdin_project_build: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Install dependencies - run: yarn install - - - name: Install ts-node - run: yarn global add ts-node - - - name: Run script - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/triggerBuild.ts - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} diff --git a/.github/workflows/crowdin-ci.yml b/.github/workflows/crowdin-ci.yml new file mode 100644 index 00000000000..7d8a7c81570 --- /dev/null +++ b/.github/workflows/crowdin-ci.yml @@ -0,0 +1,68 @@ +name: Crowdin CI + +on: + schedule: + - cron: "20 4 1 * *" # Runs at 4:20 AM on the first day of every month + workflow_dispatch: # Can be dispatched manually + +jobs: + create_approved_language_bucket_prs: + runs-on: ubuntu-latest + env: + CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} + CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + # Set up environment + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install dependencies + run: yarn install + + - name: Install ts-node + run: yarn global add ts-node + + - name: Set up git + run: | + git config --global user.email "actions@github.com" + git config --global user.name "GitHub Action" + + - name: Fetch latest dev + run: git fetch origin dev + + # Build translations + - name: Build Crowdin project + id: build-crowdin + run: | + npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/triggerBuild.ts; + grep BUILD_ID output.env >> $GITHUB_ENV; + + - name: Await latest build to finish + run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/awaitLatestBuild.ts + + - name: Check build success + run: | + if [ $(grep BUILD_SUCCESS output.env | cut -d'=' -f2) = false ]; then + echo "Build timed out, exiting" + exit 1 + fi + shell: bash + + # Prepare bucket ids + - name: Get latest translation bucket directory ids + run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getBucketDirectoryIds.ts + + # Import approved translations + - name: Get translations + run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getTranslations.ts + + # Post updates as language-specific PRs + - name: Process commits and post PRs by language + run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/postLangPRs.ts diff --git a/.github/workflows/get-translations.yml b/.github/workflows/get-translations.yml deleted file mode 100644 index 4a3fdb29b39..00000000000 --- a/.github/workflows/get-translations.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Import Crowdin translations - -on: - schedule: - - cron: "20 16 1 * *" # Runs at 4:20 PM on the first day of every month - workflow_dispatch: - -jobs: - import_crowdin_and_create_prs: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Install dependencies - run: yarn install - - - name: Install ts-node - run: yarn global add ts-node - - - name: Set up git - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Action" - - - name: Fetch latest dev - run: git fetch origin dev - - - name: Get translations - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getTranslations.ts - env: - CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} - CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} - - - name: Authenticate GitHub CLI - run: | - echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token - - - name: Process commits and post PRs by language - run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/postLangPRs.ts - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/scripts/crowdin/import/utils.ts b/src/scripts/crowdin/import/utils.ts index 637f8d9696e..d14506cfe9b 100644 --- a/src/scripts/crowdin/import/utils.ts +++ b/src/scripts/crowdin/import/utils.ts @@ -1,4 +1,4 @@ -import { copyFileSync, existsSync, mkdirSync, readdirSync } from "fs" +import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync } from "fs" import { join } from "path" import i18Config from "../../../../i18n.config.json" @@ -45,9 +45,7 @@ export const scrapeDirectory = ( copyFileSync(source, jsonDestinationPath) // Update .json tracker trackers.langs[repoLangCode].jsonCopyCount++ - } else if ( - item.endsWith(".md") - ) { + } else if (item.endsWith(".md")) { const mdDestDirPath: string = join( TRANSLATIONS_DIR, repoLangCode, @@ -60,6 +58,7 @@ export const scrapeDirectory = ( // Update .md tracker trackers.langs[repoLangCode].mdCopyCount++ } else { + if (!statSync(source).isDirectory()) return // If another directory, recursively call `scrapeDirectory` scrapeDirectory( `${path}/${item}`, diff --git a/src/scripts/crowdin/translations/awaitLatestBuild.ts b/src/scripts/crowdin/translations/awaitLatestBuild.ts new file mode 100644 index 00000000000..8ecdc8144be --- /dev/null +++ b/src/scripts/crowdin/translations/awaitLatestBuild.ts @@ -0,0 +1,57 @@ +import { writeFileSync } from "fs" +import { join } from "path" + +import crowdin from "../api-client/crowdinClient" + +const FINISHED = "finished" +const TIMEOUT = 2 * 60 * 60 * 1000 // Timeout after 2 hours +const INTERVAL = 10 * 1000 // 10 seconds between checks + +const OUTPUT_PATH = join(process.env["GITHUB_WORKSPACE"] || "", "output.env") + +async function awaitLatestBuild() { + const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 + + // BUILD_ID is provided by the triggerBuild script run in the same workflow prior to this script + const buildId = process.env.BUILD_ID + + console.log("Build ID provided:", buildId) + const initialResponse = await crowdin.translationsApi.checkBuildStatus( + projectId, + Number(buildId) + ) + let data = initialResponse.data + + let isFinished = data.status === FINISHED + + const timeoutTime = Date.now() + TIMEOUT + let tryAgainTime = Date.now() - 1 + while (!isFinished && Date.now() < timeoutTime) { + if (Date.now() < tryAgainTime) continue + tryAgainTime = Date.now() + INTERVAL + + const repeatCheck = await crowdin.translationsApi.checkBuildStatus( + projectId, + Number(buildId) + ) + data = repeatCheck.data + isFinished = data.status === FINISHED + console.log( + `id: ${buildId}, status: ${data.status}, progress ${data.progress}` + ) + } + + if (data.status !== FINISHED) { + writeFileSync(OUTPUT_PATH, `BUILD_SUCCESS=false\n`, { flag: "a" }) + throw new Error( + `Timeout: Build did not finish in ${TIMEOUT / 1000 / 60} minutes` + ) + } + + console.log("Latest build data:", data) + writeFileSync(OUTPUT_PATH, `BUILD_SUCCESS=true\n`, { flag: "a" }) +} + +awaitLatestBuild() + +export default awaitLatestBuild diff --git a/src/scripts/crowdin/translations/constants.ts b/src/scripts/crowdin/translations/constants.ts index 8f224528553..6f0756df4fc 100644 --- a/src/scripts/crowdin/translations/constants.ts +++ b/src/scripts/crowdin/translations/constants.ts @@ -1,15 +1,15 @@ -import { resolve } from "path" +import { join } from "path" export const DOT_CROWDIN = ".crowdin" export const CROWDIN_DATA_DIR = "src/data/crowdin" export const SAVE_FILE = "download.zip" -export const FILE_PATH = resolve(CROWDIN_DATA_DIR, SAVE_FILE) +export const FILE_PATH = join(CROWDIN_DATA_DIR, SAVE_FILE) export const SUMMARY_SAVE_FILE = "import-summary.json" -export const SUMMARY_PATH = resolve(CROWDIN_DATA_DIR, SUMMARY_SAVE_FILE) +export const SUMMARY_PATH = join(CROWDIN_DATA_DIR, SUMMARY_SAVE_FILE) export const BUCKETS_IMPORTED_FILE = "buckets-imported.json" -export const BUCKETS_PATH = resolve(CROWDIN_DATA_DIR, BUCKETS_IMPORTED_FILE) +export const BUCKETS_PATH = join(CROWDIN_DATA_DIR, BUCKETS_IMPORTED_FILE) export const APPROVAL_THRESHOLD = 100 diff --git a/src/scripts/crowdin/translations/getApprovedBuckets.ts b/src/scripts/crowdin/translations/getApprovedBuckets.ts index c282d64b4c5..bcc12ad29ba 100644 --- a/src/scripts/crowdin/translations/getApprovedBuckets.ts +++ b/src/scripts/crowdin/translations/getApprovedBuckets.ts @@ -7,11 +7,11 @@ import type { BucketsList } from "../import/types" import { APPROVAL_THRESHOLD } from "./constants" async function getApprovedBuckets(): Promise { + console.log("⏳ Getting approved buckets...") const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 const bucketsList: BucketsList = {} - // TODO: Consider regenerating bucketDirs list on each run for fidelity for (const bucketDir of bucketDirs) { const directoryProgress = await crowdin.translationStatusApi.getDirectoryProgress( diff --git a/src/scripts/crowdin/translations/getBucketDirectoryIds.ts b/src/scripts/crowdin/translations/getBucketDirectoryIds.ts new file mode 100644 index 00000000000..43846cdccaa --- /dev/null +++ b/src/scripts/crowdin/translations/getBucketDirectoryIds.ts @@ -0,0 +1,9 @@ +import getAndSaveDirectories from "../source-files/fetchAndSaveDirectories" + +async function main() { + await getAndSaveDirectories() +} + +main() + +export default main diff --git a/src/scripts/crowdin/translations/postLangPRs.ts b/src/scripts/crowdin/translations/postLangPRs.ts index 143126ce0e7..abd1e6eda44 100644 --- a/src/scripts/crowdin/translations/postLangPRs.ts +++ b/src/scripts/crowdin/translations/postLangPRs.ts @@ -1,7 +1,7 @@ import fs from "fs" import { LOCALES_CODES } from "../../../lib/constants" -import { BucketsList } from "../import/types" +import type { BucketsList } from "../import/types" import { BUCKETS_PATH } from "./constants" import { createLocaleTranslationPR } from "./utils" diff --git a/src/scripts/crowdin/translations/triggerBuild.ts b/src/scripts/crowdin/translations/triggerBuild.ts index 866a40b6c91..f3e54dfd403 100644 --- a/src/scripts/crowdin/translations/triggerBuild.ts +++ b/src/scripts/crowdin/translations/triggerBuild.ts @@ -1,14 +1,22 @@ +import { writeFileSync } from "fs" +import { join } from "path" + import crowdin from "../api-client/crowdinClient" -import "dotenv/config" +const OUTPUT_PATH = join(process.env["GITHUB_WORKSPACE"] || "", "output.env") async function triggerBuild() { const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359 try { - await crowdin.translationsApi.buildProject(projectId, { - exportApprovedOnly: true, - }) + const response = await crowdin.translationsApi.buildProject(projectId) + const { id, status } = response.data + const isAlreadyFinished = status === "finished" + console.log( + `Build ${isAlreadyFinished ? "already finished" : "triggered"} id:`, + id + ) + writeFileSync(OUTPUT_PATH, `BUILD_ID=${id}\n`, { flag: "a" }) } catch (error: unknown) { console.error((error as Error).message) } diff --git a/src/scripts/crowdin/translations/utils.ts b/src/scripts/crowdin/translations/utils.ts index e25492ffdb8..a8bc976aa09 100644 --- a/src/scripts/crowdin/translations/utils.ts +++ b/src/scripts/crowdin/translations/utils.ts @@ -41,6 +41,12 @@ export const decompressFile = async (filePath: string, targetDir: string) => { } const getQAMessage = (locale: string) => { + console.log("Checking summary path:", SUMMARY_PATH) + if (!fs.existsSync(SUMMARY_PATH)) { + console.error("Could not find summary path:", SUMMARY_PATH) + throw new Error("No summary file found.") + } + const summaryJson: QASummary = JSON.parse(readFileSync(SUMMARY_PATH, "utf-8")) const qaResults = summaryJson[locale] ? summaryJson[locale].map((s) => "- " + s).join("\n") @@ -56,6 +62,8 @@ yarn markdown-checker ${qaResults} + +@coderabbitai review ` } diff --git a/src/scripts/markdownChecker.ts b/src/scripts/markdownChecker.ts index 3654e2d1b89..9918261473e 100644 --- a/src/scripts/markdownChecker.ts +++ b/src/scripts/markdownChecker.ts @@ -368,6 +368,7 @@ const writeSummary = (summary: Summary, summaryWritePath: string) => { } export function checkMarkdown(summaryWritePath?: string) { + console.log("Checking markdown for common issues...") const summary = {} as Summary const markdownPaths: Array = getAllMarkdownPaths(PATH_TO_ALL_CONTENT) const markdownPathsByLang: Languages = @@ -383,8 +384,10 @@ export function checkMarkdown(summaryWritePath?: string) { if (!summary[lang].length) delete summary[lang] } + if (!summaryWritePath) return - summaryWritePath && writeSummary(summary, summaryWritePath) + writeSummary(summary, summaryWritePath) + console.log("Writing markdown checker summary to:", summaryWritePath) } checkMarkdown()