From e8bc7d717913c24ed449160122b39e3ecbcf2c39 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Thu, 7 Oct 2021 11:13:05 +0200 Subject: [PATCH] Add script to download translations from transifex --- package.json | 5 +- scripts/i18n/transifex-pull.js | 149 +++++++++++++++++++++++++++++++++ scripts/i18n/transifex.js | 52 ++++++++++++ 3 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 scripts/i18n/transifex-pull.js create mode 100644 scripts/i18n/transifex.js diff --git a/package.json b/package.json index 278cee1e5..9ae5c687b 100644 --- a/package.json +++ b/package.json @@ -35,14 +35,15 @@ }, "scripts": { "prepare": "cross-env THEIA_ELECTRON_SKIP_REPLACE_FFMPEG=1 lerna run prepare && yarn download:plugins", - "cleanup": "npx rimraf ./**/node_modules && rm -rf ./node_modules ./.browser_modules ./arduino-ide-extension/build ./arduino-ide-extension/downloads ./arduino-ide-extension/Examples ./arduino-ide-extension/lib ./browser-app/lib ./browser-app/src-gen ./browser-app/gen-webpack.config.js ./electron-app/lib ./electron-app/src-gen ./electron-app/gen-webpack.config.js", + "cleanup": "npx rimraf ./**/node_modules && rm -rf ./node_modules ./.browser_modules ./arduino-ide-extension/build ./arduino-ide-extension/downloads ./arduino-ide-extension/Examples ./arduino-ide-extension/lib ./browser-app/lib ./browser-app/src-gen ./browser-app/gen-webpack.config.js ./electron-app/lib ./electron-app/src-gen ./electron-app/gen-webpack.config.js", "rebuild:browser": "theia rebuild:browser", "rebuild:electron": "theia rebuild:electron", "start": "yarn --cwd ./electron-app start", "watch": "lerna run watch --parallel", "test": "lerna run test", "download:plugins": "theia download:plugins", - "update:version": "node ./scripts/update-version.js" + "update:version": "node ./scripts/update-version.js", + "i18n:pull": "node ./scripts/i18n/transifex-pull.js ./i18n/" }, "lint-staged": { "./arduino-ide-extension/**/*.{ts,tsx}": [ diff --git a/scripts/i18n/transifex-pull.js b/scripts/i18n/transifex-pull.js new file mode 100644 index 000000000..986df91ea --- /dev/null +++ b/scripts/i18n/transifex-pull.js @@ -0,0 +1,149 @@ +// @ts-check + +const transifex = require('./transifex'); +const util = require('util'); +const shell = require('shelljs'); +const fetch = require('node-fetch'); +const download = require('download'); + +const getLanguages = async (organization, project) => { + const url = transifex.url( + util.format('projects/o:%s:p:%s/languages', organization, project) + ); + const json = await fetch(url, { headers: transifex.authHeader() }) + .catch(err => { + shell.echo(err); + shell.exit(1); + }) + .then(res => res.json()); + let languages = []; + json['data'].forEach(e => { + const languageCode = e['attributes']['code']; + // Skip english since it's the one we generate + if (languageCode === 'en') { + return; + } + languages.push(languageCode); + }); + return languages; +}; + +const requestTranslationDownload = async (relationships) => { + let url = transifex.url('resource_translations_async_downloads'); + const data = { + data: { + relationships, + type: 'resource_translations_async_downloads' + } + }; + const headers = transifex.authHeader(); + headers['Content-Type'] = 'application/vnd.api+json'; + const json = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(data) + }) + .catch(err => { + shell.echo(err); + shell.exit(1); + }) + .then(res => res.json()); + + return json['data']['id']; +}; + +const getTranslationDownloadStatus = async (language, downloadRequestId) => { + // The download request status must be asked from time to time, if it's + // still pending we try again using exponentional backoff starting from 2.5 seconds. + let backoffMs = 2500; + while (true) { + const url = transifex.url( + util.format('resource_translations_async_downloads/%s', downloadRequestId) + ); + const options = { + headers: transifex.authHeader(), + redirect: 'manual' + }; + const res = await fetch(url, options).catch(err => { + shell.echo(err); + shell.exit(1); + }); + + if (res.status === 303) { + // When the file to download is ready we get redirected + return { + language, + downloadUrl: res.headers.get('location') + }; + } + + const json = await res.json(); + const downloadStatus = json['data']['attributes']['status']; + if (downloadStatus == 'pending' || downloadStatus == 'processing') { + await new Promise(r => setTimeout(r, backoffMs)); + backoffMs = backoffMs * 2; + // Retry the download request status again + continue; + } else if (downloadStatus == 'failed') { + const errors = []; + json['data']['attributes']['errors'].forEach(err => { + errors.push(util.format('%s: %s', err.code, err.details)); + }); + throw util.format('Download request failed: %s', errors.join(', ')); + } + throw 'Download request failed in an unforeseen way'; + } +}; + +(async () => { + const { organization, project, resource } = await transifex.credentials(); + const translationsDirectory = process.argv[2]; + if (!translationsDirectory) { + shell.echo('Traslations directory not specified'); + shell.exit(1); + } + + const languages = await getLanguages(organization, project); + shell.echo('translations found:', languages.join(', ')); + + let downloadIds = []; + for (const language of languages) { + downloadIds.push({ + language, + id: await requestTranslationDownload({ + language: { + data: { + id: util.format('l:%s', language), + type: 'languages' + } + }, + resource: { + data: { + id: util.format('o:%s:p:%s:r:%s', organization, project, resource), + type: 'resources' + } + } + }) + }); + } + + const res = await Promise.all( + downloadIds.map(d => getTranslationDownloadStatus(d['language'], d['id'])) + ).catch(err => { + shell.echo(err); + shell.exit(1); + }); + + await Promise.all( + res.map(r => { + return download(r['downloadUrl'], translationsDirectory, { + filename: r['language'] + '.json' + }); + }) + ).catch(err => { + shell.echo(err); + shell.exit(1); + }); + + shell.echo('Translation files downloaded.'); +})(); diff --git a/scripts/i18n/transifex.js b/scripts/i18n/transifex.js new file mode 100644 index 000000000..b174feb26 --- /dev/null +++ b/scripts/i18n/transifex.js @@ -0,0 +1,52 @@ +// @ts-check + +const shell = require('shelljs'); +const util = require('util'); + +const TRANSIFEX_ENDPOINT = 'https://rest.api.transifex.com/'; + +const apiKey = () => { + const apiKey = process.env.TRANSIFEX_API_KEY; + if (apiKey === '') { + shell.echo('missing TRANSIFEX_API_KEY environment variable'); + shell.exit(1) + } + return apiKey +} + +exports.credentials = async () => { + const organization = process.env.TRANSIFEX_ORGANIZATION; + const project = process.env.TRANSIFEX_PROJECT; + const resource = process.env.TRANSIFEX_RESOURCE; + + if (organization === '') { + shell.echo('missing TRANSIFEX_ORGANIZATION environment variable'); + shell.exit(1) + } + + if (project === '') { + shell.echo('missing TRANSIFEX_PROJECT environment variable'); + shell.exit(1) + } + + if (resource === '') { + shell.echo('missing TRANSIFEX_RESOURCE environment variable'); + shell.exit(1) + } + + return { organization, project, resource } +} + +exports.url = (path, queryParameters) => { + let url = util.format('%s%s', TRANSIFEX_ENDPOINT, path); + if (queryParameters) { + url = util.format('%s?%s', url, queryParameters); + } + return url +} + +exports.authHeader = () => { + return { + 'Authorization': util.format("Bearer %s", apiKey()), + } +}