Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/feat/rf-1267-update-crowdin-wo…
Browse files Browse the repository at this point in the history 
…rkflow-with-pre-translate' into next
  • Loading branch information
@yeager-eren
yeager-eren committed Feb 18, 2024
2 parents 508bde6 + f71a99b commit 78ff05d
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 153 deletions.
185 changes: 32 additions & 153 deletions scripts/crowdin/command.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import { CrowdinError } from "../common/errors.mjs";

const PROJECT_ID = process.env.CROWDIN_PROJECT_ID;
const TOKEN = process.env.CROWDIN_PERSONAL_TOKEN;

const BASE_URL = `https://api.crowdin.com/api/v2`;
const BASE_PROJECT_URL = `${BASE_URL}/projects/${PROJECT_ID}`;
const REQUEST_INTERVAL_TIMEOUT = 10_000;
const MAXIMUM_PRETRANSLATION_STATUS_CHECK = 20;

const requestData = {
import { PROJECT_ID, TOKEN } from "./constants.mjs";
import {
checkPreTranslateStatus,
getLanguageIds,
getMachineTranslationEngineID,
getSourceFileId,
sendPreTranslateRequest
} from "./pretranslate.mjs";

const preTranslationOption = {
method: 'mt',
autoApproveOption: 'all',
duplicateTranslations: false,
Expand All @@ -17,156 +16,36 @@ const requestData = {
translateWithPerfectMatchOnly: false,
};

const getMachineTranslationEngineID = async () => {
try {
const response = await fetch(`${BASE_URL}/mts`, {
headers: {
Authorization: `Bearer ${TOKEN}`,
},
});

if (!response.ok) {
throw 'Failed to get machine translation data';
}

const responseData = await response.json();
const engineId = responseData.data[0].data.id;

return engineId;
} catch (error) {
console.error('Error:', error);
throw new CrowdinError(error);
}
};

const getLanguageIds = async () => {
try {
const response = await fetch(BASE_PROJECT_URL, {
headers: {
Authorization: `Bearer ${TOKEN}`,
},
});

if (!response.ok) {
throw 'Failed to get target language ids';
}

const responseData = await response.json();
const languageIds = responseData.data.targetLanguageIds;

return languageIds;
} catch (error) {
console.error('Error:', error);
throw new CrowdinError(error);
}
};

const getSourceFileId = async () => {
try {
const response = await fetch(`${BASE_PROJECT_URL}/files`, {
headers: {
Authorization: `Bearer ${TOKEN}`,
},
});

if (!response.ok) {
throw 'Failed to get source file id';
}
async function run() {
console.log('🔨 Start pre-translation...');

const responseData = await response.json();
const sourceFileId = responseData.data[0].data.id;

return sourceFileId;
} catch (error) {
console.error('Error:', error);
throw new CrowdinError(error);
if(!PROJECT_ID || !TOKEN){
throw new CrowdinError('environments are not set correctly');
}
};

const sendPreTranslateRequest = async ({ sourceFileId, languageIds }) => {
try {
const engineId = await getMachineTranslationEngineID();
console.log('engineId', engineId);
requestData.engineId = engineId ;
const response = await fetch(`${BASE_PROJECT_URL}/pre-translations`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${TOKEN}`,
},
body: JSON.stringify({
...requestData,
fileIds: [sourceFileId],
languageIds,
}),
});
console.log('[1/5]', 'Get source file id');
const sourceFileId = await getSourceFileId();

if (!response.ok) {
throw `Failed to initiate pre-translation. Status: ${response.status}`;
}
console.log('[2/5]', 'Get target languages ids');
const languageIds = await getLanguageIds();

console.log('[3/5]', 'Get machine translation engine id');
const engineId = await getMachineTranslationEngineID();
preTranslationOption.engineId = engineId ;

const responseData = await response.json();
const preTranslationId = responseData.data.identifier;

return preTranslationId;
} catch (error) {
console.error('Error:', error);
throw new CrowdinError(error);
}
};
console.log('[4/5]', 'Send pre-translate to crowdin');
const preTranslationId = await sendPreTranslateRequest({ sourceFileId, languageIds, preTranslationOption });

const checkPreTranslateStatus = async (preTranslationId) => {
const maxAttempts = MAXIMUM_PRETRANSLATION_STATUS_CHECK;
let attempt = 0;
console.log('[5/5]', 'Check pre-translate status:',preTranslationId);
const status = await checkPreTranslateStatus(preTranslationId);

const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

while (attempt < maxAttempts) {
try {
const response = await fetch(`${BASE_PROJECT_URL}/pre-translations/${preTranslationId}`, {
headers: {
Authorization: `Bearer ${TOKEN}`,
},
});

if (!response.ok) {
throw `Failed to get pre-translation status. Status: ${response.status}`;
}

const responseData = await response.json();

const status = responseData.data.status;

if (status === 'finished') {
console.log('Pre-translation finished!');
process.exit(0);
} else {
console.log(
`Pre-translation status: ${status}. Retrying in 10 seconds...`
);
await delay(REQUEST_INTERVAL_TIMEOUT);
attempt++;
}
} catch (error) {
console.error('Error:', error);
throw new CrowdinError(error);
}
if (status === 'finished'){
console.log('✅ Pre-translation finished!');
}
throw new CrowdinError('Timeout: Pre-translation did not succeed within the specified time.');
};

(async () => {
try {
if(!PROJECT_ID || !PROJECT_ID){
throw new CrowdinError('environments are not set correctly');
}
const [sourceFileId, languageIds] = await Promise.all([getSourceFileId(), getLanguageIds()]);
console.log('sourceFileId', sourceFileId);
console.log('languageIds', languageIds);
const preTranslationId = await sendPreTranslateRequest({ sourceFileId, languageIds });
await checkPreTranslateStatus(preTranslationId);
} catch (error) {
console.error('Error:', error);
process.exit(1);
}
})();
run().catch((e) => {
console.error(e);
process.exit(1);
});
11 changes: 11 additions & 0 deletions scripts/crowdin/constants.mjs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
export const PROJECT_ID = process.env.CROWDIN_PROJECT_ID;
export const TOKEN = process.env.CROWDIN_PERSONAL_TOKEN;

export const BASE_URL = `https://api.crowdin.com/api/v2`;
export const PROJECT_API = `${BASE_URL}/projects/${PROJECT_ID}`;
export const PRETRANSLATE_API = `${PROJECT_API}/pre-translations`
export const FILE_API = `${PROJECT_API}/files`
export const MACHINE_TRANSLATE_API = `${BASE_URL}/mts`;

export const REQUEST_INTERVAL_TIMEOUT = 10_000;
export const MAXIMUM_PRETRANSLATION_STATUS_CHECK = 20;
60 changes: 60 additions & 0 deletions scripts/crowdin/pretranslate.mjs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import { CrowdinError } from "../common/errors.mjs";
import { fetchDataWithAuthorization } from "./utils.mjs";

import {
MAXIMUM_PRETRANSLATION_STATUS_CHECK,
PRETRANSLATE_API,
REQUEST_INTERVAL_TIMEOUT,
FILE_API,
MACHINE_TRANSLATE_API,
PROJECT_API,
} from "./constants.mjs";


export const getMachineTranslationEngineID = async () =>{
const responseData = await fetchDataWithAuthorization(MACHINE_TRANSLATE_API);
if(!responseData.data || !responseData.data.length){
throw new CrowdinError('No data received for machine translation');
}
return responseData.data[0].data.id;
}

export const getLanguageIds = async () => {
const responseData = await fetchDataWithAuthorization(PROJECT_API);
return responseData.data.targetLanguageIds;
};

export const getSourceFileId = async () => {
const responseData = await fetchDataWithAuthorization(FILE_API);
if (!responseData.data || !responseData.data.length) {
throw new CrowdinError('No data received for source file id');
}
return responseData.data[0].data.id;
};

export const sendPreTranslateRequest = async ({ sourceFileId, languageIds, preTranslationOption }) => {
const responseData = await fetchDataWithAuthorization(PRETRANSLATE_API, 'POST', {
...preTranslationOption,
fileIds: [sourceFileId],
languageIds,
});
return responseData.data.identifier;
};

export const checkPreTranslateStatus = async (preTranslationId) => {
const maxAttempts = MAXIMUM_PRETRANSLATION_STATUS_CHECK;
let attempt = 0;
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
while (attempt < maxAttempts) {
const responseData = await fetchDataWithAuthorization(`${PRETRANSLATE_API}/${preTranslationId}`);
const status = responseData.data.status;
if (status === 'finished') {
return status;
} else {
console.log(`Pre-translation status: ${status}. Retrying in 10 seconds...`);
await delay(REQUEST_INTERVAL_TIMEOUT);
attempt++;
}
}
throw new CrowdinError('Timeout: Pre-translation did not succeed within the specified time.');
};
24 changes: 24 additions & 0 deletions scripts/crowdin/utils.mjs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import {TOKEN } from "./constants.mjs";

// Reusable function to handle fetch requests with authorization headers
export const fetchDataWithAuthorization = async (url, method = 'GET', body = null) => {
const options = {
method,
headers: {
Authorization: `Bearer ${TOKEN}`,
'Content-Type': 'application/json',
},
};

if (body) {
options.body = JSON.stringify(body);
}

const response = await fetch(url, options);

if (!response.ok) {
throw new CrowdinError(`Failed to fetch data from ${url}. Status: ${response.status}`);
}

return await response.json();
};

0 comments on commit 78ff05d

Please sign in to comment.