Skip to content

Commit

Permalink
Merge pull request #14 from yuriykuzin/migrate-to-api-v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@yuriykuzin
yuriykuzin authored Dec 23, 2021
2 parents 7983d5d + 9583fa7 commit 9bcbe30
Show file tree
Hide file tree
Showing 19 changed files with 2,432 additions and 4,981 deletions.
32 changes: 22 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,24 @@

Unofficial Crowdin client to automate continuous integration workflow.

(Since version 1.0.0-rc.1 is not dependent on the official Crowdin CLI)

This is a current version, which we experiment with to use in our team's workflow. For now some things are harcoded inside, accordingly to our own needs. You're welcome to use it as a source of inspiration, fork it and modify code to make it usable in your projects. We would appreciate any kind of feedback, suggestions, pull requests.

#### Disclaimer:
Crowdin and the Crowdin API are the property of Crowdin, LLC.

## To do:

- [ ] Restore/fix temporary removed specs

## Changes in 2.0.0

- Migrated to Crowdin API v2
- Removed `daysSinceLastUpdatedToDeleteBranchSafely` config variable

## Changes in 1.0.0-rc.1

- It is not dependent on Crowdin CLI anymore
- `projectKey` and `projectIdentifier` have to be configured in `crowdin-helper.json` in the root of your project
- `projectId` and `token` have to be configured in `crowdin-helper.json` in the root of your project
- Automatic translation (in case of "perfect match" with Translation Memory) is triggered after each file upload and progress checking on master branch (you can also disable this if you want)

## Installation from npm
Expand All @@ -30,19 +37,26 @@ Add crowdin-helper.json to the root of your project. Here is an example:

```
{
"projectIdentifier": "my-project-name",
"projectKey": "my-project-api-key",
"projectId": "my-project-id",
"token": "my-personal-access-token",
"source": "/src/i18n/en.json",
"translation": "/src/i18n/%two_letters_code%.json",
"languageToCheck": "nl",
"languagesToAutoTranslate": ["nl", "fi"],
"daysSinceLastUpdatedToDeleteBranchSafely": 3,
"minutesSinceLastMasterMergeToPurgeSafely": 20,
"disableAutoTranslation": false
"disableAutoTranslation": false,
"masterBranchName": "main"
}
```

Also, you can use patterns in "source" property:
Some comments on these properties:

- `projectId` - can be seen on a project's home page,
- `token` - from https://crowdin.com/settings#api-key
- `minutesSinceLastMasterMergeToPurgeSafely` - default is 20
- `disableAutoTranslation` - default is false
- `masterBranchName` - default is "master"
- `source` - you can use patterns:

```
"source": "/**/en.json"
Expand Down Expand Up @@ -134,8 +148,6 @@ From time to time one of team leads calls `./node_modules/.bin/crowdin-helper pu

- crowdin branch do not have relevant branch on github (in our process we delete a branch on github after merging PR into master),

- at least 3 days passed after last updating branch(is configured in "daysSinceLastUpdatedToDeleteBranchSafely" of crowdin-helper.json),

- at least 20 minutes passed since last merge to github master (is configured in "minutesSinceLastMasterMergeToPurgeSafely" of crowdin-helper.json) - so we can be sure that crowdin performed all syncronization with master branch and took necessary translations from feature crowdin-branch.

## Contributing
Expand Down
52 changes: 30 additions & 22 deletions lib/commands/check-progress-on-branch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ const COLORS = require('../utilities/colors');
const configManager = require('../utilities/config-manager');
const triggerAutoTranslation = require('./trigger-auto-translation');

const MASTER_BRANCH_NAME = configManager.get('masterBranchName');

async function checkProgressOnBranch() {
if (crowdinBranchName === 'master') {
if (crowdinBranchName === MASTER_BRANCH_NAME) {
// Let's trigger auto-translate since it is probably a run on Semaphore or other CI tool
await triggerAutoTranslation('master');
await triggerAutoTranslation(MASTER_BRANCH_NAME);

console.log(
`Crowdin: ${COLORS.GREEN}No validation performed since it is a master branch${COLORS.WHITE}`,
Expand All @@ -18,34 +20,40 @@ async function checkProgressOnBranch() {

console.log(`Crowdin: Checking language: ${configManager.get('languageToCheck')}`);

await CrowdinApi.getLanguageStatus().then((json) => {
const currentBranch = json.files.filter((file) => {
return file.node_type === 'branch' && file.name === crowdinBranchName;
})[0];
const branchData = await CrowdinApi.getBranchData(crowdinBranchName);

if (!currentBranch) {
console.log(`Crowdin: ${COLORS.GREEN}Okay, no such branch on crowdin${COLORS.WHITE}`);
if (!branchData) {
console.log(`Crowdin: ${COLORS.GREEN}Okay, no such branch on crowdin${COLORS.WHITE}`);

return;
}
return;
}

if (currentBranch.phrases === 0) {
console.log(`Crowdin: ${COLORS.GREEN}Okay, no new phrases in this branch${COLORS.WHITE}`);
const branchProgress = await CrowdinApi.getBranchProgress(branchData.data.id);

return;
}
const currentLanguageProgress = branchProgress.data.filter(
(dataLanguage) => dataLanguage.data.languageId === configManager.get('languageToCheck'),
)[0];

if (currentBranch.translated === currentBranch.phrases) {
console.log(`Crowdin: ${COLORS.GREEN}Okay, translations are ready${COLORS.WHITE}`);
if (currentLanguageProgress.data.phrases.total === 0) {
console.log(`Crowdin: ${COLORS.GREEN}Okay, no new phrases in this branch${COLORS.WHITE}`);

return;
}
return;
}

if (
currentLanguageProgress.data.phrases.translated === currentLanguageProgress.data.phrases.total
) {
console.log(`Crowdin: ${COLORS.GREEN}Okay, translations are ready${COLORS.WHITE}`);

return;
}

console.log(`Crowdin: translated ${currentBranch.translated} / ${currentBranch.phrases}`);
console.log(`Crowdin: ${COLORS.RED}Error: There are some missing translations${COLORS.WHITE}`);
console.log(
`Crowdin: translated ${currentLanguageProgress.data.phrases.translated} / ${currentLanguageProgress.data.phrases.total}`,
);
console.log(`Crowdin: ${COLORS.RED}Error: There are some missing translations${COLORS.WHITE}`);

process.exit(1);
});
process.exit(1);
}

module.exports = checkProgressOnBranch;
33 changes: 9 additions & 24 deletions lib/commands/delete-old-branches.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,30 +25,27 @@ async function deleteOldBranches() {
process.exit(1);
}

const projectInfoJson = await CrowdinApi.getInfo();
const crowdinBranches = await CrowdinApi.getBranches().then((branchesData) =>
branchesData.data.map((branchData) => ({
name: branchData.data.name,
id: branchData.data.id,
})),
);

const crowdinBranches = projectInfoJson.files.filter((file) => file.node_type === 'branch');
const gitBranchesConverted = _getGitRemoteBranches().map((gitBranch) =>
gitBranch.replace(/\//g, '--'),
);

let isSomeBranchesDeleted = false;

// we have to process requests to crowdin sequentially
await crowdinBranches.reduce(async (sequentialPromise, branch) => {
await sequentialPromise;

const daysSinceLastBranchUpdate = _getDateDiffInDays(
new Date(_getFileLastUpdated(branch)),
new Date(),
);

if (
gitBranchesConverted.indexOf(branch.name) === -1 &&
daysSinceLastBranchUpdate >= configManager.get('daysSinceLastUpdatedToDeleteBranchSafely')
) {
if (!gitBranchesConverted.includes(branch.name)) {
isSomeBranchesDeleted = true;

return CrowdinApi.deleteBranch(branch.name)
return CrowdinApi.deleteBranch(branch.id)
.then(() => {
console.log(`Branch "${branch.name}" is removed from crowdin`);
})
Expand All @@ -72,18 +69,6 @@ function _getGitRemoteBranches() {
.map((s) => s.replace('refs/heads/', ''));
}

function _getFileLastUpdated(crowdinBranchObj) {
if (crowdinBranchObj.node_type === 'file') {
return crowdinBranchObj.last_updated;
}

const OLDEST_POSSIBLE_DATE = new Date(-8640000000000000);

return crowdinBranchObj.files[0]
? _getFileLastUpdated(crowdinBranchObj.files[0])
: OLDEST_POSSIBLE_DATE; // which means never
}

function _getDateDiff(date1, date2, ms) {
const utc1 = Date.UTC(
date1.getFullYear(),
Expand Down
77 changes: 43 additions & 34 deletions lib/commands/download-translations.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
const spawn = require('child_process').spawnSync;
const unzipper = require('unzipper');
const fs = require('fs');
const fetch = require('node-fetch');

const COLORS = require('../utilities/colors');
const CrowdinApi = require('../utilities/crowdin-api');
const { crowdinBranchName, gitBranchName } = require('../utilities/branch-name');
const sourceFilesPromise = require('../utilities/source-files-promise');
const uploadSources = require('./upload-sources');

const CROWDIN_API__BUILD_STATUS__FINISHED = 'finished';

async function downloadTranslations(shouldIgnoreUnmergedMaster = false) {
if (!shouldIgnoreUnmergedMaster && !(await _isLastSourceFilesFromMasterMergedIntoCurrent())) {
console.log(
Expand All @@ -22,36 +25,47 @@ async function downloadTranslations(shouldIgnoreUnmergedMaster = false) {
await uploadSources();

console.log('Crowdin: Triggering branch build before downloading');
await CrowdinApi.buildBranch(crowdinBranchName);
const branchData = await CrowdinApi.getBranchData(crowdinBranchName);
const buildData = await CrowdinApi.buildBranch(branchData.data.id);

let buildStatusData;

do {
buildStatusData = await CrowdinApi.checkBuildStatus(buildData.data.id);

console.log('Crowdin: Build progress:', buildStatusData.data.progress);
} while (buildStatusData && buildStatusData.data.status !== CROWDIN_API__BUILD_STATUS__FINISHED);

console.log(`Crowdin: ${COLORS.GREEN}Build is done!${COLORS.WHITE}`);
console.log('Crowdin: Downloading branch', crowdinBranchName);

return await CrowdinApi.getAllTranslations(crowdinBranchName).then((res) => {
res.body
.pipe(unzipper.Parse())
.on('entry', (entry) => {
if (entry.type === 'File') {
const fileName = entry.path.replace(crowdinBranchName, '').replace(/^\//, '');

entry.pipe(
fs.createWriteStream(fileName).on('finish', () => {
console.log(`Crowdin: ${COLORS.GREEN}Unzipped ${fileName} ${COLORS.WHITE}`);
}),
);

return;
}

entry.autodrain();
})
.promise()
.catch((e) => {
console.log(
`Crowdin: ${COLORS.RED}Unzipping failed. Probably broken ZIP file${COLORS.WHITE}`,
const downloadUrlData = await CrowdinApi.getAllTranslations(buildData.data.id);
const zip = await fetch(downloadUrlData.data.url);

zip.body
.pipe(unzipper.Parse())
.on('entry', (entry) => {
if (entry.type === 'File') {
const fileName = entry.path.replace(crowdinBranchName, '').replace(/^\//, '');

entry.pipe(
fs.createWriteStream(fileName).on('finish', () => {
console.log(`Crowdin: ${COLORS.GREEN}Unzipped ${fileName} ${COLORS.WHITE}`);
}),
);
console.log(e);
});
});

return;
}

entry.autodrain();
})
.promise()
.catch((e) => {
console.log(
`Crowdin: ${COLORS.RED}Unzipping failed. Probably broken ZIP file${COLORS.WHITE}`,
);
console.log(e);
});
}

async function _isLastSourceFilesFromMasterMergedIntoCurrent() {
Expand All @@ -68,14 +82,9 @@ async function _isLastSourceFilesFromMasterMergedIntoCurrent() {
.map((lastMasterCommitId) => {
return spawn('git', ['branch', '--contains', lastMasterCommitId]).stdout.toString();
})
.every((branchesWithLatestCommit) => {
return (
branchesWithLatestCommit
.replace(/[\* ]/g, '')
.split('\n')
.indexOf(gitBranchName) !== -1
);
});
.every((branchesWithLatestCommit) =>
branchesWithLatestCommit.replace(/[\* ]/g, '').split('\n').includes(gitBranchName),
);
}

module.exports = downloadTranslations;
34 changes: 20 additions & 14 deletions lib/commands/trigger-auto-translation.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,28 @@ async function triggerAutoTranslation() {
console.log(`Crowdin: Triggering auto-translation of a branch: ${crowdinBranchName}`);

const sourceFilesWithBranchName = (await sourceFilesPromise).map(
(fileName) => crowdinBranchName + '/' + fileName,
(fileName) => `/${crowdinBranchName}/${fileName}`,
);

return CrowdinApi.preTranslate(sourceFilesWithBranchName)
.then((json) => {
if (!json.success) {
console.log(`Crowdin: ${COLORS.RED}Error:`);
console.log(json, COLORS.WHITE);
}
})
.catch((e) => {
console.log(
`Crowdin: ${COLORS.RED}Error: Failed to auto-translate branch ${branch.name}${COLORS.WHITE}`,
);
console.log(`Original error: ${e}`);
});
const branchData = await CrowdinApi.getBranchData(crowdinBranchName);
const filesData = await CrowdinApi.listFiles(branchData.data.id);
const fileIds = filesData.data
.filter((fileData) => sourceFilesWithBranchName.includes(fileData.data.path))
.map((fileData) => fileData.data.id);

try {
const translateResData = await CrowdinApi.preTranslate(fileIds);

if (translateResData.data.status !== 'created') {
console.log(`Crowdin: ${COLORS.RED}Error:`);
console.log(res, COLORS.WHITE);
}
} catch (e) {
console.log(
`Crowdin: ${COLORS.RED}Error: Failed to auto-translate branch ${crowdinBranchName}${COLORS.WHITE}`,
);
console.log(`Original error: ${e}`);
}
}

module.exports = triggerAutoTranslation;
Loading

0 comments on commit 9bcbe30

Please sign in to comment.