Skip to content

Commit

Permalink
Merge pull request #354 from crowdin/dev
Browse files Browse the repository at this point in the history
Improvements
  • Loading branch information
andrii-bodnar authored Apr 30, 2021
2 parents 603fd98 + 4898fb2 commit 95582db
Show file tree
Hide file tree
Showing 20 changed files with 493 additions and 10 deletions.
4 changes: 4 additions & 0 deletions src/main/java/com/crowdin/cli/BaseCli.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,8 @@ public class BaseCli {
public static final String HTTP_PROXY_PASSWORD_ENV = "HTTP_PROXY_PASSWORD";

public static final String IGNORE_HIDDEN_FILES_PATTERN = "**/.*";

public enum LanguageCode {
two_letters_code, three_letters_code, locale, android_code, osx_code, osx_locale
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected static <T, R extends Exception> T executeRequest(Map<BiPredicate<Strin
for (HttpBadRequestException.Error error : eh.getError().errors) {
String code = (error.code != null) ? error.code : "<empty_code>";
String message = (error.message != null) ? error.message : "<empty_message>";
searchErrorHandler(errorHandlers, error.getCode(), error.getMessage());
searchErrorHandler(errorHandlers, code, message);
}
}
String errorMessage = "Wrong parameters: \n" + e.getErrors()
Expand Down
18 changes: 17 additions & 1 deletion src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
import com.crowdin.client.sourcestrings.model.AddSourceStringRequest;
import com.crowdin.client.sourcestrings.model.SourceString;
import com.crowdin.client.storage.model.Storage;
import com.crowdin.client.translations.model.ApplyPreTranslationRequest;
import com.crowdin.client.translations.model.BuildProjectTranslationRequest;
import com.crowdin.client.translations.model.ExportProjectTranslationRequest;
import com.crowdin.client.translations.model.PreTranslationStatus;
import com.crowdin.client.translations.model.ProjectBuild;
import com.crowdin.client.translations.model.UploadTranslationsRequest;
import com.crowdin.client.translationstatus.model.LanguageProgress;
Expand Down Expand Up @@ -144,7 +146,7 @@ public void updateSource(Long sourceId, UpdateFileRequest request) {
public void addSource(AddFileRequest request) throws ResponseException {
Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException());
put((code, message) -> StringUtils.containsAny(message, "Name must be unique"), new ExistsResponseException());
put((code, message) -> StringUtils.contains(message, "Name must be unique"), new ExistsResponseException());
}};
executeRequestWithPossibleRetry(
errorHandlers,
Expand Down Expand Up @@ -248,4 +250,18 @@ public URL downloadFile(Long fileId) {
.downloadFile(this.projectId, fileId)
.getData()));
}

@Override
public PreTranslationStatus startPreTranslation(ApplyPreTranslationRequest request) {
return executeRequest(() ->this.client.getTranslationsApi()
.applyPreTranslation(this.projectId, request)
.getData());
}

@Override
public PreTranslationStatus checkPreTranslation(String preTranslationId) {
return executeRequest(() -> this.client.getTranslationsApi()
.preTranslationStatus(this.projectId, preTranslationId)
.getData());
}
}
6 changes: 6 additions & 0 deletions src/main/java/com/crowdin/cli/client/ProjectClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
import com.crowdin.client.sourcefiles.model.UpdateFileRequest;
import com.crowdin.client.sourcestrings.model.AddSourceStringRequest;
import com.crowdin.client.sourcestrings.model.SourceString;
import com.crowdin.client.translations.model.ApplyPreTranslationRequest;
import com.crowdin.client.translations.model.BuildProjectTranslationRequest;
import com.crowdin.client.translations.model.ExportProjectTranslationRequest;
import com.crowdin.client.translations.model.PreTranslationStatus;
import com.crowdin.client.translations.model.ProjectBuild;
import com.crowdin.client.translations.model.UploadTranslationsRequest;
import com.crowdin.client.translationstatus.model.LanguageProgress;
Expand Down Expand Up @@ -66,4 +68,8 @@ public interface ProjectClient extends Client {
Label addLabel(AddLabelRequest request);

URL downloadFile(Long fileId);

PreTranslationStatus startPreTranslation(ApplyPreTranslationRequest request);

PreTranslationStatus checkPreTranslation(String preTranslationId);
}
8 changes: 8 additions & 0 deletions src/main/java/com/crowdin/cli/commands/Actions.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.crowdin.cli.commands;

import com.crowdin.cli.BaseCli;
import com.crowdin.cli.client.ClientGlossary;
import com.crowdin.cli.client.ClientTm;
import com.crowdin.cli.client.NoClient;
Expand All @@ -11,6 +12,8 @@
import com.crowdin.cli.properties.PropertiesWithFiles;
import com.crowdin.client.glossaries.model.GlossariesFormat;
import com.crowdin.client.translationmemory.model.TranslationMemoryFormat;
import com.crowdin.client.translations.model.AutoApproveOption;
import com.crowdin.client.translations.model.Method;

import java.io.File;
import java.nio.file.Path;
Expand All @@ -37,6 +40,8 @@ NewAction<PropertiesWithFiles, ProjectClient> listSources(
NewAction<PropertiesWithFiles, ProjectClient> listTranslations(
boolean noProgress, boolean treeView, boolean isLocal, boolean plainView, boolean useServerSources, boolean withInContextLang);

NewAction<PropertiesWithFiles, ProjectClient> listLanguages(BaseCli.LanguageCode code, boolean noProgress, boolean plainView);

NewAction<PropertiesWithFiles, ProjectClient> status(
boolean noProgress, String languageId, boolean isVerbose, boolean showTranslated, boolean showApproved);

Expand Down Expand Up @@ -82,5 +87,8 @@ NewAction<PropertiesWithTargets, ProjectClient> downloadTargets(

NewAction<NoProperties, NoClient> checkNewVersion();

NewAction<PropertiesWithFiles, ProjectClient> preTranslate(
List<String> languageIds, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean debug, boolean verbose, boolean plainView);

}
16 changes: 16 additions & 0 deletions src/main/java/com/crowdin/cli/commands/actions/CliActions.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.crowdin.cli.commands.actions;

import com.crowdin.cli.BaseCli;
import com.crowdin.cli.client.ClientGlossary;
import com.crowdin.cli.client.ClientTm;
import com.crowdin.cli.client.NoClient;
Expand All @@ -13,6 +14,8 @@
import com.crowdin.cli.properties.PropertiesWithFiles;
import com.crowdin.client.glossaries.model.GlossariesFormat;
import com.crowdin.client.translationmemory.model.TranslationMemoryFormat;
import com.crowdin.client.translations.model.AutoApproveOption;
import com.crowdin.client.translations.model.Method;

import java.io.File;
import java.nio.file.Path;
Expand Down Expand Up @@ -60,6 +63,11 @@ public NewAction<PropertiesWithFiles, ProjectClient> listTranslations(
return new ListTranslationsAction(noProgress, treeView, isLocal, plainView, useServerSources, withInContextLang);
}

@Override
public NewAction<PropertiesWithFiles, ProjectClient> listLanguages(BaseCli.LanguageCode code, boolean noProgress, boolean plainView) {
return new ListLanguagesAction(code, noProgress, plainView);
}

@Override
public NewAction<PropertiesWithFiles, ProjectClient> status(
boolean noProgress, String languageId, boolean isVerbose, boolean showTranslated, boolean showApproved
Expand Down Expand Up @@ -162,5 +170,13 @@ public NewAction<PropertiesWithTargets, ProjectClient> downloadTargets(
return new DownloadTargetsAction(targetNames, files, noProgress, langIds, isVerbose, plainView, debug, branchName);
}

@Override
public NewAction<PropertiesWithFiles, ProjectClient> preTranslate(
List<String> languageIds, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean debug, boolean verbose, boolean plainView
) {
return new PreTranslateAction(languageIds, method, engineId, branchName, autoApproveOption, duplicateTranslations,
translateUntranslatedOnly, translateWithPerfectMatchOnly, noProgress, debug, verbose, plainView);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package com.crowdin.cli.commands.actions;

import com.crowdin.cli.BaseCli;
import com.crowdin.cli.client.CrowdinProjectInfo;
import com.crowdin.cli.client.LanguageMapping;
import com.crowdin.cli.client.ProjectClient;
import com.crowdin.cli.commands.NewAction;
import com.crowdin.cli.commands.Outputter;
import com.crowdin.cli.properties.PropertiesWithFiles;
import com.crowdin.cli.utils.console.ConsoleSpinner;
import com.crowdin.client.languages.model.Language;

import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE;
import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_ANDROID_CODE;
import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_LOCALE;
import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_OSX_CODE;
import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_OSX_LOCALE;
import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_THREE_LETTERS_CODE;
import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_TWO_LETTERS_CODE;
import static com.crowdin.cli.utils.console.ExecutionStatus.OK;
import static com.crowdin.cli.utils.console.ExecutionStatus.WARNING;

class ListLanguagesAction implements NewAction<PropertiesWithFiles, ProjectClient> {

private BaseCli.LanguageCode code;
private boolean noProgress;
private boolean plainView;

public ListLanguagesAction(BaseCli.LanguageCode code, boolean noProgress, boolean plainView) {
this.code = code;
this.noProgress = noProgress || plainView;
this.plainView = plainView;
}

@Override
public void act(Outputter out, PropertiesWithFiles properties, ProjectClient client) {
CrowdinProjectInfo project = ConsoleSpinner.execute(out, "message.spinner.fetching_project_info", "error.collect_project_info",
this.noProgress, this.plainView, client::downloadProjectInfo);

if (!project.isManagerAccess()) {
if (!plainView) {
out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("message.no_manager_access")));
return;
} else {
throw new RuntimeException(RESOURCE_BUNDLE.getString("message.no_manager_access"));
}
}

LanguageMapping langMapping = project.getLanguageMapping();
if (!plainView) {
project.getProjectLanguages(true).stream()
.map(lang -> String.format(RESOURCE_BUNDLE.getString("message.description"), lang.getName(), this.getCode(langMapping, lang)))
.map(OK::withIcon)
.forEach(out::println);
} else {
project.getProjectLanguages(true).stream()
.map(lang -> this.getCode(langMapping, lang))
.forEach(out::println);
}
}

private String getCode(LanguageMapping langMapping, Language language) {
if (code == null) {
return language.getTwoLettersCode();
}
switch (code) {
case three_letters_code:
return langMapping.getValueOrDefault(language.getId(),
PLACEHOLDER_THREE_LETTERS_CODE.replaceAll("%", ""), language.getThreeLettersCode());
case android_code:
return langMapping.getValueOrDefault(language.getId(), PLACEHOLDER_ANDROID_CODE.replaceAll("%", ""), language.getAndroidCode());
case locale:
return langMapping.getValueOrDefault(language.getId(), PLACEHOLDER_LOCALE.replaceAll("%", ""), language.getLocale());
case osx_code:
return langMapping.getValueOrDefault(language.getId(), PLACEHOLDER_OSX_CODE.replaceAll("%", ""), language.getOsxCode());
case osx_locale:
return langMapping.getValueOrDefault(language.getId(), PLACEHOLDER_OSX_LOCALE.replaceAll("%", ""), language.getOsxLocale());
case two_letters_code:
default:
return langMapping.getValueOrDefault(language.getId(),
PLACEHOLDER_TWO_LETTERS_CODE.replaceAll("%", ""), language.getTwoLettersCode());
}
}
}
154 changes: 154 additions & 0 deletions src/main/java/com/crowdin/cli/commands/actions/PreTranslateAction.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package com.crowdin.cli.commands.actions;

import com.crowdin.cli.BaseCli;
import com.crowdin.cli.client.CrowdinProjectFull;
import com.crowdin.cli.client.CrowdinProjectInfo;
import com.crowdin.cli.client.ProjectClient;
import com.crowdin.cli.commands.NewAction;
import com.crowdin.cli.commands.Outputter;
import com.crowdin.cli.commands.functionality.ProjectFilesUtils;
import com.crowdin.cli.commands.functionality.PropertiesBeanUtils;
import com.crowdin.cli.commands.functionality.RequestBuilder;
import com.crowdin.cli.commands.functionality.SourcesUtils;
import com.crowdin.cli.properties.PropertiesWithFiles;
import com.crowdin.cli.utils.PlaceholderUtil;
import com.crowdin.cli.utils.Utils;
import com.crowdin.cli.utils.console.ConsoleSpinner;
import com.crowdin.client.languages.model.Language;
import com.crowdin.client.sourcefiles.model.FileInfo;
import com.crowdin.client.translations.model.ApplyPreTranslationRequest;
import com.crowdin.client.translations.model.AutoApproveOption;
import com.crowdin.client.translations.model.Method;
import com.crowdin.client.translations.model.PreTranslationStatus;
import org.apache.commons.lang3.StringUtils;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE;
import static com.crowdin.cli.utils.console.ExecutionStatus.WARNING;

class PreTranslateAction implements NewAction<PropertiesWithFiles, ProjectClient> {

private List<String> languageIds;
private Method method;
private Long engineId;
private String branchName;
private AutoApproveOption autoApproveOption;
private Boolean duplicateTranslations;
private Boolean translateUntranslatedOnly;
private Boolean translateWithPerfectMatchOnly;
private boolean noProgress;
private boolean debug;
private boolean verbose;
private boolean plainView;

public PreTranslateAction(
List<String> languageIds, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean debug, boolean verbose, boolean plainView
) {
this.languageIds = languageIds;
this.method = method;
this.engineId = engineId;
this.branchName = branchName;
this.autoApproveOption = autoApproveOption;
this.duplicateTranslations = duplicateTranslations;
this.translateUntranslatedOnly = translateUntranslatedOnly;
this.translateWithPerfectMatchOnly = translateWithPerfectMatchOnly;
this.noProgress = noProgress;
this.debug = debug;
this.verbose = verbose;
this.plainView = plainView;
}

@Override
public void act(Outputter out, PropertiesWithFiles properties, ProjectClient client) {
CrowdinProjectFull project = ConsoleSpinner.execute(out, "message.spinner.fetching_project_info", "error.collect_project_info",
this.noProgress, this.plainView, client::downloadFullProject);

List<String> languages = this.prepareLanguageIds(project);
List<Long> fileIds = this.prepareFileIds(out, properties, project);
ApplyPreTranslationRequest request = RequestBuilder.applyPreTranslation(
languages, fileIds, method, engineId, autoApproveOption,
duplicateTranslations, translateUntranslatedOnly, translateWithPerfectMatchOnly);
this.applyPreTranslation(out, client, request);

}

private List<String> prepareLanguageIds(CrowdinProjectInfo projectInfo) {
List<String> projectLanguages = projectInfo.getProjectLanguages(false).stream()
.map(Language::getId)
.collect(Collectors.toList());
if (languageIds.size() == 1 && BaseCli.ALL.equals(languageIds.get(0))) {
return projectLanguages;
} else {
String wrongLanguageIds = languageIds.stream()
.filter(langId -> !projectLanguages.contains(langId))
.map(id -> "'" + id + "'")
.collect(Collectors.joining(", "));
if (!wrongLanguageIds.isEmpty()) {
throw new RuntimeException(
String.format(RESOURCE_BUNDLE.getString("error.languages_not_exist"), wrongLanguageIds));
}
return languageIds;
}
}

private List<Long> prepareFileIds(Outputter out, PropertiesWithFiles pb, CrowdinProjectFull project) {
Map<String, FileInfo> paths = ProjectFilesUtils.buildFilePaths(project.getDirectories(), project.getBranches(), project.getFileInfos());
PlaceholderUtil placeholderUtil = new PlaceholderUtil(project.getSupportedLanguages(), project.getProjectLanguages(false), pb.getBasePath());
List<String> sourcePaths = pb.getFiles().stream()
.flatMap(file -> {
List<String> sources = SourcesUtils.getFiles(pb.getBasePath(), file.getSource(), file.getIgnore(), placeholderUtil)
.map(File::getAbsolutePath)
.collect(Collectors.toList());
String commonPath = (pb.getPreserveHierarchy()) ? "" : SourcesUtils.getCommonPath(sources, pb.getBasePath());
return sources.stream()
.map(source -> (file.getDest() != null)
? PropertiesBeanUtils.prepareDest(file.getDest(), source) : StringUtils.removeStart(source, pb.getBasePath() + commonPath))
.map(source -> (branchName != null ? branchName + Utils.PATH_SEPARATOR : "") + source);
})
.distinct()
.collect(Collectors.toList());
List<String> onlyLocalSources = new ArrayList<>();
List<String> foundSources = new ArrayList<>();
for (String sourcePath : sourcePaths) {
if (paths.containsKey(sourcePath)) {
foundSources.add(sourcePath);
} else {
onlyLocalSources.add(sourcePath);
}
}
if (!onlyLocalSources.isEmpty()) {
if (verbose) {
out.println(WARNING.withIcon(String.format(RESOURCE_BUNDLE.getString("message.pre_translate.local_files_message_verbose"), sourcePaths.size(), onlyLocalSources.size())));
onlyLocalSources.forEach(source -> out.println(String.format(RESOURCE_BUNDLE.getString("message.item_list"), source)));
} else {
out.println(WARNING.withIcon(String.format(RESOURCE_BUNDLE.getString("message.pre_translate.local_files_message"), sourcePaths.size(), onlyLocalSources.size())));
}
}
return foundSources.stream()
.map(paths::get)
.map(FileInfo::getId)
.collect(Collectors.toList());
}

private PreTranslationStatus applyPreTranslation(Outputter out, ProjectClient client,ApplyPreTranslationRequest request) {
return ConsoleSpinner.execute(out, "message.spinner.pre_translate", "error.spinner.pre_translate", this.noProgress, this.plainView, () -> {
PreTranslationStatus preTranslationStatus = client.startPreTranslation(request);

while (!preTranslationStatus.getStatus().equalsIgnoreCase("finished")) {
ConsoleSpinner.update(
String.format(RESOURCE_BUNDLE.getString("message.spinner.pre_translate_percents"),
Math.toIntExact(preTranslationStatus.getProgress())));
Thread.sleep(100);
preTranslationStatus = client.checkPreTranslation(preTranslationStatus.getIdentifier());
}
ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.spinner.pre_translate_done"), 100));
return preTranslationStatus;
});
}
}
Loading

0 comments on commit 95582db

Please sign in to comment.