Skip to content

Commit

Permalink
Merge pull request #378 from crowdin/fix-omitted-files
Browse files Browse the repository at this point in the history 
Fix not showing all omitted files and improve building export patterns
  • Loading branch information
@andrii-bodnar
andrii-bodnar authored Aug 19, 2021
2 parents 45172f4 + 3aee0de commit a2b17d5
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 23 deletions.
47 changes: 29 additions & 18 deletions src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {

LanguageMapping serverLanguageMapping = project.getLanguageMapping();

Map<Pair<File, List<String>>, List<Map<String, String>>> fileBeansWithDownloadedFiles = new TreeMap<>();
List<File> tempDirs = new ArrayList<>();
Map<File, List<Map<String, String>>> fileBeansWithDownloadedFiles = new TreeMap<>();
Map<File, List<String>> tempDirs = new HashMap<>();
try {
if (pseudo) {
List<Language> forLanguages = project.getSupportedLanguages();
Expand All @@ -133,10 +133,10 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
Pair<File, List<String>> downloadedFiles = this.download(request, client, pb.getBasePath());
for (FileBean fb : pb.getFiles()) {
Map<String, String> filesWithMapping = this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy());
fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles, new ArrayList<>());
fileBeansWithDownloadedFiles.get(downloadedFiles).add(filesWithMapping);
fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles.getLeft(), new ArrayList<>());
fileBeansWithDownloadedFiles.get(downloadedFiles.getLeft()).add(filesWithMapping);
}
tempDirs.add(downloadedFiles.getLeft());
tempDirs.put(downloadedFiles.getLeft(), downloadedFiles.getRight());
} else {
List<Language> forLanguages = language
.map(Collections::singletonList)
Expand Down Expand Up @@ -176,17 +176,17 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {

Map<String, String> filesWithMapping =
this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy());
fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles, new ArrayList<>());
fileBeansWithDownloadedFiles.get(downloadedFiles).add(filesWithMapping);
fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles.getLeft(), new ArrayList<>());
fileBeansWithDownloadedFiles.get(downloadedFiles.getLeft()).add(filesWithMapping);
}
}
tempDirs.add(downloadedFiles.getLeft());
tempDirs.put(downloadedFiles.getLeft(), downloadedFiles.getRight());
});
}

Map<Pair<File, List<String>>, Set<Pair<String, String>>> fileBeansWithDownloadedFilesNoRepetitions = new TreeMap<>();
for (Pair<File, List<String>> key : fileBeansWithDownloadedFiles.keySet()) {
fileBeansWithDownloadedFilesNoRepetitions.put(key, this.flattenInnerMap(fileBeansWithDownloadedFiles.get(key)));
Map<File, Set<Pair<String, String>>> fileBeansWithDownloadedFilesNoRepetitions = new TreeMap<>();
for (File tempDir : fileBeansWithDownloadedFiles.keySet()) {
fileBeansWithDownloadedFilesNoRepetitions.put(tempDir, this.flattenInnerMap(fileBeansWithDownloadedFiles.get(tempDir)));
}

Map<Long, String> directoryPaths = (branch.isPresent())
Expand All @@ -198,12 +198,11 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
placeholderUtil, serverLanguageMapping, pb.getBasePath());

Map<String, List<String>> totalOmittedFiles = null;
List<String> totalOmittedFilesNoSources = new ArrayList<>();
List<List<String>> omittedFilesNoSources = new ArrayList<>();

for (Pair<File, List<String>> key : fileBeansWithDownloadedFilesNoRepetitions.keySet()) {
Set<Pair<String, String>> filesWithMapping = fileBeansWithDownloadedFilesNoRepetitions.get(key);
File tempDir = key.getKey();
List<String> downloadedFiles = key.getValue();
for (File tempDir : fileBeansWithDownloadedFilesNoRepetitions.keySet()) {
Set<Pair<String, String>> filesWithMapping = fileBeansWithDownloadedFilesNoRepetitions.get(tempDir);
List<String> downloadedFiles = tempDirs.get(tempDir);

Pair<Map<File, File>, List<String>> result =
sortFiles(downloadedFiles, filesWithMapping, pb.getBasePath(), tempDir.getAbsolutePath() + Utils.PATH_SEPARATOR);
Expand Down Expand Up @@ -240,7 +239,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
}
}
}
totalOmittedFilesNoSources.retainAll(allOmittedFilesNoSources);
omittedFilesNoSources.add(allOmittedFilesNoSources);
}

if (!ignoreMatch && !plainView) {
Expand All @@ -258,6 +257,11 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
}
});
}

List<String> totalOmittedFilesNoSources = omittedFilesNoSources.isEmpty() ? new ArrayList<>() : omittedFilesNoSources.get(0);
for (List<String> eachOmittedFilesNoSources : omittedFilesNoSources) {
totalOmittedFilesNoSources.retainAll(eachOmittedFilesNoSources);
}
if (!totalOmittedFilesNoSources.isEmpty()) {
out.println(
WARNING.withIcon(
Expand All @@ -268,7 +272,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
}
} finally {
try {
for (File tempDir : tempDirs) {
for (File tempDir : tempDirs.keySet()) {
files.deleteDirectory(tempDir);
}
} catch (IOException e) {
Expand All @@ -277,6 +281,13 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
}
}

/**
* Download archive, extract it and return information about that temporary directory
* @param request request body to download archive
* @param client api to Crowdin
* @param basePath base path
* @return pair of temporary directory and list of files in it(relative paths to that directory)
*/
private Pair<File, List<String>> download(BuildProjectTranslationRequest request, ProjectClient client, String basePath) {
ProjectBuild projectBuild = buildTranslation(client, request);
String randomHash = RandomStringUtils.random(11, false, true);
Expand Down
19 changes: 14 additions & 5 deletions src/main/java/com/crowdin/cli/commands/functionality/ProjectFilesUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_LANGUAGE_ID;

public class ProjectFilesUtils {

public static <T extends FileInfo> Map<String, T> buildFilePaths(
Expand Down Expand Up @@ -71,15 +73,22 @@ public static Map<String, List<String>> buildAllProjectTranslations(
continue;
}
String path = getParentId(fe).map(directoryPaths::get).orElse("") + fe.getName();
Stream<String> translations = isMultilingualFile(fe)
? Stream.concat(
Stream<String> translations;
if (isMultilingualFile(fe)) {
translations = Stream.concat(
Stream.of(Utils.normalizePath(fe.getName())),
placeholderUtil.replaceLanguageDependentPlaceholders(
Utils.normalizePath("%language_id%/" + fe.getName()), languageMapping).stream())
: placeholderUtil.replaceLanguageDependentPlaceholders(Utils.normalizePath(getExportPattern(fe.getExportOptions())), languageMapping)
Utils.joinPaths(PLACEHOLDER_LANGUAGE_ID, fe.getName()), languageMapping).stream());
} else {
String exportPattern = Utils.normalizePath(getExportPattern(fe.getExportOptions()));
if (!PlaceholderUtil.containsLangPlaceholders(exportPattern)) {
exportPattern = Utils.joinPaths(PLACEHOLDER_LANGUAGE_ID, exportPattern);
}
String fileDependentPlaceholdersReplaced = placeholderUtil.replaceFileDependentPlaceholders(exportPattern, new java.io.File(basePath + path));
translations = placeholderUtil.replaceLanguageDependentPlaceholders(fileDependentPlaceholdersReplaced, languageMapping)
.stream()
.map(tr -> placeholderUtil.replaceFileDependentPlaceholders(tr, new java.io.File(basePath + path)))
.map(translation -> ((fe.getBranchId() != null) ? directoryPaths.getOrDefault(fe.getBranchId(), "") : "") + translation);
}
allProjectTranslations.put(path, translations.collect(Collectors.toList()));
}
return allProjectTranslations;
Expand Down

0 comments on commit a2b17d5

Please sign in to comment.