From 3aee0de651a8882aed5986db131b4137acb0de68 Mon Sep 17 00:00:00 2001 From: Daniil Barabash Date: Thu, 8 Jul 2021 15:34:11 +0300 Subject: [PATCH] Fix not showing all omitted files and improve building export patterns --- .../cli/commands/actions/DownloadAction.java | 47 ++++++++++++------- .../functionality/ProjectFilesUtils.java | 19 ++++++-- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java b/src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java index 8d7e3b90a..50be197f3 100644 --- a/src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java +++ b/src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java @@ -117,8 +117,8 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { LanguageMapping serverLanguageMapping = project.getLanguageMapping(); - Map>, List>> fileBeansWithDownloadedFiles = new TreeMap<>(); - List tempDirs = new ArrayList<>(); + Map>> fileBeansWithDownloadedFiles = new TreeMap<>(); + Map> tempDirs = new HashMap<>(); try { if (pseudo) { List forLanguages = project.getSupportedLanguages(); @@ -133,10 +133,10 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { Pair> downloadedFiles = this.download(request, client, pb.getBasePath()); for (FileBean fb : pb.getFiles()) { Map filesWithMapping = this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy()); - fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles, new ArrayList<>()); - fileBeansWithDownloadedFiles.get(downloadedFiles).add(filesWithMapping); + fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles.getLeft(), new ArrayList<>()); + fileBeansWithDownloadedFiles.get(downloadedFiles.getLeft()).add(filesWithMapping); } - tempDirs.add(downloadedFiles.getLeft()); + tempDirs.put(downloadedFiles.getLeft(), downloadedFiles.getRight()); } else { List forLanguages = language .map(Collections::singletonList) @@ -176,17 +176,17 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { Map filesWithMapping = this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy()); - fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles, new ArrayList<>()); - fileBeansWithDownloadedFiles.get(downloadedFiles).add(filesWithMapping); + fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles.getLeft(), new ArrayList<>()); + fileBeansWithDownloadedFiles.get(downloadedFiles.getLeft()).add(filesWithMapping); } } - tempDirs.add(downloadedFiles.getLeft()); + tempDirs.put(downloadedFiles.getLeft(), downloadedFiles.getRight()); }); } - Map>, Set>> fileBeansWithDownloadedFilesNoRepetitions = new TreeMap<>(); - for (Pair> key : fileBeansWithDownloadedFiles.keySet()) { - fileBeansWithDownloadedFilesNoRepetitions.put(key, this.flattenInnerMap(fileBeansWithDownloadedFiles.get(key))); + Map>> fileBeansWithDownloadedFilesNoRepetitions = new TreeMap<>(); + for (File tempDir : fileBeansWithDownloadedFiles.keySet()) { + fileBeansWithDownloadedFilesNoRepetitions.put(tempDir, this.flattenInnerMap(fileBeansWithDownloadedFiles.get(tempDir))); } Map directoryPaths = (branch.isPresent()) @@ -198,12 +198,11 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { placeholderUtil, serverLanguageMapping, pb.getBasePath()); Map> totalOmittedFiles = null; - List totalOmittedFilesNoSources = new ArrayList<>(); + List> omittedFilesNoSources = new ArrayList<>(); - for (Pair> key : fileBeansWithDownloadedFilesNoRepetitions.keySet()) { - Set> filesWithMapping = fileBeansWithDownloadedFilesNoRepetitions.get(key); - File tempDir = key.getKey(); - List downloadedFiles = key.getValue(); + for (File tempDir : fileBeansWithDownloadedFilesNoRepetitions.keySet()) { + Set> filesWithMapping = fileBeansWithDownloadedFilesNoRepetitions.get(tempDir); + List downloadedFiles = tempDirs.get(tempDir); Pair, List> result = sortFiles(downloadedFiles, filesWithMapping, pb.getBasePath(), tempDir.getAbsolutePath() + Utils.PATH_SEPARATOR); @@ -240,7 +239,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { } } } - totalOmittedFilesNoSources.retainAll(allOmittedFilesNoSources); + omittedFilesNoSources.add(allOmittedFilesNoSources); } if (!ignoreMatch && !plainView) { @@ -258,6 +257,11 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { } }); } + + List totalOmittedFilesNoSources = omittedFilesNoSources.isEmpty() ? new ArrayList<>() : omittedFilesNoSources.get(0); + for (List eachOmittedFilesNoSources : omittedFilesNoSources) { + totalOmittedFilesNoSources.retainAll(eachOmittedFilesNoSources); + } if (!totalOmittedFilesNoSources.isEmpty()) { out.println( WARNING.withIcon( @@ -268,7 +272,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { } } finally { try { - for (File tempDir : tempDirs) { + for (File tempDir : tempDirs.keySet()) { files.deleteDirectory(tempDir); } } catch (IOException e) { @@ -277,6 +281,13 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) { } } + /** + * Download archive, extract it and return information about that temporary directory + * @param request request body to download archive + * @param client api to Crowdin + * @param basePath base path + * @return pair of temporary directory and list of files in it(relative paths to that directory) + */ private Pair> download(BuildProjectTranslationRequest request, ProjectClient client, String basePath) { ProjectBuild projectBuild = buildTranslation(client, request); String randomHash = RandomStringUtils.random(11, false, true); diff --git a/src/main/java/com/crowdin/cli/commands/functionality/ProjectFilesUtils.java b/src/main/java/com/crowdin/cli/commands/functionality/ProjectFilesUtils.java index 582ebbd67..6e15db383 100644 --- a/src/main/java/com/crowdin/cli/commands/functionality/ProjectFilesUtils.java +++ b/src/main/java/com/crowdin/cli/commands/functionality/ProjectFilesUtils.java @@ -19,6 +19,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.crowdin.cli.utils.PlaceholderUtil.PLACEHOLDER_LANGUAGE_ID; + public class ProjectFilesUtils { public static Map buildFilePaths( @@ -67,15 +69,22 @@ public static Map> buildAllProjectTranslations( continue; } String path = getParentId(fe).map(directoryPaths::get).orElse("") + fe.getName(); - Stream translations = isMultilingualFile(fe) - ? Stream.concat( + Stream translations; + if (isMultilingualFile(fe)) { + translations = Stream.concat( Stream.of(Utils.normalizePath(fe.getName())), placeholderUtil.replaceLanguageDependentPlaceholders( - Utils.normalizePath("%language_id%/" + fe.getName()), languageMapping).stream()) - : placeholderUtil.replaceLanguageDependentPlaceholders(Utils.normalizePath(getExportPattern(fe.getExportOptions())), languageMapping) + Utils.joinPaths(PLACEHOLDER_LANGUAGE_ID, fe.getName()), languageMapping).stream()); + } else { + String exportPattern = Utils.normalizePath(getExportPattern(fe.getExportOptions())); + if (!PlaceholderUtil.containsLangPlaceholders(exportPattern)) { + exportPattern = Utils.joinPaths(PLACEHOLDER_LANGUAGE_ID, exportPattern); + } + String fileDependentPlaceholdersReplaced = placeholderUtil.replaceFileDependentPlaceholders(exportPattern, new java.io.File(basePath + path)); + translations = placeholderUtil.replaceLanguageDependentPlaceholders(fileDependentPlaceholdersReplaced, languageMapping) .stream() - .map(tr -> placeholderUtil.replaceFileDependentPlaceholders(tr, new java.io.File(basePath + path))) .map(translation -> ((fe.getBranchId() != null) ? directoryPaths.getOrDefault(fe.getBranchId(), "") : "") + translation); + } allProjectTranslations.put(path, translations.collect(Collectors.toList())); } return allProjectTranslations;