From cc910c6f53fcad38b0266b968af9f0242310dea7 Mon Sep 17 00:00:00 2001 From: unknown <1463567152@qq.com> Date: Tue, 15 Apr 2025 13:11:11 +0800 Subject: [PATCH 1/8] zip encoding detect --- Directory.Packages.props | 1 + .../Archives/Decompress/DecompressArchive.cs | 8 +++- .../Data/Contracts/IStorageArchiveService.cs | 9 +++- src/Files.App/Data/Items/EncodingItem.cs | 21 +++++++++- src/Files.App/Files.App.csproj | 1 + .../Services/Storage/StorageArchiveService.cs | 39 +++++++++++++++++ src/Files.App/Strings/en-US/Resources.resw | 3 ++ .../DecompressArchiveDialogViewModel.cs | 42 +++++++++++++------ 8 files changed, 109 insertions(+), 15 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 66cbcf3e49cd..81465d39eb4e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -38,6 +38,7 @@ + diff --git a/src/Files.App/Actions/Content/Archives/Decompress/DecompressArchive.cs b/src/Files.App/Actions/Content/Archives/Decompress/DecompressArchive.cs index 0bff8f7ce6f3..ecff3a898fed 100644 --- a/src/Files.App/Actions/Content/Archives/Decompress/DecompressArchive.cs +++ b/src/Files.App/Actions/Content/Archives/Decompress/DecompressArchive.cs @@ -43,6 +43,11 @@ public override async Task ExecuteAsync(object? parameter = null) var isArchiveEncrypted = await FilesystemTasks.Wrap(() => StorageArchiveService.IsEncryptedAsync(archive.Path)); var isArchiveEncodingUndetermined = await FilesystemTasks.Wrap(() => StorageArchiveService.IsEncodingUndeterminedAsync(archive.Path)); + Encoding? detectedEncoding = null; + if (isArchiveEncodingUndetermined) + { + detectedEncoding = await FilesystemTasks.Wrap(() => StorageArchiveService.DetectEncodingAsync(archive.Path)); + } var password = string.Empty; Encoding? encoding = null; @@ -51,7 +56,8 @@ public override async Task ExecuteAsync(object? parameter = null) { IsArchiveEncrypted = isArchiveEncrypted, IsArchiveEncodingUndetermined = isArchiveEncodingUndetermined, - ShowPathSelection = true + ShowPathSelection = true, + DetectedEncoding = detectedEncoding, }; decompressArchiveDialog.ViewModel = decompressArchiveViewModel; diff --git a/src/Files.App/Data/Contracts/IStorageArchiveService.cs b/src/Files.App/Data/Contracts/IStorageArchiveService.cs index 27487eb1763c..ee0ee550bfdc 100644 --- a/src/Files.App/Data/Contracts/IStorageArchiveService.cs +++ b/src/Files.App/Data/Contracts/IStorageArchiveService.cs @@ -59,10 +59,17 @@ public interface IStorageArchiveService /// /// Gets the value that indicates whether the archive file's encoding is undetermined. /// - /// The archive file path to check if the item is encrypted. + /// The archive file path to check if the encoding is undetermined. /// True if the archive file's encoding is undetermined; otherwise, false. Task IsEncodingUndeterminedAsync(string archiveFilePath); + /// + /// Detect encoding for a zip file whose encoding is undetermined. + /// + /// The archive file path to detect encoding + /// Null if the archive file doesn't need to detect encoding or its encoding can't be detected; otherwise, the encoding detected. + Task DetectEncodingAsync(string archiveFilePath); + /// /// Gets the instance from the archive file path. /// diff --git a/src/Files.App/Data/Items/EncodingItem.cs b/src/Files.App/Data/Items/EncodingItem.cs index c342c64fc9c8..511cbd3578fb 100644 --- a/src/Files.App/Data/Items/EncodingItem.cs +++ b/src/Files.App/Data/Items/EncodingItem.cs @@ -36,6 +36,25 @@ public EncodingItem(string code) } } - public override string ToString() => Name; + public EncodingItem(Encoding encoding, string name) + { + Encoding = encoding; + Name = name; + } + + public static EncodingItem[] Defaults = new string?[] { + null,//System Default + "UTF-8", + "shift_jis", + "gb2312", + "big5", + "ks_c_5601-1987", + "Windows-1252", + "macintosh", + } + .Select(x => new EncodingItem(x)) + .ToArray(); + + public override string ToString() => Name; } } diff --git a/src/Files.App/Files.App.csproj b/src/Files.App/Files.App.csproj index 78196a615a92..6ed8fc5d9166 100644 --- a/src/Files.App/Files.App.csproj +++ b/src/Files.App/Files.App.csproj @@ -95,6 +95,7 @@ + diff --git a/src/Files.App/Services/Storage/StorageArchiveService.cs b/src/Files.App/Services/Storage/StorageArchiveService.cs index 1b30e1adaddf..8f6793f128ca 100644 --- a/src/Files.App/Services/Storage/StorageArchiveService.cs +++ b/src/Files.App/Services/Storage/StorageArchiveService.cs @@ -5,9 +5,12 @@ using ICSharpCode.SharpZipLib.Core; using ICSharpCode.SharpZipLib.Zip; using SevenZip; +using System.Collections; using System.IO; using System.Linq; using System.Text; +using UtfUnknown; +using Vanara.PInvoke; using Windows.Storage; using Windows.Win32; @@ -365,6 +368,42 @@ public async Task IsEncodingUndeterminedAsync(string archiveFilePath) } } + public async Task DetectEncodingAsync(string archiveFilePath) + { + //Temporarily using cp437 to decode zip file + //because SharpZipLib requires an encoding when decoding + //and cp437 contains all bytes as character + //which means that we can store any byte array as cp437 string losslessly + var cp437 = Encoding.GetEncoding(437); + try + { + using (ZipFile zipFile = new ZipFile(archiveFilePath, StringCodec.FromEncoding(cp437))) + { + var fileNameBytes = cp437.GetBytes( + String.Join("\n", + zipFile.Cast() + .Where(e => !e.IsUnicodeText) + .Select(e => e.Name) + ) + ); + var detectionResult = CharsetDetector.DetectFromBytes(fileNameBytes); + if (detectionResult.Detected != null && detectionResult.Detected.Confidence > 0.5) + { + return detectionResult.Detected.Encoding; + } + else + { + return null; + } + } + } + catch (Exception ex) + { + Console.WriteLine($"SharpZipLib error: {ex.Message}"); + return null; + } + } + /// public async Task GetSevenZipExtractorAsync(string archiveFilePath, string password = "") { diff --git a/src/Files.App/Strings/en-US/Resources.resw b/src/Files.App/Strings/en-US/Resources.resw index c829bb40622f..dd7e302ef424 100644 --- a/src/Files.App/Strings/en-US/Resources.resw +++ b/src/Files.App/Strings/en-US/Resources.resw @@ -2099,6 +2099,9 @@ Encoding + + (detected) + Path diff --git a/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs b/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs index b1ff52bc2b35..f96ba27bac86 100644 --- a/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs +++ b/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs @@ -44,6 +44,16 @@ public bool IsArchiveEncodingUndetermined set => SetProperty(ref isArchiveEncodingUndetermined, value); } + private Encoding? detectedEncoding; + public Encoding? DetectedEncoding + { + get => detectedEncoding; + set { + SetProperty(ref detectedEncoding, value); + RefreshEncodingOptions(); + } + } + private bool showPathSelection; public bool ShowPathSelection { @@ -53,19 +63,27 @@ public bool ShowPathSelection public DisposableArray? Password { get; private set; } - public EncodingItem[] EncodingOptions { get; set; } = new string?[] { - null,//System Default - "UTF-8", - "shift_jis", - "gb2312", - "big5", - "ks_c_5601-1987", - "Windows-1252", - "macintosh", - } - .Select(x=>new EncodingItem(x)) - .ToArray(); + public EncodingItem[] EncodingOptions { get; set; } = EncodingItem.Defaults; public EncodingItem SelectedEncoding { get; set; } + void RefreshEncodingOptions() + { + if (detectedEncoding != null) + { + EncodingOptions = EncodingItem.Defaults + .Prepend(new EncodingItem( + detectedEncoding, + detectedEncoding.EncodingName + Strings.EncodingDetected.GetLocalizedResource()) + ) + .ToArray(); + } + else + { + EncodingOptions = EncodingItem.Defaults; + } + SelectedEncoding = EncodingOptions.FirstOrDefault(); + } + + public IRelayCommand PrimaryButtonClickCommand { get; private set; } From b4f3ac295153f4833f481dbdfbb193e331a905bc Mon Sep 17 00:00:00 2001 From: unknown <1463567152@qq.com> Date: Tue, 15 Apr 2025 14:14:51 +0800 Subject: [PATCH 2/8] fix sharpziplib blocking UI thread --- .../Services/Storage/StorageArchiveService.cs | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/Files.App/Services/Storage/StorageArchiveService.cs b/src/Files.App/Services/Storage/StorageArchiveService.cs index 8f6793f128ca..dc84a44539a8 100644 --- a/src/Files.App/Services/Storage/StorageArchiveService.cs +++ b/src/Files.App/Services/Storage/StorageArchiveService.cs @@ -236,51 +236,52 @@ async Task DecompressAsyncWithSharpZipLib(string archiveFilePath, string d { long processedBytes = 0; int processedFiles = 0; - - foreach (ZipEntry zipEntry in zipFile) + await Task.Run(async () => { - if (statusCard.CancellationToken.IsCancellationRequested) + foreach (ZipEntry zipEntry in zipFile) { - isSuccess = false; - break; - } - - if (!zipEntry.IsFile) - { - continue; // Ignore directories - } + if (statusCard.CancellationToken.IsCancellationRequested) + { + isSuccess = false; + break; + } - string entryFileName = zipEntry.Name; - string fullZipToPath = Path.Combine(destinationFolderPath, entryFileName); - string directoryName = Path.GetDirectoryName(fullZipToPath); + if (!zipEntry.IsFile) + { + continue; // Ignore directories + } - if (!Directory.Exists(directoryName)) - { - Directory.CreateDirectory(directoryName); - } + string entryFileName = zipEntry.Name; + string fullZipToPath = Path.Combine(destinationFolderPath, entryFileName); + string directoryName = Path.GetDirectoryName(fullZipToPath); - byte[] buffer = new byte[4096]; // 4K is a good default - using (Stream zipStream = zipFile.GetInputStream(zipEntry)) - using (FileStream streamWriter = File.Create(fullZipToPath)) - { - await ThreadingService.ExecuteOnUiThreadAsync(() => + if (!Directory.Exists(directoryName)) { - fsProgress.FileName = entryFileName; - fsProgress.Report(); - }); + Directory.CreateDirectory(directoryName); + } - StreamUtils.Copy(zipStream, streamWriter, buffer); - } - processedBytes += zipEntry.Size; - if (fsProgress.TotalSize > 0) - { - fsProgress.Report(processedBytes / (double)fsProgress.TotalSize * 100); + byte[] buffer = new byte[4096]; // 4K is a good default + using (Stream zipStream = zipFile.GetInputStream(zipEntry)) + using (FileStream streamWriter = File.Create(fullZipToPath)) + { + await ThreadingService.ExecuteOnUiThreadAsync(() => + { + fsProgress.FileName = entryFileName; + fsProgress.Report(); + }); + + StreamUtils.Copy(zipStream, streamWriter, buffer); + } + processedBytes += zipEntry.Size; + if (fsProgress.TotalSize > 0) + { + fsProgress.Report(processedBytes / (double)fsProgress.TotalSize * 100); + } + processedFiles++; + fsProgress.AddProcessedItemsCount(1); + fsProgress.Report(); } - processedFiles++; - fsProgress.AddProcessedItemsCount(1); - fsProgress.Report(); - } - + }); if (!statusCard.CancellationToken.IsCancellationRequested) { isSuccess = true; From f58958bfc71380b634f7e5dbc25f3d8b8c238dd3 Mon Sep 17 00:00:00 2001 From: unknown <1463567152@qq.com> Date: Tue, 15 Apr 2025 21:08:37 +0800 Subject: [PATCH 3/8] don't need Vanara.PInvoke --- src/Files.App/Services/Storage/StorageArchiveService.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Files.App/Services/Storage/StorageArchiveService.cs b/src/Files.App/Services/Storage/StorageArchiveService.cs index dc84a44539a8..bf32429177b0 100644 --- a/src/Files.App/Services/Storage/StorageArchiveService.cs +++ b/src/Files.App/Services/Storage/StorageArchiveService.cs @@ -10,7 +10,6 @@ using System.Linq; using System.Text; using UtfUnknown; -using Vanara.PInvoke; using Windows.Storage; using Windows.Win32; From ef7ae92f3da09cd82114cbc988b1d5a90f378c20 Mon Sep 17 00:00:00 2001 From: unknown <1463567152@qq.com> Date: Fri, 18 Apr 2025 19:47:36 +0800 Subject: [PATCH 4/8] add all possible windows system encodings --- src/Files.App/Data/Items/EncodingItem.cs | 36 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/Files.App/Data/Items/EncodingItem.cs b/src/Files.App/Data/Items/EncodingItem.cs index 511cbd3578fb..24c79bc35bf2 100644 --- a/src/Files.App/Data/Items/EncodingItem.cs +++ b/src/Files.App/Data/Items/EncodingItem.cs @@ -22,7 +22,7 @@ public sealed class EncodingItem /// Initializes a new instance of the class. /// /// The code of the language. - public EncodingItem(string code) + public EncodingItem(string? code) { if (string.IsNullOrEmpty(code)) { @@ -44,13 +44,33 @@ public EncodingItem(Encoding encoding, string name) public static EncodingItem[] Defaults = new string?[] { null,//System Default - "UTF-8", - "shift_jis", - "gb2312", - "big5", - "ks_c_5601-1987", - "Windows-1252", - "macintosh", + "UTF-8", + + //All possible Windows system encodings + //reference: https://en.wikipedia.org/wiki/Windows_code_page + //East Asian + "shift_jis", //Japanese + "gb2312", //Simplified Chinese + "big5", //Traditional Chinese + "ks_c_5601-1987", //Korean + + //Southeast Asian + "Windows-1258", //Vietnamese + "Windows-874", //Thai + + //Middle East + "Windows-1256", //Arabic + "Windows-1255", //Hebrew + "Windows-1254", //Turkish + + //European + "Windows-1252", //Western European + "Windows-1250", //Central European + "Windows-1251", //Cyrillic + "Windows-1253", //Greek + "Windows-1257", //Baltic + + "macintosh", } .Select(x => new EncodingItem(x)) .ToArray(); From 1db65092ba55de641079d0866bb462fbc2def0b5 Mon Sep 17 00:00:00 2001 From: Yair <39923744+yaira2@users.noreply.github.com> Date: Tue, 22 Apr 2025 10:59:06 -0400 Subject: [PATCH 5/8] Fix spacing --- .../Services/Storage/StorageArchiveService.cs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Files.App/Services/Storage/StorageArchiveService.cs b/src/Files.App/Services/Storage/StorageArchiveService.cs index bf32429177b0..b1ef04a72a87 100644 --- a/src/Files.App/Services/Storage/StorageArchiveService.cs +++ b/src/Files.App/Services/Storage/StorageArchiveService.cs @@ -5,9 +5,7 @@ using ICSharpCode.SharpZipLib.Core; using ICSharpCode.SharpZipLib.Zip; using SevenZip; -using System.Collections; using System.IO; -using System.Linq; using System.Text; using UtfUnknown; using Windows.Storage; @@ -92,7 +90,8 @@ public async Task CompressAsync(ICompressArchiveModel compressionModel) /// public Task DecompressAsync(string archiveFilePath, string destinationFolderPath, string password = "", Encoding? encoding = null) { - if(encoding == null){ + if (encoding == null) + { return DecompressAsyncWithSevenZip(archiveFilePath, destinationFolderPath, password); } else @@ -205,10 +204,10 @@ async Task DecompressAsyncWithSharpZipLib(string archiveFilePath, string d string.IsNullOrEmpty(destinationFolderPath)) return false; using var zipFile = new ZipFile(archiveFilePath, StringCodec.FromEncoding(encoding)); - if(zipFile is null) + if (zipFile is null) return false; - - if(!string.IsNullOrEmpty(password)) + + if (!string.IsNullOrEmpty(password)) zipFile.Password = password; // Initialize a new in-progress status card @@ -216,11 +215,11 @@ async Task DecompressAsyncWithSharpZipLib(string archiveFilePath, string d archiveFilePath.CreateEnumerable(), destinationFolderPath.CreateEnumerable(), ReturnResult.InProgress); - + // Check if the decompress operation canceled if (statusCard.CancellationToken.IsCancellationRequested) return false; - + StatusCenterItemProgressModel fsProgress = new( statusCard.ProgressEventSource, enumerationCompleted: true, @@ -324,7 +323,7 @@ await ThreadingService.ExecuteOnUiThreadAsync(() => return isSuccess; } - + /// public string GenerateArchiveNameFromItems(IReadOnlyList items) { @@ -358,7 +357,7 @@ public async Task IsEncodingUndeterminedAsync(string archiveFilePath) { using (ZipFile zipFile = new ZipFile(archiveFilePath)) { - return !zipFile.Cast().All(entry=>entry.IsUnicodeText); + return !zipFile.Cast().All(entry => entry.IsUnicodeText); } } catch (Exception ex) @@ -380,7 +379,7 @@ public async Task IsEncodingUndeterminedAsync(string archiveFilePath) using (ZipFile zipFile = new ZipFile(archiveFilePath, StringCodec.FromEncoding(cp437))) { var fileNameBytes = cp437.GetBytes( - String.Join("\n", + String.Join("\n", zipFile.Cast() .Where(e => !e.IsUnicodeText) .Select(e => e.Name) From ca5b16947b5df8573b1f9e0af0656e7b3005c49f Mon Sep 17 00:00:00 2001 From: oxygen-dioxide <54425948+oxygen-dioxide@users.noreply.github.com> Date: Tue, 22 Apr 2025 23:10:29 +0800 Subject: [PATCH 6/8] Update src/Files.App/Strings/en-US/Resources.resw Co-authored-by: Yair <39923744+yaira2@users.noreply.github.com> Signed-off-by: oxygen-dioxide <54425948+oxygen-dioxide@users.noreply.github.com> --- src/Files.App/Strings/en-US/Resources.resw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Files.App/Strings/en-US/Resources.resw b/src/Files.App/Strings/en-US/Resources.resw index dd7e302ef424..741f191eb73b 100644 --- a/src/Files.App/Strings/en-US/Resources.resw +++ b/src/Files.App/Strings/en-US/Resources.resw @@ -2100,7 +2100,7 @@ Encoding - (detected) + {0} (detected) Path From 3e1b820cea4d224253dfd3761a5598dd5239041d Mon Sep 17 00:00:00 2001 From: oxygen-dioxide <54425948+oxygen-dioxide@users.noreply.github.com> Date: Tue, 22 Apr 2025 23:10:38 +0800 Subject: [PATCH 7/8] Update src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs Co-authored-by: Yair <39923744+yaira2@users.noreply.github.com> Signed-off-by: oxygen-dioxide <54425948+oxygen-dioxide@users.noreply.github.com> --- .../ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs b/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs index f96ba27bac86..fc84507648b5 100644 --- a/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs +++ b/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs @@ -72,7 +72,7 @@ void RefreshEncodingOptions() EncodingOptions = EncodingItem.Defaults .Prepend(new EncodingItem( detectedEncoding, - detectedEncoding.EncodingName + Strings.EncodingDetected.GetLocalizedResource()) + string.Format(Strings.EncodingDetected.GetLocalizedResource(), detectedEncoding.EncodingName) ) .ToArray(); } From 3a6b34b71293d0a20e56a373c8a6db12aa46228d Mon Sep 17 00:00:00 2001 From: oxygen-dioxide <54425948+oxygen-dioxide@users.noreply.github.com> Date: Tue, 22 Apr 2025 23:24:17 +0800 Subject: [PATCH 8/8] Update DecompressArchiveDialogViewModel.cs --- .../ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs b/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs index fc84507648b5..63ea6b6ca442 100644 --- a/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs +++ b/src/Files.App/ViewModels/Dialogs/DecompressArchiveDialogViewModel.cs @@ -73,7 +73,7 @@ void RefreshEncodingOptions() .Prepend(new EncodingItem( detectedEncoding, string.Format(Strings.EncodingDetected.GetLocalizedResource(), detectedEncoding.EncodingName) - ) + )) .ToArray(); } else