From 6f54bab53af0c6598b6791f3371d615053675fe2 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk Date: Fri, 13 Aug 2021 13:49:07 +0200 Subject: [PATCH 1/4] Added thread pool for unpacking --- .../Scripts/AppData/Local/Pack1Unarchiver.cs | 96 +++++++++++++++++-- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs index 283e4e3c..5b060bf8 100755 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs @@ -2,6 +2,7 @@ using System.IO; using System.Security.Cryptography; using System.Text; +using System.Threading; using Ionic.Zlib; using PatchKit.Network; using PatchKit.Unity.Patcher.AppData.FileSystem; @@ -9,11 +10,10 @@ using PatchKit.Unity.Patcher.Data; using PatchKit.Unity.Patcher.Debug; using PatchKit.Unity.Utilities; -using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.Xz; using SharpRaven; using SharpRaven.Data; -using SharpRaven.Utilities; +using FileMode = System.IO.FileMode; namespace PatchKit.Unity.Patcher.AppData.Local { @@ -42,6 +42,7 @@ public class Pack1Unarchiver : IUnarchiver private readonly BytesRange _range; private MapHashExtractedFiles _mapHashExtractedFiles; + private volatile int _space = 0; public event UnarchiveProgressChangedHandler UnarchiveProgressChanged; @@ -113,7 +114,7 @@ public void Unarchive(CancellationToken cancellationToken) Unpack(file, progress => { OnUnarchiveProgressChanged(currentFile.Name, currentFile.Type == Pack1Meta.RegularFileType, currentEntry, _metaData.Files.Length, progress); - }, cancellationToken); + }, cancellationToken, false); } else { @@ -124,6 +125,19 @@ public void Unarchive(CancellationToken cancellationToken) entry++; } + while (true) + { + lock (this) + { + if (_space == 0) + { + break; + } + UnityEngine.Debug.Log(_space); + } + Thread.Sleep(1000); + } + DebugLogger.Log("Unpacking finished succesfully!"); } @@ -136,8 +150,19 @@ public void UnarchiveSingleFile(Pack1Meta.FileEntry file, CancellationToken canc throw new ArgumentOutOfRangeException("file", file, null); } - Unpack(file, progress => OnUnarchiveProgressChanged(file.Name, file.Type == Pack1Meta.RegularFileType, 1, 1, progress), cancellationToken, destinationDirPath); - + Unpack(file, progress => OnUnarchiveProgressChanged(file.Name, file.Type == Pack1Meta.RegularFileType, 1, 1, progress), cancellationToken, true, destinationDirPath); + while (true) + { + lock (this) + { + if (_space == 0) + { + break; + } + UnityEngine.Debug.Log(_space); + } + Thread.Sleep(1000); + } OnUnarchiveProgressChanged(file.Name, file.Type == Pack1Meta.RegularFileType, 0, 1, 1.0); } @@ -156,14 +181,41 @@ private bool CanUnpack(Pack1Meta.FileEntry file) return file.Offset >= _range.Start && file.Offset + file.Size <= _range.End; } - private void Unpack(Pack1Meta.FileEntry file, Action progress, CancellationToken cancellationToken, string destinationDirPath = null) + private void Unpack(Pack1Meta.FileEntry file, Action progress, CancellationToken cancellationToken, bool isSingle, string destinationDirPath = null) { switch (file.Type) { case Pack1Meta.RegularFileType: try { - UnpackRegularFile(file, progress, cancellationToken, destinationDirPath); + if (file.Size < 524288 && !isSingle) + { + while (true) + { + int tmp; + lock (this) + { + tmp = _space; + } + if (tmp < 16) + { + break; + } + Thread.Sleep(1); + } + + lock (this) + { + _space++; + } + + ThreadPool.QueueUserWorkItem(state => UnpackProc(file, progress, cancellationToken, + destinationDirPath)); + } + else + { + UnpackRegularFile(file, progress, cancellationToken, destinationDirPath); + } } catch (Ionic.Zlib.ZlibException e) { @@ -201,6 +253,27 @@ private void Unpack(Pack1Meta.FileEntry file, Action progress, Cancellat } + private void UnpackProc(Pack1Meta.FileEntry file, Action onProgress, CancellationToken cancellationToken, string destinationDirPath = null) + { + try + { + UnpackRegularFile(file, onProgress, cancellationToken, + destinationDirPath); + } + catch (Exception e) + { + UnityEngine.Debug.LogError(e); + throw; + } + finally + { + lock (this) + { + _space--; + } + } + } + private void UnpackDirectory(Pack1Meta.FileEntry file, CancellationToken cancellationToken) { string destPath = Path.Combine(_destinationDirPath, _mapHashExtractedFiles.Add(file.Name)); @@ -234,7 +307,12 @@ private DecompressorCreator ResolveDecompressor(Pack1Meta meta) private void UnpackRegularFile(Pack1Meta.FileEntry file, Action onProgress, CancellationToken cancellationToken, string destinationDirPath = null) { - string destPath = Path.Combine(destinationDirPath == null ? _destinationDirPath : destinationDirPath, _mapHashExtractedFiles.Add(file.Name) + _suffix); + string nameUnPackFile; + lock (_mapHashExtractedFiles) + { + nameUnPackFile = _mapHashExtractedFiles.Add(file.Name) + _suffix; + } + string destPath = Path.Combine(destinationDirPath == null ? _destinationDirPath : destinationDirPath, nameUnPackFile); DebugLogger.LogFormat("Unpacking regular file {0} to {1}", file, destPath); @@ -269,7 +347,7 @@ private void UnpackRegularFile(Pack1Meta.FileEntry file, Action onProgre DecompressorCreator decompressorCreator = ResolveDecompressor(_metaData); - using (var fs = new FileStream(_packagePath, FileMode.Open)) + using (var fs = new FileStream(_packagePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { fs.Seek(file.Offset.Value - _range.Start, SeekOrigin.Begin); From c8000fa5f65d76447d7ee0113885dd8e54b118cf Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk Date: Tue, 12 Oct 2021 15:11:19 +0200 Subject: [PATCH 2/4] Created ThredingPool and added Semaphore --- .../AppData/Local/MapHashExtractedFiles.cs | 17 ++-- .../Scripts/AppData/Local/Pack1Unarchiver.cs | 89 +++---------------- .../Scripts/Utilities/ThreadingPool.cs | 53 +++++++++++ .../Scripts/Utilities/ThreadingPool.cs.meta | 11 +++ 4 files changed, 88 insertions(+), 82 deletions(-) create mode 100644 Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs create mode 100644 Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs.meta diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/MapHashExtractedFiles.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/MapHashExtractedFiles.cs index 674bc33b..eb042c9d 100644 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/MapHashExtractedFiles.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/MapHashExtractedFiles.cs @@ -4,25 +4,32 @@ namespace PatchKit.Unity.Patcher.AppData.Local { public class MapHashExtractedFiles { - private Dictionary MapHash; + private volatile Dictionary _mapHash; public MapHashExtractedFiles() { - MapHash = new Dictionary(); + _mapHash = new Dictionary(); } public string Add(string path) { string nameHash = HashCalculator.ComputeMD5Hash(path); - MapHash.Add(path, nameHash); + lock (_mapHash) + { + _mapHash.Add(path, nameHash); + } + return nameHash; } public bool TryGetHash(string path,out string nameHash) { - if (MapHash.TryGetValue(path, out nameHash)) + lock (_mapHash) { - return true; + if (_mapHash.TryGetValue(path, out nameHash)) + { + return true; + } } return false; diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs index 5b060bf8..d9c65629 100755 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs @@ -41,8 +41,7 @@ public class Pack1Unarchiver : IUnarchiver /// private readonly BytesRange _range; - private MapHashExtractedFiles _mapHashExtractedFiles; - private volatile int _space = 0; + private volatile MapHashExtractedFiles _mapHashExtractedFiles; public event UnarchiveProgressChangedHandler UnarchiveProgressChanged; @@ -125,19 +124,8 @@ public void Unarchive(CancellationToken cancellationToken) entry++; } - while (true) - { - lock (this) - { - if (_space == 0) - { - break; - } - UnityEngine.Debug.Log(_space); - } - Thread.Sleep(1000); - } + ThreadingPool.WaitOne(cancellationToken); DebugLogger.Log("Unpacking finished succesfully!"); } @@ -151,18 +139,7 @@ public void UnarchiveSingleFile(Pack1Meta.FileEntry file, CancellationToken canc } Unpack(file, progress => OnUnarchiveProgressChanged(file.Name, file.Type == Pack1Meta.RegularFileType, 1, 1, progress), cancellationToken, true, destinationDirPath); - while (true) - { - lock (this) - { - if (_space == 0) - { - break; - } - UnityEngine.Debug.Log(_space); - } - Thread.Sleep(1000); - } + OnUnarchiveProgressChanged(file.Name, file.Type == Pack1Meta.RegularFileType, 0, 1, 1.0); } @@ -188,34 +165,17 @@ private void Unpack(Pack1Meta.FileEntry file, Action progress, Cancellat case Pack1Meta.RegularFileType: try { - if (file.Size < 524288 && !isSingle) + progress(0.0); + if (file.Size.Value < 524288 && !isSingle) { - while (true) - { - int tmp; - lock (this) - { - tmp = _space; - } - if (tmp < 16) - { - break; - } - Thread.Sleep(1); - } - - lock (this) - { - _space++; - } - - ThreadPool.QueueUserWorkItem(state => UnpackProc(file, progress, cancellationToken, - destinationDirPath)); + ThreadingPool.ThreadingPoolExecute(cancellationToken, + () => UnpackRegularFile(file, cancellationToken, destinationDirPath)); } else { - UnpackRegularFile(file, progress, cancellationToken, destinationDirPath); + UnpackRegularFile(file, cancellationToken, destinationDirPath); } + progress(1.0); } catch (Ionic.Zlib.ZlibException e) { @@ -253,27 +213,6 @@ private void Unpack(Pack1Meta.FileEntry file, Action progress, Cancellat } - private void UnpackProc(Pack1Meta.FileEntry file, Action onProgress, CancellationToken cancellationToken, string destinationDirPath = null) - { - try - { - UnpackRegularFile(file, onProgress, cancellationToken, - destinationDirPath); - } - catch (Exception e) - { - UnityEngine.Debug.LogError(e); - throw; - } - finally - { - lock (this) - { - _space--; - } - } - } - private void UnpackDirectory(Pack1Meta.FileEntry file, CancellationToken cancellationToken) { string destPath = Path.Combine(_destinationDirPath, _mapHashExtractedFiles.Add(file.Name)); @@ -305,13 +244,14 @@ private DecompressorCreator ResolveDecompressor(Pack1Meta meta) } } - private void UnpackRegularFile(Pack1Meta.FileEntry file, Action onProgress, CancellationToken cancellationToken, string destinationDirPath = null) + private void UnpackRegularFile(Pack1Meta.FileEntry file, CancellationToken cancellationToken, string destinationDirPath = null) { string nameUnPackFile; lock (_mapHashExtractedFiles) { nameUnPackFile = _mapHashExtractedFiles.Add(file.Name) + _suffix; } + string destPath = Path.Combine(destinationDirPath == null ? _destinationDirPath : destinationDirPath, nameUnPackFile); DebugLogger.LogFormat("Unpacking regular file {0} to {1}", file, destPath); @@ -357,7 +297,7 @@ private void UnpackRegularFile(Pack1Meta.FileEntry file, Action onProgre { using (var target = new FileStream(destPath, FileMode.Create)) { - ExtractFileFromStream(limitedStream, target, file.Size.Value, decryptor, decompressorCreator, onProgress, cancellationToken); + ExtractFileFromStream(limitedStream, target, decryptor, decompressorCreator, cancellationToken); DebugTestCorruption(target); } } @@ -416,10 +356,8 @@ private Stream CreateGzipDecompressor(Stream source) private void ExtractFileFromStream( Stream sourceStream, Stream targetStream, - long fileSize, ICryptoTransform decryptor, DecompressorCreator createDecompressor, - Action onProgress, CancellationToken cancellationToken) { using (var cryptoStream = new CryptoStream(sourceStream, decryptor, CryptoStreamMode.Read)) @@ -442,9 +380,6 @@ private void ExtractFileFromStream( { cancellationToken.ThrowIfCancellationRequested(); targetStream.Write(buffer, 0, count); - - long bytesProcessed = sourceStream.Position; - onProgress(bytesProcessed / (double) fileSize); } } catch (OperationCanceledException) diff --git a/Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs b/Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs new file mode 100644 index 00000000..6f185a6d --- /dev/null +++ b/Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs @@ -0,0 +1,53 @@ +using System; +using System.Threading; +using PatchKit.Unity.Patcher.Cancellation; +using PatchKit.Unity.Patcher.Debug; + +namespace PatchKit.Unity.Utilities +{ + public static class ThreadingPool + { + private static readonly DebugLogger DebugLogger = new DebugLogger(typeof(ThreadingPool)); + + private static Semaphore _pool = new Semaphore(16, 17); + private static ManualResetEvent _awaiter = new ManualResetEvent(false); + + public static void ThreadingPoolExecute(CancellationToken cancellationToken, Action action) + { + _pool.WaitOne(); + ThreadPool.QueueUserWorkItem(state => ThreadingPoolProc(cancellationToken, action)); + } + + private static void ThreadingPoolProc(CancellationToken cancellationToken, Action action) + { + try + { + action(); + } + catch (Exception e) + { + DebugLogger.LogError(e.ToString()); + throw; + } + finally + { + int release = _pool.Release(); + if (release == 16 || cancellationToken.IsCancelled) + { + _awaiter.Set(); + } + } + } + + public static void WaitOne(CancellationToken cancellationToken) + { + var release = _pool.Release(); + if (release == 16 || cancellationToken.IsCancelled) + { + _awaiter.Set(); + } + + _awaiter.WaitOne(); + } + } +} \ No newline at end of file diff --git a/Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs.meta b/Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs.meta new file mode 100644 index 00000000..ceea43d7 --- /dev/null +++ b/Assets/PatchKit Patcher/Scripts/Utilities/ThreadingPool.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 54f8f0c8ae07164488ddf2191d3a3977 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 8179907faef024bf0bd11c4a0890ceaf72108a1e Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk Date: Tue, 12 Oct 2021 15:14:49 +0200 Subject: [PATCH 3/4] Refactoring --- .../Scripts/AppData/Local/Pack1Unarchiver.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs index d9c65629..f7b14fbe 100755 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs @@ -158,7 +158,7 @@ private bool CanUnpack(Pack1Meta.FileEntry file) return file.Offset >= _range.Start && file.Offset + file.Size <= _range.End; } - private void Unpack(Pack1Meta.FileEntry file, Action progress, CancellationToken cancellationToken, bool isSingle, string destinationDirPath = null) + private void Unpack(Pack1Meta.FileEntry file, Action progress, CancellationToken cancellationToken, bool canUseThreadPool, string destinationDirPath = null) { switch (file.Type) { @@ -166,7 +166,7 @@ private void Unpack(Pack1Meta.FileEntry file, Action progress, Cancellat try { progress(0.0); - if (file.Size.Value < 524288 && !isSingle) + if (file.Size.Value < 524288 && !canUseThreadPool) { ThreadingPool.ThreadingPoolExecute(cancellationToken, () => UnpackRegularFile(file, cancellationToken, destinationDirPath)); @@ -246,13 +246,13 @@ private DecompressorCreator ResolveDecompressor(Pack1Meta meta) private void UnpackRegularFile(Pack1Meta.FileEntry file, CancellationToken cancellationToken, string destinationDirPath = null) { - string nameUnPackFile; + string fileRealName; lock (_mapHashExtractedFiles) { - nameUnPackFile = _mapHashExtractedFiles.Add(file.Name) + _suffix; + fileRealName = _mapHashExtractedFiles.Add(file.Name) + _suffix; } - string destPath = Path.Combine(destinationDirPath == null ? _destinationDirPath : destinationDirPath, nameUnPackFile); + string destPath = Path.Combine(destinationDirPath == null ? _destinationDirPath : destinationDirPath, fileRealName); DebugLogger.LogFormat("Unpacking regular file {0} to {1}", file, destPath); From 1cea67124562b9b68439710f0fe29172efa70f12 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk Date: Tue, 12 Oct 2021 15:35:31 +0200 Subject: [PATCH 4/4] Refactoring --- Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs index f7b14fbe..37ac54a1 100755 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/Pack1Unarchiver.cs @@ -2,7 +2,6 @@ using System.IO; using System.Security.Cryptography; using System.Text; -using System.Threading; using Ionic.Zlib; using PatchKit.Network; using PatchKit.Unity.Patcher.AppData.FileSystem;