From c1856881073779155ca7c7b92d4a52a8b8744b0f Mon Sep 17 00:00:00 2001 From: VladiStep Date: Wed, 1 Feb 2023 01:15:43 +0300 Subject: [PATCH] Initial changes for unserializing the total count of all child objects for the object pool dictionaries optimization. --- .../Models/UndertaleGeneralInfo.cs | 15 +++ UndertaleModLib/UndertaleBaseTypes.cs | 16 +++ UndertaleModLib/UndertaleChunkTypes.cs | 53 +++++++- UndertaleModLib/UndertaleChunks.cs | 52 ++++++++ UndertaleModLib/UndertaleDebugChunks.cs | 5 + UndertaleModLib/UndertaleIO.cs | 124 +++++++++++++++++- UndertaleModLib/UndertaleLists.cs | 113 +++++++++++++++- 7 files changed, 370 insertions(+), 8 deletions(-) diff --git a/UndertaleModLib/Models/UndertaleGeneralInfo.cs b/UndertaleModLib/Models/UndertaleGeneralInfo.cs index ba69818c1..56ac21b86 100644 --- a/UndertaleModLib/Models/UndertaleGeneralInfo.cs +++ b/UndertaleModLib/Models/UndertaleGeneralInfo.cs @@ -451,6 +451,21 @@ public void Unserialize(UndertaleReader reader) reader.Bytecode14OrLower = BytecodeVersion <= 14; } + /// + /// Deserializes the total child object count of this object from specified . + /// + /// Where to deserialize from. + /// The object count. + public static uint UnserializeChildObjectCount(UndertaleReader reader) + { + reader.Position++; // "IsDebuggerDisabled" + bool readDebugPort = reader.ReadByte() >= 14; // "BytecodeVersion" >= 14 + reader.Position += (uint)(190 + (readDebugPort ? 8 : 0)); + + // "RoomOrder" + return reader.GetChildObjectCount>(); + } + /// /// Generates "info number" used for GMS2 UIDs. /// diff --git a/UndertaleModLib/UndertaleBaseTypes.cs b/UndertaleModLib/UndertaleBaseTypes.cs index efb4828b4..6e8aa64c3 100644 --- a/UndertaleModLib/UndertaleBaseTypes.cs +++ b/UndertaleModLib/UndertaleBaseTypes.cs @@ -20,6 +20,17 @@ public interface UndertaleObject /// /// Where to deserialize from. void Unserialize(UndertaleReader reader); + + /* + * As for C# 10, it's impossible to inherit static methods from an interface :( + * + /// + /// Deserializes the total child object count of this object from specified . + /// + /// Where to deserialize from. + /// The object count. + static uint UnserializeChildObjectCount(UndertaleReader reader) => 0; + */ } public interface UndertaleObjectLenCheck : UndertaleObject @@ -107,4 +118,9 @@ public interface ISearchable /// is the empty string (""); otherwise, false. bool SearchMatches(string filter); } + + public interface IStaticChildObjCount + { + public static readonly uint ChildObjectCount = 0; + } } diff --git a/UndertaleModLib/UndertaleChunkTypes.cs b/UndertaleModLib/UndertaleChunkTypes.cs index 194203fad..3d87cdefe 100644 --- a/UndertaleModLib/UndertaleChunkTypes.cs +++ b/UndertaleModLib/UndertaleChunkTypes.cs @@ -17,6 +17,7 @@ public abstract class UndertaleChunk internal abstract void SerializeChunk(UndertaleWriter writer); internal abstract void UnserializeChunk(UndertaleReader reader); + internal abstract uint UnserializeObjectCount(UndertaleReader reader); public void Serialize(UndertaleWriter writer) { @@ -79,7 +80,8 @@ public static UndertaleChunk Unserialize(UndertaleReader reader) } UndertaleChunk chunk = (UndertaleChunk)Activator.CreateInstance(type); - Util.DebugUtil.Assert(chunk.Name == name); + Util.DebugUtil.Assert(chunk.Name == name, + $"Chunk name mismatch: expected \"{name}\", got \"{chunk.Name}\"."); chunk.Length = length; reader.SubmitMessage("Reading chunk " + chunk.Name); @@ -124,6 +126,41 @@ public static UndertaleChunk Unserialize(UndertaleReader reader) throw new UndertaleSerializationException(e.Message + "\nat " + reader.Position.ToString("X8") + " while reading chunk " + name, e); } } + public static uint CountChunkChildObjects(UndertaleReader reader) + { + string name = "(unknown)"; + try + { + name = reader.ReadChars(4); + uint length = reader.ReadUInt32(); + + Type type = Type.GetType(typeof(UndertaleChunk).FullName + name); + if (type == null) + throw new IOException("Unknown chunk " + name + "!!!"); + + UndertaleChunk chunk = (UndertaleChunk)Activator.CreateInstance(type); + Util.DebugUtil.Assert(chunk.Name == name, + $"Chunk name mismatch: expected \"{name}\", got \"{chunk.Name}\"."); + chunk.Length = length; + + uint chunkStart = reader.Position; + + reader.SubmitMessage("Counting objects of chunk " + chunk.Name); + uint count = chunk.UnserializeObjectCount(reader); + + reader.Position = chunkStart + chunk.Length; + + return count; + } + catch (UndertaleSerializationException e) + { + throw new UndertaleSerializationException(e.Message + " in chunk " + name, e); + } + catch (Exception e) + { + throw new UndertaleSerializationException(e.Message + "\nat " + reader.Position.ToString("X8") + " while counting objects of chunk " + name, e); + } + } } public interface IUndertaleSingleChunk @@ -156,6 +193,8 @@ internal override void UnserializeChunk(UndertaleReader reader) Object = reader.ReadUndertaleObject(); } + internal override uint UnserializeObjectCount(UndertaleReader reader) => 1; + public UndertaleObject GetObject() => Object; public override string ToString() @@ -179,6 +218,11 @@ internal override void UnserializeChunk(UndertaleReader reader) List.Unserialize(reader); } + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + return reader.ReadUInt32(); + } + public IList GetList() => List; public void GenerateIndexDict() { @@ -256,6 +300,11 @@ internal override void UnserializeChunk(UndertaleReader reader) List.Unserialize(reader); } + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + return reader.ReadUInt32(); + } + public IList GetList() => List; } @@ -268,5 +317,7 @@ internal override void SerializeChunk(UndertaleWriter writer) internal override void UnserializeChunk(UndertaleReader reader) { } + + internal override uint UnserializeObjectCount(UndertaleReader reader) => 0; } } diff --git a/UndertaleModLib/UndertaleChunks.cs b/UndertaleModLib/UndertaleChunks.cs index 1e6c39019..af116b8dd 100644 --- a/UndertaleModLib/UndertaleChunks.cs +++ b/UndertaleModLib/UndertaleChunks.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using System.Text; using System.Threading.Tasks; using UndertaleModLib.Models; @@ -96,16 +97,45 @@ internal override void UnserializeChunk(UndertaleReader reader) } } } + + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + uint totalCount = 0; + uint startPos = reader.Position; + + while (reader.Position < startPos + Length) + totalCount += reader.CountChunkChildObjects(); + + return totalCount; + } } public class UndertaleChunkGEN8 : UndertaleSingleChunk { public override string Name => "GEN8"; + + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + uint count = 1; + + count += UndertaleGeneralInfo.UnserializeChildObjectCount(reader); + + return count; + } } public class UndertaleChunkOPTN : UndertaleSingleChunk { public override string Name => "OPTN"; + + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + uint count = 1; + + count += reader.GetChildObjectCount(); + + return count; + } } public class UndertaleChunkLANG : UndertaleSingleChunk @@ -655,6 +685,12 @@ internal override void UnserializeChunk(UndertaleReader reader) while (reader.Position + varLength <= startPosition + Length) List.Add(reader.ReadUndertaleObject()); } + + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + int varLength = reader.Bytecode14OrLower ? 12 : 20; + return Length / (uint)varLength; + } } public class UndertaleChunkFUNC : UndertaleChunk @@ -698,6 +734,7 @@ internal override void UnserializeChunk(UndertaleReader reader) { uint startPosition = reader.Position; Functions.Clear(); + Functions.SetCapacity(Length / 12); while (reader.Position + 12 <= startPosition + Length) Functions.Add(reader.ReadUndertaleObject()); } @@ -707,6 +744,21 @@ internal override void UnserializeChunk(UndertaleReader reader) CodeLocals = reader.ReadUndertaleObject>(); } } + + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + uint count = 0; + + if (!reader.Bytecode14OrLower) + { + count += reader.GetChildObjectCount>(); + count += reader.GetChildObjectCount>(); + } + else + count = Length / 12; + + return count; + } } public class UndertaleChunkSTRG : UndertaleAlignUpdatedListChunk diff --git a/UndertaleModLib/UndertaleDebugChunks.cs b/UndertaleModLib/UndertaleDebugChunks.cs index 42aca4fd9..ff353ccd4 100644 --- a/UndertaleModLib/UndertaleDebugChunks.cs +++ b/UndertaleModLib/UndertaleDebugChunks.cs @@ -44,6 +44,11 @@ internal override void UnserializeChunk(UndertaleReader reader) } } } + + internal override uint UnserializeObjectCount(UndertaleReader reader) + { + throw new NotImplementedException(); + } } public class UndertaleDebugChunkSCPT : UndertaleListChunk diff --git a/UndertaleModLib/UndertaleIO.cs b/UndertaleModLib/UndertaleIO.cs index 7b50a593d..4052119be 100644 --- a/UndertaleModLib/UndertaleIO.cs +++ b/UndertaleModLib/UndertaleIO.cs @@ -4,6 +4,7 @@ using System.IO; using System.IO.Ports; using System.Linq; +using System.Reflection; using System.Text; using System.Threading.Tasks; using UndertaleModLib.Compiler; @@ -177,6 +178,8 @@ public UndertaleReader(Stream input, FilePath = fs.Name; Directory = Path.GetDirectoryName(FilePath); } + + FillUnserializeCountDictionaries(); } // TODO: This would be more useful if it reported location like the exceptions did @@ -205,6 +208,10 @@ public UndertaleChunk ReadUndertaleChunk() { return UndertaleChunk.Unserialize(this); } + public uint CountChunkChildObjects() + { + return UndertaleChunk.CountChunkChildObjects(this); + } private List resUpdate = new List(); internal UndertaleData undertaleData; @@ -224,6 +231,21 @@ public UndertaleData ReadUndertaleData() DebugUtil.Assert(data.FORM.Name == name); data.FORM.Length = length; + uint startPos = Position; + uint poolSize = 0; + try + { + poolSize = data.FORM.UnserializeObjectCount(this); + } + catch (Exception e) + { + Debug.WriteLine(e); + } + + InitializePools(poolSize); + + Position = startPos; + var lenReader = EnsureLengthFromHere(data.FORM.Length); data.FORM.UnserializeChunk(this); lenReader.ToHere(); @@ -255,10 +277,83 @@ public override bool ReadBoolean() throw new IOException("Invalid boolean value: " + a); } - private Dictionary objectPool = new Dictionary(); - private Dictionary objectPoolRev = new Dictionary(); + private Dictionary objectPool; + private Dictionary objectPoolRev; private HashSet unreadObjects = new HashSet(); + private readonly Dictionary> unserializeFuncDict = new(); + private readonly Dictionary staticObjCountDict = new(); + + private void FillUnserializeCountDictionaries() + { + Assembly currAssem = Assembly.GetExecutingAssembly(); + Type[] allTypes = currAssem.GetExportedTypes(); + + Type utObjectType = typeof(UndertaleObject); + BindingFlags publicStatic = BindingFlags.Public | BindingFlags.Static; + Type delegateType = typeof(Func); + Type staticObjCountType = typeof(IStaticChildObjCount); + + allTypes = allTypes.Where(t => t.IsAssignableTo(utObjectType)).ToArray(); + foreach (Type t in allTypes) + { + MethodInfo mi = t.GetMethod("UnserializeChildObjectCount", publicStatic); + if (mi is null) + continue; + + var func = Delegate.CreateDelegate(delegateType, mi) as Func; + if (func is null) + { + Debug.WriteLine($"Can't create a delegate from MethodInfo of type \"{t.FullName}\""); + continue; + } + + unserializeFuncDict[t] = func; + } + + foreach (Type t in allTypes) + { + if (t.IsAssignableTo(staticObjCountType)) + { + FieldInfo fi = t.GetField("ChildObjectCount", publicStatic); + if (fi is null) + { + Debug.WriteLine($"Can't get \"ChildObjectCount\" field of \"{t.FullName}\""); + continue; + } + + object res = fi.GetValue(null); + if (res is null) + { + Debug.WriteLine($"Can't get value of \"ChildObjectCount\" of \"{t.FullName}\""); + continue; + } + + staticObjCountDict[t] = (uint)res; + } + } + } + public Func GetUnserializeCountFunc(Type objType) + { + if (!unserializeFuncDict.TryGetValue(objType, out var res)) + { + Debug.WriteLine($"\"UndertaleReader.unserializeFuncDict\" doesn't contain a method for \"{objType.FullName}\"."); + return null; + } + + return res; + } + public uint GetStaticChildCount(Type objType) + { + if (!staticObjCountDict.TryGetValue(objType, out uint res)) + { + Debug.WriteLine($"\"UndertaleReader.staticObjCountDict\" doesn't contain type \"{objType.FullName}\"."); + return 0; + } + + return res; + } + public Dictionary GetOffsetMap() { return objectPool; @@ -269,6 +364,31 @@ public Dictionary GetOffsetMapRev() return objectPoolRev; } + public void InitializePools(uint objCount) + { + if (objCount == 0) + { + objectPool = new(); + objectPoolRev = new(); + } + else + { + int objCountInt = (int)objCount; + objectPool = new(objCountInt); + objectPoolRev = new(objCountInt); + } + } + + public uint GetChildObjectCount() where T : UndertaleObject + { + Type t = typeof(T); + if (!unserializeFuncDict.TryGetValue(t, out var func)) + throw new UndertaleSerializationException( + $"\"UndertaleReader.unserializeFuncDict\" doesn't contain a method for \"{t.FullName}\"."); + + return func(this); + } + public T GetUndertaleObjectAtAddress(uint address) where T : UndertaleObject, new() { if (address == 0) diff --git a/UndertaleModLib/UndertaleLists.cs b/UndertaleModLib/UndertaleLists.cs index 60a25f082..01aad56eb 100644 --- a/UndertaleModLib/UndertaleLists.cs +++ b/UndertaleModLib/UndertaleLists.cs @@ -8,7 +8,7 @@ namespace UndertaleModLib { - public abstract class UndertaleListBase : ObservableCollection + public abstract class UndertaleListBase : ObservableCollection, UndertaleObject { private readonly List internalList; @@ -33,6 +33,13 @@ public UndertaleListBase() /// public abstract void Unserialize(UndertaleReader reader); + /// + /// Deserializes the total child object count of this list from specified . + /// + /// Where to deserialize from. + /// The object count. + public abstract uint UnserializeChildObjectCount(UndertaleReader reader); + public void SetCapacity(int capacity) { try @@ -52,7 +59,7 @@ public void AddDirect(T item) } } - public class UndertaleSimpleList : UndertaleListBase, UndertaleObject where T : UndertaleObject, new() + public class UndertaleSimpleList : UndertaleListBase where T : UndertaleObject, new() { /// public override void Serialize(UndertaleWriter writer) @@ -89,9 +96,41 @@ public override void Unserialize(UndertaleReader reader) } } } + + /// + public override uint UnserializeChildObjectCount(UndertaleReader reader) + { + uint count = reader.ReadUInt32(); + + Type t = typeof(T); + if (t.IsAssignableTo(typeof(UndertaleResourceRef))) + return count; + + if (t.IsAssignableTo(typeof(IStaticChildObjCount))) + { + uint subCount = reader.GetStaticChildCount(t); + + return count * subCount; + } + + var unserializeFunc = reader.GetUnserializeCountFunc(t); + for (uint i = 0; i < count; i++) + { + try + { + count += unserializeFunc(reader); + } + catch (UndertaleSerializationException e) + { + throw new UndertaleSerializationException(e.Message + "\nwhile reading child object count of item " + (i + 1) + " of " + count + " in a list of " + typeof(T).FullName, e); + } + } + + return count; + } } - public class UndertaleSimpleListString : UndertaleListBase, UndertaleObject + public class UndertaleSimpleListString : UndertaleListBase { /// public override void Serialize(UndertaleWriter writer) @@ -128,9 +167,15 @@ public override void Unserialize(UndertaleReader reader) } } } + + /// + public override uint UnserializeChildObjectCount(UndertaleReader reader) + { + return reader.ReadUInt32(); + } } - public class UndertaleSimpleListShort : UndertaleListBase, UndertaleObject where T : UndertaleObject, new() + public class UndertaleSimpleListShort : UndertaleListBase where T : UndertaleObject, new() { private void EnsureShortCount() { @@ -174,9 +219,38 @@ public override void Unserialize(UndertaleReader reader) } } } + + /// + public override uint UnserializeChildObjectCount(UndertaleReader reader) + { + uint count = reader.ReadUInt32(); + + Type t = typeof(T); + if (t.IsAssignableTo(typeof(IStaticChildObjCount))) + { + uint subCount = reader.GetStaticChildCount(t); + + return count * subCount; + } + + var unserializeFunc = reader.GetUnserializeCountFunc(t); + for (uint i = 0; i < count; i++) + { + try + { + count += unserializeFunc(reader); + } + catch (UndertaleSerializationException e) + { + throw new UndertaleSerializationException(e.Message + "\nwhile reading child object count of item " + (i + 1) + " of " + count + " in a list of " + typeof(T).FullName, e); + } + } + + return count; + } } - public class UndertalePointerList : UndertaleListBase, UndertaleObject where T : UndertaleObject, new() + public class UndertalePointerList : UndertaleListBase where T : UndertaleObject, new() { /// public override void Serialize(UndertaleWriter writer) @@ -267,6 +341,35 @@ public override void Unserialize(UndertaleReader reader) } } } + + /// + public override uint UnserializeChildObjectCount(UndertaleReader reader) + { + uint count = reader.ReadUInt32(); + + Type t = typeof(T); + if (t.IsAssignableTo(typeof(IStaticChildObjCount))) + { + uint subCount = reader.GetStaticChildCount(t); + + return count * subCount; + } + + var unserializeFunc = reader.GetUnserializeCountFunc(t); + for (uint i = 0; i < count; i++) + { + try + { + count += unserializeFunc(reader); + } + catch (UndertaleSerializationException e) + { + throw new UndertaleSerializationException(e.Message + "\nwhile reading child object count of item " + (i + 1) + " of " + count + " in a list of " + typeof(T).FullName, e); + } + } + + return count; + } } public class UndertalePointerListLenCheck : UndertalePointerList, UndertaleObjectEndPos where T : UndertaleObjectLenCheck, new()