feat: Updates serialization to use MMF (considerable memory savings) (#…

…1841)

### Summary
Updates the serialization strategy to use `MemoryMappedFile` instead of thick buffers. This has the benefit of being on-par with the current implementation (based on hardware/OS), however won't incur the double-memory issue.

> [!Important]
> **Developer Note**
> The `BinaryFileWriter` and `BinaryFileReader` has been removed in favor of `MemoryMapFileWriter` and `UnmanagedDataReader`

.config/dotnet-tools.json

@@ -3,7 +3,7 @@
  "isRoot": true,
  "tools": {
  "modernuoschemagenerator": {
- "version": "2.10.9",
+ "version": "2.11.3",
  "commands": [
  "ModernUOSchemaGenerator"
  ]

Projects/Server.Tests/Tests/Network/Packets/Outgoing/AccountPacketTests.cs

@@ -48,8 +48,6 @@ public Mobile this[int index]
  }
 
  public DateTime Created { get; set; }
- public long SavePosition { get; set; }
- public BufferWriter SaveBuffer { get; set; }
  public Serial Serial { get; }
  public void Deserialize(IGenericReader reader) => throw new NotImplementedException();
 

Projects/Server/Guild.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO *
- * Copyright 2019-2023 - ModernUO Development Team *
+ * Copyright 2019-2024 - ModernUO Development Team *
  * Email: hi@modernuo.com *
  * File: Guild.cs *
  * *
@@ -52,12 +52,6 @@ protected BaseGuild()
  [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
  public DateTime Created { get; set; } = Core.Now;
 
- [IgnoreDupe]
- public long SavePosition { get; set; } = -1;
-
- [IgnoreDupe]
- public BufferWriter SaveBuffer { get; set; }
-
  public abstract void Serialize(IGenericWriter writer);
 
  public abstract void Deserialize(IGenericReader reader);

Projects/Server/IEntity.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO *
- * Copyright 2019-2023 - ModernUO Development Team *
+ * Copyright 2019-2024 - ModernUO Development Team *
  * Email: hi@modernuo.com *
  * File: IEntity.cs *
  * *
@@ -51,10 +51,6 @@ public Entity(Serial serial, Point3D loc, Map map) : this(serial)
 
  public DateTime Created { get; set; } = Core.Now;
 
- public long SavePosition { get; set; } = -1;
-
- public BufferWriter SaveBuffer { get; set; }
-
  public Serial Serial { get; }
 
  public Point3D Location { get; private set; }

Projects/Server/Items/Item.cs

@@ -1,3 +1,18 @@
+/*************************************************************************
+ * ModernUO *
+ * Copyright 2019-2024 - ModernUO Development Team *
+ * Email: hi@modernuo.com *
+ * File: Item.cs *
+ * *
+ * This program is free software: you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation, either version 3 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
 using System;
 using System.Collections.Generic;
 using System.Reflection;
@@ -783,12 +798,6 @@ public virtual void GetProperties(IPropertyList list)
  [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
  public DateTime Created { get; set; } = Core.Now;
 
- [IgnoreDupe]
- public long SavePosition { get; set; } = -1;
-
- [IgnoreDupe]
- public BufferWriter SaveBuffer { get; set; }
-
  [IgnoreDupe]
  [CommandProperty(AccessLevel.Counselor)]
  public Serial Serial { get; }

Projects/Server/Main.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO *
- * Copyright 2019-2023 - ModernUO Development Team *
+ * Copyright 2019-2024 - ModernUO Development Team *
  * Email: hi@modernuo.com *
  * File: Main.cs *
  * *
@@ -40,6 +40,7 @@ public static class Core
  private static bool _performProcessKill;
  private static bool _restartOnKill;
  private static bool _performSnapshot;
+ private static string _snapshotPath;
  private static bool _crashed;
  private static string _baseDirectory;
 
@@ -411,7 +412,6 @@ private static void HandleClosed()
  logger.Information("Shutting down");
 
  World.WaitForWriteCompletion();
-
  World.ExitSerializationThreads();
 
  if (!_crashed)
@@ -553,12 +553,13 @@ public static void RunEventLoop()
  if (_performSnapshot)
  {
  // Return value is the offset that can be used to fix timers that should drift
- World.Snapshot();
+ World.Snapshot(_snapshotPath);
  _performSnapshot = false;
  }
 
  if (_performProcessKill)
  {
+ World.WaitForWriteCompletion();
  break;
  }
 
@@ -594,7 +595,11 @@ public static void RunEventLoop()
  DoKill(_restartOnKill);
  }
 
- internal static void RequestSnapshot() => _performSnapshot = true;
+ internal static void RequestSnapshot(string snapshotPath)
+ {
+ _performSnapshot = true;
+ _snapshotPath = snapshotPath;
+ }
 
  public static void VerifySerialization()
  {

Projects/Server/Mobiles/Mobile.cs

@@ -1,3 +1,18 @@
+/*************************************************************************
+ * ModernUO *
+ * Copyright 2019-2024 - ModernUO Development Team *
+ * Email: hi@modernuo.com *
+ * File: Mobile.cs *
+ * *
+ * This program is free software: you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation, either version 3 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>. *
+ *************************************************************************/
+
 using System;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
@@ -2262,12 +2277,6 @@ public virtual void GetProperties(IPropertyList list)
  [CommandProperty(AccessLevel.GameMaster, readOnly: true)]
  public DateTime Created { get; set; } = Core.Now;
 
- [IgnoreDupe]
- public long SavePosition { get; set; } = -1;
-
- [IgnoreDupe]
- public BufferWriter SaveBuffer { get; set; }
-
  [IgnoreDupe]
  [CommandProperty(AccessLevel.Counselor)]
  public Serial Serial { get; }

Projects/Server/Serialization/AdhocPersistence.cs

@@ -1,6 +1,6 @@
 /*************************************************************************
  * ModernUO *
- * Copyright 2019-2023 - ModernUO Development Team *
+ * Copyright 2019-2024 - ModernUO Development Team *
  * Email: hi@modernuo.com *
  * File: AdhocPersistence.cs *
  * *
@@ -25,87 +25,91 @@ namespace Server;
 public static class AdhocPersistence
 {
  /**
- * Serializes to memory synchronously. Optional buffer can be provided.
- * Note: The buffer may not be the same after returning from the function if more data is written
- * than the initial buffer can handle.
+ * Serializes to memory.
  */
- public static BufferWriter Serialize(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types)
+ public static IGenericWriter SerializeToBuffer(Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
  {
  var saveBuffer = new BufferWriter(true, types);
  serializer(saveBuffer);
  return saveBuffer;
  }
 
  /**
- * Writes a buffer to disk. This function should be called asynchronously.
- * Writes the filePath for the binary data, and an accompanying SerializedTypes.db file of all possible types.
+ * Deserializes from a buffer.
  */
- public static void WriteSnapshot(FileInfo file, Span<byte> buffer)
+ public static IGenericReader DeserializeFromBuffer(
+ byte[] buffer, Action<IGenericReader> deserializer, Dictionary<ulong, string> typesDb = null
+ )
  {
- var dirPath = file.DirectoryName;
- PathUtility.EnsureDirectory(dirPath);
-
- using var fs = new FileStream(file.FullName, FileMode.Create, FileAccess.Write);
- fs.Write(buffer);
+ var reader = new BufferReader(buffer, typesDb);
+ deserializer(reader);
+ return reader;
  }
 
  /**
- * Serializes to a memory buffer synchronously, then flushes to the path asynchronously.
- * See WriteSnapshot for more info about how to snapshot.
+ * Serializes to a Memory Mapped file synchronously, then flushes to the file asynchronously.
  */
- public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer, ConcurrentQueue<Type> types = null)
+ public static void SerializeAndSnapshot(
+ string filePath, Action<IGenericWriter> serializer, long sizeHint = 1024 * 1024 * 32
+ )
  {
- types ??= new ConcurrentQueue<Type>();
- var saveBuffer = Serialize(serializer, types);
+ var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
+ PathUtility.EnsureDirectory(Path.GetDirectoryName(fullPath));
+ ConcurrentQueue<Type> types = [];
+ var writer = new MemoryMapFileWriter(new FileStream(filePath, FileMode.Create), sizeHint, types);
+ serializer(writer);
+
  Task.Run(
  () =>
  {
- var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
- var file = new FileInfo(fullPath);
+ var fs = writer.FileStream;
 
- WriteSnapshot(file, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position));
+ writer.Dispose();
+ fs.Dispose();
 
- // TODO: Create a PooledHashSet if performance becomes an issue.
- var typesSet = new HashSet<Type>();
+ HashSet<Type> typesSet = [];
 
  // Dedupe the queue.
  foreach (var type in types)
  {
  typesSet.Add(type);
  }
 
- Persistence.WriteSerializedTypesSnapshot(file.DirectoryName, typesSet);
- });
+ Persistence.WriteSerializedTypesSnapshot(Path.GetDirectoryName(fullPath), typesSet);
+ },
+ Core.ClosingTokenSource.Token
+ );
  }
 
- public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
+ public static unsafe void Deserialize(string filePath, Action<IGenericReader> deserializer)
  {
  var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
  var file = new FileInfo(fullPath);
 
- if (!file.Exists)
+ if (!file.Exists || file.Length == 0)
  {
  return;
  }
 
  var fileLength = file.Length;
- if (fileLength == 0)
- {
- return;
- }
 
  string error;
 
  try
  {
  using var mmf = MemoryMappedFile.CreateFromFile(fullPath, FileMode.Open);
- using var stream = mmf.CreateViewStream();
- using var br = new BinaryFileReader(stream);
- deserializer(br);
+ using var accessor = mmf.CreateViewStream();
 
- error = br.Position != fileLength
- ? $"Serialized {fileLength} bytes, but {br.Position} bytes deserialized"
+ byte* ptr = null;
+ accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
+ UnmanagedDataReader dataReader = new UnmanagedDataReader(ptr, accessor.Length);
+ deserializer(dataReader);
+
+ error = dataReader.Position != fileLength
+ ? $"Serialized {fileLength} bytes, but {dataReader.Position} bytes deserialized"
  : null;
+
+ accessor.SafeMemoryMappedViewHandle.ReleasePointer();
  }
  catch (Exception e)
  {