Optimize Chunk Storage in DepotManifest Serialization

Replaced List<byte[]> with HashSet<byte[]> using a custom ChunkIdComparer for efficient chunk ID uniqueness checks
SteamRE · Feb 22, 2024 · c6dd216 · c6dd216
1 parent 08f6b97
commit c6dd216
Showing 1 changed file with 20 additions and 5 deletions.
diff --git a/SteamKit2/SteamKit2/Types/DepotManifest.cs b/SteamKit2/SteamKit2/Types/DepotManifest.cs
@@ -371,12 +371,30 @@ void ParseProtobufManifestMetadata(ContentManifestMetadata metadata)
             EncryptedCRC = metadata.crc_encrypted;
         }
 
+        class ChunkIdComparer : IEqualityComparer<byte[]>
+        {
+            public bool Equals( byte[]? x, byte[]? y )
+            {
+                if ( ReferenceEquals( x, y ) ) return true;
+                if ( x == null || y == null ) return false;
+                return x.SequenceEqual( y );
+            }
+
+            public int GetHashCode( byte[] obj )
+            {
+                ArgumentNullException.ThrowIfNull( obj );
+
+                // ChunkID is SHA-1, so we can just use the first 4 bytes
+                return BitConverter.ToInt32( obj, 0 );
+            }
+        }
+
         byte[]? Serialize()
         {
             DebugLog.Assert( Files != null, nameof( DepotManifest ), "Files was null when attempting to serialize manifest." );
 
             var payload = new ContentManifestPayload();
-            var uniqueChunks = new List<byte[]>();
+            var uniqueChunks = new HashSet<byte[]>( new ChunkIdComparer() );
 
             foreach ( var file in Files )
             {
@@ -409,10 +427,7 @@ void ParseProtobufManifestMetadata(ContentManifestMetadata metadata)
                     protochunk.cb_compressed = chunk.CompressedLength;
 
                     protofile.chunks.Add( protochunk );
-                    if ( !uniqueChunks.Exists( x => x.SequenceEqual( chunk.ChunkID! ) ) )
-                    {
-                        uniqueChunks.Add( chunk.ChunkID! );
-                    }
+                    uniqueChunks.Add( chunk.ChunkID! );
                 }
 
                 payload.mappings.Add( protofile );