diff --git a/neo.UnitTests/IO/UT_IOHelper.cs b/neo.UnitTests/IO/UT_IOHelper.cs index e113ce8cd4..b0593c02ec 100644 --- a/neo.UnitTests/IO/UT_IOHelper.cs +++ b/neo.UnitTests/IO/UT_IOHelper.cs @@ -234,9 +234,9 @@ public void TestReadBytesWithGrouping() else { byte[] caseArray = new byte[] { 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,0x11}; MemoryStream stream = new MemoryStream(); @@ -415,11 +415,11 @@ public void TestWriteBytesWithGrouping() byte[] byteArray = new byte[stream.Length]; stream.Read(byteArray, 0, (int)stream.Length); Assert.AreEqual(Encoding.Default.GetString(new byte[] { 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,0x0C}), Encoding.Default.GetString(byteArray)); + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,0x04}), Encoding.Default.GetString(byteArray)); } [TestMethod] @@ -546,4 +546,4 @@ public void TestWriteVarString() Assert.AreEqual(0x61, byteArray[1]); } } -} \ No newline at end of file +} diff --git a/neo.UnitTests/UT_DataCache.cs b/neo.UnitTests/UT_DataCache.cs new file mode 100644 index 0000000000..85df272db5 --- /dev/null +++ b/neo.UnitTests/UT_DataCache.cs @@ -0,0 +1,126 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO.Caching; +using Neo.Ledger; +using System.Linq; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_DataCache + { + [TestInitialize] + public void TestSetup() + { + TestBlockchain.InitializeMockNeoSystem(); + } + + [TestMethod] + public void TestCachedFind_Between() + { + var snapshot = TestBlockchain.GetStore().GetSnapshot(); + var storages = snapshot.Storages; + var cache = new CloneCache(storages); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x03 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + + CollectionAssert.AreEqual( + cache.Find(new byte[21]).Select(u => u.Key.Key[1]).ToArray(), + new byte[] { 0x01, 0x02, 0x03 } + ); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + } + + [TestMethod] + public void TestCachedFind_Last() + { + var snapshot = TestBlockchain.GetStore().GetSnapshot(); + var storages = snapshot.Storages; + var cache = new CloneCache(storages); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + + CollectionAssert.AreEqual( + cache.Find(new byte[21]).Select(u => u.Key.Key[1]).ToArray(), + new byte[] { 0x01, 0x02 } + ); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + } + + [TestMethod] + public void TestCachedFind_Empty() + { + var snapshot = TestBlockchain.GetStore().GetSnapshot(); + var storages = snapshot.Storages; + var cache = new CloneCache(storages); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + + CollectionAssert.AreEqual( + cache.Find(new byte[21]).Select(u => u.Key.Key[1]).ToArray(), + new byte[] { 0x02 } + ); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + } + } +} diff --git a/neo/IO/ByteArrayComparer.cs b/neo/IO/ByteArrayComparer.cs new file mode 100644 index 0000000000..956ab758e6 --- /dev/null +++ b/neo/IO/ByteArrayComparer.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; + +namespace Neo.IO +{ + internal class ByteArrayComparer : IComparer + { + public static readonly ByteArrayComparer Default = new ByteArrayComparer(); + + public int Compare(byte[] x, byte[] y) + { + int length = Math.Min(x.Length, y.Length); + for (int i = 0; i < length; i++) + { + int r = x[i].CompareTo(y[i]); + if (r != 0) return r; + } + return x.Length.CompareTo(y.Length); + } + } +} diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index 5e9886d10c..278c01d650 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -118,16 +118,58 @@ public void DeleteWhere(Func predicate) } } + /// + /// Find the entries that start with the `key_prefix` + /// + /// Must maintain the deserialized format of TKey + /// Entries found with the desired prefix public IEnumerable> Find(byte[] key_prefix = null) { + IEnumerable<(byte[], TKey, TValue)> cached; lock (dictionary) { - foreach (var pair in FindInternal(key_prefix ?? new byte[0])) - if (!dictionary.ContainsKey(pair.Key)) - yield return pair; - foreach (var pair in dictionary) - if (pair.Value.State != TrackState.Deleted && (key_prefix == null || pair.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix))) - yield return new KeyValuePair(pair.Key, pair.Value.Item); + cached = dictionary + .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix))) + .Select(p => + ( + KeyBytes: p.Key.ToArray(), + p.Key, + p.Value.Item + )) + .OrderBy(p => p.KeyBytes, ByteArrayComparer.Default) + .ToArray(); + } + var uncached = FindInternal(key_prefix ?? new byte[0]) + .Where(p => !dictionary.ContainsKey(p.Key)) + .Select(p => + ( + KeyBytes: p.Key.ToArray(), + p.Key, + p.Value + )); + using (var e1 = cached.GetEnumerator()) + using (var e2 = uncached.GetEnumerator()) + { + (byte[] KeyBytes, TKey Key, TValue Item) i1, i2; + bool c1 = e1.MoveNext(); + bool c2 = e2.MoveNext(); + i1 = c1 ? e1.Current : default; + i2 = c2 ? e2.Current : default; + while (c1 || c2) + { + if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) < 0)) + { + yield return new KeyValuePair(i1.Key, i1.Item); + c1 = e1.MoveNext(); + i1 = c1 ? e1.Current : default; + } + else + { + yield return new KeyValuePair(i2.Key, i2.Item); + c2 = e2.MoveNext(); + i2 = c2 ? e2.Current : default; + } + } } } diff --git a/neo/IO/Helper.cs b/neo/IO/Helper.cs index 647ae2cfb1..a306e297d5 100644 --- a/neo/IO/Helper.cs +++ b/neo/IO/Helper.cs @@ -92,17 +92,16 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) { using (MemoryStream ms = new MemoryStream()) { - int padding = 0; + int count; do { byte[] group = reader.ReadBytes(GroupingSizeInBytes); - padding = reader.ReadByte(); - if (padding > GroupingSizeInBytes) + count = reader.ReadByte(); + if (count > GroupingSizeInBytes) throw new FormatException(); - int count = GroupingSizeInBytes - padding; if (count > 0) ms.Write(group, 0, count); - } while (padding == 0); + } while (count == GroupingSizeInBytes); return ms.ToArray(); } } @@ -200,7 +199,7 @@ public static void WriteBytesWithGrouping(this BinaryWriter writer, byte[] value while (remain >= GroupingSizeInBytes) { writer.Write(value, index, GroupingSizeInBytes); - writer.Write((byte)0); + writer.Write((byte)GroupingSizeInBytes); index += GroupingSizeInBytes; remain -= GroupingSizeInBytes; } @@ -209,7 +208,7 @@ public static void WriteBytesWithGrouping(this BinaryWriter writer, byte[] value int padding = GroupingSizeInBytes - remain; for (int i = 0; i < padding; i++) writer.Write((byte)0); - writer.Write((byte)padding); + writer.Write((byte)remain); } public static void WriteFixedString(this BinaryWriter writer, string value, int length) diff --git a/neo/SmartContract/InteropService.NEO.cs b/neo/SmartContract/InteropService.NEO.cs index 5787cebdb4..669a9b7073 100644 --- a/neo/SmartContract/InteropService.NEO.cs +++ b/neo/SmartContract/InteropService.NEO.cs @@ -353,7 +353,7 @@ private static bool Storage_Find(ApplicationEngine engine) while (remain >= 16) { ms.Write(prefix, index, 16); - ms.WriteByte(0); + ms.WriteByte(16); index += 16; remain -= 16; }