From 99059aab46b8dcd6feb6be15c710267ee74309dd Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 24 Jul 2019 14:07:21 +0800 Subject: [PATCH 01/11] Fixes #946 --- neo/IO/ByteArrayComparer.cs | 21 ++++++++++++++++ neo/IO/Caching/DataCache.cs | 49 ++++++++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 neo/IO/ByteArrayComparer.cs diff --git a/neo/IO/ByteArrayComparer.cs b/neo/IO/ByteArrayComparer.cs new file mode 100644 index 0000000000..956ab758e6 --- /dev/null +++ b/neo/IO/ByteArrayComparer.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; + +namespace Neo.IO +{ + internal class ByteArrayComparer : IComparer + { + public static readonly ByteArrayComparer Default = new ByteArrayComparer(); + + public int Compare(byte[] x, byte[] y) + { + int length = Math.Min(x.Length, y.Length); + for (int i = 0; i < length; i++) + { + int r = x[i].CompareTo(y[i]); + if (r != 0) return r; + } + return x.Length.CompareTo(y.Length); + } + } +} diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index 5e9886d10c..acbe097be2 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -120,14 +120,51 @@ public void DeleteWhere(Func predicate) public IEnumerable> Find(byte[] key_prefix = null) { + IEnumerable<(byte[], TKey, TValue)> cached; lock (dictionary) { - foreach (var pair in FindInternal(key_prefix ?? new byte[0])) - if (!dictionary.ContainsKey(pair.Key)) - yield return pair; - foreach (var pair in dictionary) - if (pair.Value.State != TrackState.Deleted && (key_prefix == null || pair.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix))) - yield return new KeyValuePair(pair.Key, pair.Value.Item); + cached = dictionary + .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix))) + .Select(p => + ( + KeyBytes: p.Key.ToArray(), + p.Key, + p.Value.Item + )) + .OrderBy(p => p.KeyBytes, ByteArrayComparer.Default) + .ToArray(); + } + var uncached = FindInternal(key_prefix ?? new byte[0]) + .Where(p => !dictionary.ContainsKey(p.Key)) + .Select(p => + ( + KeyBytes: p.Key.ToArray(), + p.Key, + p.Value + )); + using (var e1 = cached.GetEnumerator()) + using (var e2 = uncached.GetEnumerator()) + { + (byte[] KeyBytes, TKey Key, TValue Item) i1, i2; + bool c1 = e1.MoveNext(); + bool c2 = e2.MoveNext(); + i1 = c1 ? e1.Current : default; + i2 = c2 ? e2.Current : default; + while (c1 || c2) + { + if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) < 0)) + { + yield return new KeyValuePair(i1.Key, i1.Item); + c1 = e1.MoveNext(); + i1 = c1 ? e1.Current : default; + } + else + { + yield return new KeyValuePair(i2.Key, i2.Item); + c2 = e2.MoveNext(); + i2 = c2 ? e2.Current : default; + } + } } } From ad6722386d13b4101414d62f34ce50931ae0485a Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 11:50:30 +0200 Subject: [PATCH 02/11] Copy ut from https://github.com/neo-project/neo/pull/947 (#952) --- neo.UnitTests/UT_DataCache.cs | 101 ++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 neo.UnitTests/UT_DataCache.cs diff --git a/neo.UnitTests/UT_DataCache.cs b/neo.UnitTests/UT_DataCache.cs new file mode 100644 index 0000000000..2993e27955 --- /dev/null +++ b/neo.UnitTests/UT_DataCache.cs @@ -0,0 +1,101 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO.Caching; +using Neo.Ledger; +using System.Linq; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_DataCache + { + [TestInitialize] + public void TestSetup() + { + TestBlockchain.InitializeMockNeoSystem(); + } + + [TestMethod] + public void TestCachedFind_Between() + { + var snapshot = TestBlockchain.GetStore().GetSnapshot(); + var storages = snapshot.Storages; + var cache = new CloneCache(storages); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x03 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + + CollectionAssert.AreEqual( + cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), + new byte[] { 0x01, 0x02, 0x03 } + ); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + } + + [TestMethod] + public void TestCachedFind_Last() + { + var snapshot = TestBlockchain.GetStore().GetSnapshot(); + var storages = snapshot.Storages; + var cache = new CloneCache(storages); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + + CollectionAssert.AreEqual( + cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), + new byte[] { 0x01, 0x02 } + ); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + } + + [TestMethod] + public void TestCachedFind_Empty() + { + var snapshot = TestBlockchain.GetStore().GetSnapshot(); + var storages = snapshot.Storages; + var cache = new CloneCache(storages); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); + + CollectionAssert.AreEqual( + cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), + new byte[] { 0x02 } + ); + + storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + } + } +} \ No newline at end of file From 2dfac185c6028763a07a3baa08b181934819ceb0 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 11:55:25 +0200 Subject: [PATCH 03/11] Added entries with wrong prefix --- neo.UnitTests/UT_DataCache.cs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/neo.UnitTests/UT_DataCache.cs b/neo.UnitTests/UT_DataCache.cs index 2993e27955..40c3ee8a3b 100644 --- a/neo.UnitTests/UT_DataCache.cs +++ b/neo.UnitTests/UT_DataCache.cs @@ -23,6 +23,11 @@ public void TestCachedFind_Between() storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); storages.Add ( new StorageKey() { Key = new byte[] { 0x00, 0x01 }, ScriptHash = UInt160.Zero }, @@ -33,6 +38,11 @@ public void TestCachedFind_Between() new StorageKey() { Key = new byte[] { 0x00, 0x03 }, ScriptHash = UInt160.Zero }, new StorageItem() { IsConstant = false, Value = new byte[] { } } ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); cache.Add ( new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, @@ -61,11 +71,21 @@ public void TestCachedFind_Last() new StorageKey() { Key = new byte[] { 0x00, 0x01 }, ScriptHash = UInt160.Zero }, new StorageItem() { IsConstant = false, Value = new byte[] { } } ); + storages.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x01 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); cache.Add ( new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, new StorageItem() { IsConstant = false, Value = new byte[] { } } ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); CollectionAssert.AreEqual( cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), @@ -89,6 +109,11 @@ public void TestCachedFind_Empty() new StorageKey() { Key = new byte[] { 0x00, 0x02 }, ScriptHash = UInt160.Zero }, new StorageItem() { IsConstant = false, Value = new byte[] { } } ); + cache.Add + ( + new StorageKey() { Key = new byte[] { 0x01, 0x02 }, ScriptHash = UInt160.Zero }, + new StorageItem() { IsConstant = false, Value = new byte[] { } } + ); CollectionAssert.AreEqual( cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), @@ -98,4 +123,4 @@ public void TestCachedFind_Empty() storages.DeleteWhere((k, v) => k.ScriptHash == UInt160.Zero); } } -} \ No newline at end of file +} From 8f689e1e677d9a92fb13838501bb61c76526c12a Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 12:18:42 +0200 Subject: [PATCH 04/11] Fix --- neo/IO/Caching/DataCache.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index acbe097be2..4a077e2813 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -118,13 +118,25 @@ public void DeleteWhere(Func predicate) } } + public byte[] ToByteArray(TKey x) + { + if (x is StorageKey xx) + { + return xx.Key; + } + else + { + return x.ToArray(); + } + } + public IEnumerable> Find(byte[] key_prefix = null) { IEnumerable<(byte[], TKey, TValue)> cached; lock (dictionary) { cached = dictionary - .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix))) + .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || ToByteArray(p.Key).Take(key_prefix.Length).SequenceEqual(key_prefix))) .Select(p => ( KeyBytes: p.Key.ToArray(), From 8ac83c65de203eaff7b9c6738e3cbef6841531ba Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 12:20:34 +0200 Subject: [PATCH 05/11] Complete fix --- neo/IO/Caching/DataCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index 4a077e2813..c97fa55726 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -150,7 +150,7 @@ public IEnumerable> Find(byte[] key_prefix = null) .Where(p => !dictionary.ContainsKey(p.Key)) .Select(p => ( - KeyBytes: p.Key.ToArray(), + KeyBytes: ToByteArray(p.Key), p.Key, p.Value )); From 06349e34109c1e5255735f3fca9d67dfa4efac2d Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 12:25:06 +0200 Subject: [PATCH 06/11] Update DataCache.cs --- neo/IO/Caching/DataCache.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index c97fa55726..92db244394 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -1,4 +1,5 @@ -using System; +using Neo.Ledger; +using System; using System.Collections.Generic; using System.Linq; From f2606e1d433126f31471afc5e90a936cee0592a2 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 24 Jul 2019 20:14:27 +0800 Subject: [PATCH 07/11] Revert DataCache.cs --- neo/IO/Caching/DataCache.cs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index 92db244394..acbe097be2 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -1,5 +1,4 @@ -using Neo.Ledger; -using System; +using System; using System.Collections.Generic; using System.Linq; @@ -119,25 +118,13 @@ public void DeleteWhere(Func predicate) } } - public byte[] ToByteArray(TKey x) - { - if (x is StorageKey xx) - { - return xx.Key; - } - else - { - return x.ToArray(); - } - } - public IEnumerable> Find(byte[] key_prefix = null) { IEnumerable<(byte[], TKey, TValue)> cached; lock (dictionary) { cached = dictionary - .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || ToByteArray(p.Key).Take(key_prefix.Length).SequenceEqual(key_prefix))) + .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix))) .Select(p => ( KeyBytes: p.Key.ToArray(), @@ -151,7 +138,7 @@ public IEnumerable> Find(byte[] key_prefix = null) .Where(p => !dictionary.ContainsKey(p.Key)) .Select(p => ( - KeyBytes: ToByteArray(p.Key), + KeyBytes: p.Key.ToArray(), p.Key, p.Value )); From ac85174e37a60df083a4676228c57f39ee4257d0 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 24 Jul 2019 20:25:30 +0800 Subject: [PATCH 08/11] real fix --- neo.UnitTests/UT_DataCache.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo.UnitTests/UT_DataCache.cs b/neo.UnitTests/UT_DataCache.cs index 40c3ee8a3b..85df272db5 100644 --- a/neo.UnitTests/UT_DataCache.cs +++ b/neo.UnitTests/UT_DataCache.cs @@ -50,7 +50,7 @@ public void TestCachedFind_Between() ); CollectionAssert.AreEqual( - cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), + cache.Find(new byte[21]).Select(u => u.Key.Key[1]).ToArray(), new byte[] { 0x01, 0x02, 0x03 } ); @@ -88,7 +88,7 @@ public void TestCachedFind_Last() ); CollectionAssert.AreEqual( - cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), + cache.Find(new byte[21]).Select(u => u.Key.Key[1]).ToArray(), new byte[] { 0x01, 0x02 } ); @@ -116,7 +116,7 @@ public void TestCachedFind_Empty() ); CollectionAssert.AreEqual( - cache.Find(new byte[] { 0x00 }).Select(u => u.Key.Key[1]).ToArray(), + cache.Find(new byte[21]).Select(u => u.Key.Key[1]).ToArray(), new byte[] { 0x02 } ); From 50c51da440d80cf2db7d3c1017f61aa7b3318a7c Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 25 Jul 2019 10:09:33 +0200 Subject: [PATCH 09/11] Update DataCache.cs --- neo/IO/Caching/DataCache.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/neo/IO/Caching/DataCache.cs b/neo/IO/Caching/DataCache.cs index acbe097be2..278c01d650 100644 --- a/neo/IO/Caching/DataCache.cs +++ b/neo/IO/Caching/DataCache.cs @@ -118,6 +118,11 @@ public void DeleteWhere(Func predicate) } } + /// + /// Find the entries that start with the `key_prefix` + /// + /// Must maintain the deserialized format of TKey + /// Entries found with the desired prefix public IEnumerable> Find(byte[] key_prefix = null) { IEnumerable<(byte[], TKey, TValue)> cached; From 41ce9180246ae720428ca1e1dd062c69c8c87979 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Fri, 26 Jul 2019 14:32:35 +0800 Subject: [PATCH 10/11] Sort unrelated grouping --- neo/IO/Helper.cs | 13 ++++++------- neo/SmartContract/InteropService.NEO.cs | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/neo/IO/Helper.cs b/neo/IO/Helper.cs index 647ae2cfb1..a306e297d5 100644 --- a/neo/IO/Helper.cs +++ b/neo/IO/Helper.cs @@ -92,17 +92,16 @@ public static byte[] ReadBytesWithGrouping(this BinaryReader reader) { using (MemoryStream ms = new MemoryStream()) { - int padding = 0; + int count; do { byte[] group = reader.ReadBytes(GroupingSizeInBytes); - padding = reader.ReadByte(); - if (padding > GroupingSizeInBytes) + count = reader.ReadByte(); + if (count > GroupingSizeInBytes) throw new FormatException(); - int count = GroupingSizeInBytes - padding; if (count > 0) ms.Write(group, 0, count); - } while (padding == 0); + } while (count == GroupingSizeInBytes); return ms.ToArray(); } } @@ -200,7 +199,7 @@ public static void WriteBytesWithGrouping(this BinaryWriter writer, byte[] value while (remain >= GroupingSizeInBytes) { writer.Write(value, index, GroupingSizeInBytes); - writer.Write((byte)0); + writer.Write((byte)GroupingSizeInBytes); index += GroupingSizeInBytes; remain -= GroupingSizeInBytes; } @@ -209,7 +208,7 @@ public static void WriteBytesWithGrouping(this BinaryWriter writer, byte[] value int padding = GroupingSizeInBytes - remain; for (int i = 0; i < padding; i++) writer.Write((byte)0); - writer.Write((byte)padding); + writer.Write((byte)remain); } public static void WriteFixedString(this BinaryWriter writer, string value, int length) diff --git a/neo/SmartContract/InteropService.NEO.cs b/neo/SmartContract/InteropService.NEO.cs index 5787cebdb4..669a9b7073 100644 --- a/neo/SmartContract/InteropService.NEO.cs +++ b/neo/SmartContract/InteropService.NEO.cs @@ -353,7 +353,7 @@ private static bool Storage_Find(ApplicationEngine engine) while (remain >= 16) { ms.Write(prefix, index, 16); - ms.WriteByte(0); + ms.WriteByte(16); index += 16; remain -= 16; } From 68ab63a488d88d318ee2e50b312e675458f7cc56 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 29 Jul 2019 07:34:02 +0200 Subject: [PATCH 11/11] Fix unit tests --- neo.UnitTests/IO/UT_IOHelper.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/neo.UnitTests/IO/UT_IOHelper.cs b/neo.UnitTests/IO/UT_IOHelper.cs index e113ce8cd4..b0593c02ec 100644 --- a/neo.UnitTests/IO/UT_IOHelper.cs +++ b/neo.UnitTests/IO/UT_IOHelper.cs @@ -234,9 +234,9 @@ public void TestReadBytesWithGrouping() else { byte[] caseArray = new byte[] { 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,0x11}; MemoryStream stream = new MemoryStream(); @@ -415,11 +415,11 @@ public void TestWriteBytesWithGrouping() byte[] byteArray = new byte[stream.Length]; stream.Read(byteArray, 0, (int)stream.Length); Assert.AreEqual(Encoding.Default.GetString(new byte[] { 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, - 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x00, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,0x10, 0xAA, 0xAA, 0xAA, 0xAA, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,0x0C}), Encoding.Default.GetString(byteArray)); + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,0x04}), Encoding.Default.GetString(byteArray)); } [TestMethod] @@ -546,4 +546,4 @@ public void TestWriteVarString() Assert.AreEqual(0x61, byteArray[1]); } } -} \ No newline at end of file +}