Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a change list to optimize datacache commit #1619

Merged
merged 14 commits into from
May 7, 2020
Merged

Add a change list to optimize datacache commit #1619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
48c4b18
Add a change list to optimize datacache commit
Apr 29, 2020
69371a5
Datacache changeSet logic optimization
Apr 30, 2020
26de072
Code simplification
Apr 30, 2020
61181cb
Merge branch 'master' into enhance_datacache
shargon Apr 30, 2020
6735e6a
Rename to changSet
Apr 30, 2020
bbb66de
Merge branch 'enhance_datacache' of https://github.com/Qiao-Jin/neo i…
Apr 30, 2020
f41e386
optimize code
Apr 30, 2020
8d0d325
optimize code
Apr 30, 2020
69f32f7
reset neo.csporj
Apr 30, 2020
18d8bfd
remove white blank
Apr 30, 2020
4330773
remove DeleteWhere
May 1, 2020
b5d44fe
fix ut
May 1, 2020
c80b3bf
Merge branch 'master' into enhance_datacache
Qiao-Jin May 6, 2020
7eadbfe
Merge branch 'master' into enhance_datacache
erikzhang May 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions src/neo/IO/Caching/DataCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class Trackable
}

private readonly Dictionary<TKey, Trackable> dictionary = new Dictionary<TKey, Trackable>();
private readonly HashSet<TKey> changeSet = new HashSet<TKey>();

public TValue this[TKey key]
{
Expand Down Expand Up @@ -65,6 +66,7 @@ public void Add(TKey key, TValue value)
Item = value,
State = trackable == null ? TrackState.Added : TrackState.Changed
};
changeSet.Add(key);
}
}

Expand Down Expand Up @@ -96,6 +98,7 @@ public void Commit()
{
dictionary.Remove(key);
}
changeSet.Clear();
}

public DataCache<TKey, TValue> CreateSnapshot()
Expand All @@ -114,9 +117,15 @@ public void Delete(TKey key)
if (dictionary.TryGetValue(key, out Trackable trackable))
{
if (trackable.State == TrackState.Added)
{
dictionary.Remove(key);
changeSet.Remove(key);
}
else
{
trackable.State = TrackState.Deleted;
changeSet.Add(key);
}
}
else
{
Expand All @@ -128,21 +137,13 @@ public void Delete(TKey key)
Item = item,
State = TrackState.Deleted
});
changeSet.Add(key);
}
}
}

protected abstract void DeleteInternal(TKey key);

public void DeleteWhere(Func<TKey, TValue, bool> predicate)
{
lock (dictionary)
{
foreach (Trackable trackable in dictionary.Where(p => p.Value.State != TrackState.Deleted && predicate(p.Key, p.Value.Item)).Select(p => p.Value))
trackable.State = TrackState.Deleted;
}
}

/// <summary>
/// Find the entries that start with the `key_prefix`
/// </summary>
Expand Down Expand Up @@ -204,8 +205,8 @@ public IEnumerable<Trackable> GetChangeSet()
{
lock (dictionary)
{
foreach (Trackable trackable in dictionary.Values.Where(p => p.State != TrackState.None))
yield return trackable;
foreach (TKey key in changeSet)
yield return dictionary[key];
}
}

Expand Down Expand Up @@ -234,6 +235,7 @@ public TValue GetAndChange(TKey key, Func<TValue> factory = null)
else if (trackable.State == TrackState.None)
{
trackable.State = TrackState.Changed;
changeSet.Add(key);
}
}
else
Expand All @@ -254,6 +256,7 @@ public TValue GetAndChange(TKey key, Func<TValue> factory = null)
trackable.State = TrackState.Changed;
}
dictionary.Add(key, trackable);
changeSet.Add(key);
}
return trackable.Item;
}
Expand Down Expand Up @@ -282,6 +285,7 @@ public TValue GetOrAdd(TKey key, Func<TValue> factory)
{
trackable.Item = factory();
trackable.State = TrackState.Added;
changeSet.Add(key);
}
else
{
Expand Down
17 changes: 0 additions & 17 deletions tests/neo.UnitTests/IO/Caching/UT_DataCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,23 +246,6 @@ public void TestDelete()
myDataCache.InnerDict.ContainsKey(new MyKey("key2")).Should().BeFalse();
}

[TestMethod]
public void TestDeleteWhere()
{
myDataCache.Add(new MyKey("key1"), new MyValue("value1"));
myDataCache.Add(new MyKey("key2"), new MyValue("value2"));

myDataCache.InnerDict.Add(new MyKey("key3"), new MyValue("value3"));
myDataCache.InnerDict.Add(new MyKey("key4"), new MyValue("value4"));

myDataCache.DeleteWhere((k, v) => k.Key.StartsWith("key"));
myDataCache.Commit();
myDataCache.TryGet(new MyKey("key1")).Should().BeNull();
myDataCache.TryGet(new MyKey("key2")).Should().BeNull();
myDataCache.InnerDict.ContainsKey(new MyKey("key1")).Should().BeFalse();
myDataCache.InnerDict.ContainsKey(new MyKey("key2")).Should().BeFalse();
}

[TestMethod]
public void TestFind()
{
Expand Down
8 changes: 4 additions & 4 deletions tests/neo.UnitTests/SmartContract/UT_Syscalls.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ public void System_Runtime_GasLeft()
public void System_Runtime_GetInvocationCounter()
{
ContractState contractA, contractB, contractC;
var snapshot = Blockchain.Singleton.GetSnapshot();
var snapshot = Blockchain.Singleton.GetSnapshot().Clone();
var contracts = snapshot.Contracts;

// Create dummy contracts
Expand All @@ -339,9 +339,9 @@ public void System_Runtime_GetInvocationCounter()
// Init A,B,C contracts
// First two drops is for drop method and arguments

contracts.DeleteWhere((a, b) => a.ToArray().SequenceEqual(contractA.ScriptHash.ToArray()));
contracts.DeleteWhere((a, b) => a.ToArray().SequenceEqual(contractB.ScriptHash.ToArray()));
contracts.DeleteWhere((a, b) => a.ToArray().SequenceEqual(contractC.ScriptHash.ToArray()));
contracts.Delete(contractA.ScriptHash);
contracts.Delete(contractB.ScriptHash);
contracts.Delete(contractC.ScriptHash);
contractA.Manifest = TestUtils.CreateDefaultManifest(contractA.ScriptHash, "dummyMain");
contractB.Manifest = TestUtils.CreateDefaultManifest(contractA.ScriptHash, "dummyMain");
contractC.Manifest = TestUtils.CreateDefaultManifest(contractA.ScriptHash, "dummyMain");
Expand Down
18 changes: 3 additions & 15 deletions tests/neo.UnitTests/UT_DataCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@ public void TestSetup()
public void TestCachedFind_Between()
{
var snapshot = Blockchain.Singleton.GetSnapshot();
var storages = snapshot.Storages;
var storages = snapshot.Storages.CreateSnapshot();
var cache = new CloneCache<StorageKey, StorageItem>(storages);

storages.DeleteWhere((k, v) => k.Id == 0);

storages.Add
(
new StorageKey() { Key = new byte[] { 0x01, 0x01 }, Id = 0 },
Expand Down Expand Up @@ -53,19 +51,15 @@ public void TestCachedFind_Between()
cache.Find(new byte[5]).Select(u => u.Key.Key[1]).ToArray(),
new byte[] { 0x01, 0x02, 0x03 }
);

storages.DeleteWhere((k, v) => k.Id == 0);
}

[TestMethod]
public void TestCachedFind_Last()
{
var snapshot = Blockchain.Singleton.GetSnapshot();
var storages = snapshot.Storages;
var storages = snapshot.Storages.CreateSnapshot();
var cache = new CloneCache<StorageKey, StorageItem>(storages);

storages.DeleteWhere((k, v) => k.Id == 0);

storages.Add
(
new StorageKey() { Key = new byte[] { 0x00, 0x01 }, Id = 0 },
Expand All @@ -89,19 +83,15 @@ public void TestCachedFind_Last()
CollectionAssert.AreEqual(cache.Find(new byte[5]).Select(u => u.Key.Key[1]).ToArray(),
new byte[] { 0x01, 0x02 }
);

storages.DeleteWhere((k, v) => k.Id == 0);
}

[TestMethod]
public void TestCachedFind_Empty()
{
var snapshot = Blockchain.Singleton.GetSnapshot();
var storages = snapshot.Storages;
var storages = snapshot.Storages.CreateSnapshot();
var cache = new CloneCache<StorageKey, StorageItem>(storages);

storages.DeleteWhere((k, v) => k.Id == 0);

cache.Add
(
new StorageKey() { Key = new byte[] { 0x00, 0x02 }, Id = 0 },
Expand All @@ -117,8 +107,6 @@ public void TestCachedFind_Empty()
cache.Find(new byte[5]).Select(u => u.Key.Key[1]).ToArray(),
new byte[] { 0x02 }
);

storages.DeleteWhere((k, v) => k.Id == 0);
}
}
}