Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace function exceptwith and unionwith with faster functions #1174

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
36c1d16
Merge pull request #1 from neo-project/master
Qiao-Jin Oct 22, 2019
b7f9c02
Replace ExceptWith & UnionWith with equal but faster functionality
Oct 22, 2019
318c99f
Optimization
shargon Oct 22, 2019
0d6b842
Merge branch 'master' into replace-func-exceptwith-unionwith
shargon Oct 22, 2019
af23059
Optimization
shargon Oct 22, 2019
82930da
Optimize remove
shargon Oct 22, 2019
54d2af8
Update neo/Network/P2P/TaskManager.cs
Qiao-Jin Oct 23, 2019
a745880
Code optimization
Oct 23, 2019
e0f9145
Merge branch 'replace-func-exceptwith-unionwith' into review
Qiao-Jin Oct 23, 2019
a84eff9
Update Helper.cs
Qiao-Jin Oct 23, 2019
dd36f20
Merge pull request #2 from shargon/review
Qiao-Jin Oct 23, 2019
089b935
Small change
shargon Oct 23, 2019
a571c22
Optimization
shargon Oct 23, 2019
4381b4c
Update Helper.cs
shargon Oct 23, 2019
dd4a435
Revert
shargon Oct 23, 2019
95136ed
Optimization
shargon Oct 23, 2019
edc0333
Optimize FIFOSet
shargon Oct 23, 2019
4ddc299
Rename
shargon Oct 23, 2019
6da2fc2
Inline
shargon Oct 23, 2019
bcf1c62
Merge pull request #4 from shargon/optimization
Qiao-Jin Oct 23, 2019
88753a6
Update UT_FIFOSet.cs
shargon Oct 23, 2019
59ee120
Fix
shargon Oct 23, 2019
921a343
Update UT_FIFOSet.cs
shargon Oct 23, 2019
621c4e3
Update FIFOSet.cs
shargon Oct 23, 2019
051b7c1
Update FIFOSet.cs
shargon Oct 23, 2019
96418c4
Merge branch 'master' into replace-func-exceptwith-unionwith
vncoelho Oct 23, 2019
f325f96
Merge branch 'master' into replace-func-exceptwith-unionwith
Qiao-Jin Oct 28, 2019
b1f5914
Merge branch 'master' into replace-func-exceptwith-unionwith
erikzhang Oct 30, 2019
f1cef96
Revert FIFOSet
erikzhang Oct 30, 2019
2e704e6
Update Helper.cs
erikzhang Oct 30, 2019
12aa262
Optimize
erikzhang Oct 30, 2019
9b71d5f
Merge branch 'master' into replace-func-exceptwith-unionwith
vncoelho Oct 30, 2019
b23244f
Merge branch 'master' into replace-func-exceptwith-unionwith
vncoelho Oct 30, 2019
360f20a
Reverting independet byte checks to SequenceEqual
vncoelho Oct 31, 2019
ec2bfd7
Merge branch 'master' into replace-func-exceptwith-unionwith
vncoelho Oct 31, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions neo.UnitTests/IO/Caching/UT_FIFOSet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,24 @@ public void TestExceptWith()
};
set.ExceptWith(new UInt256[] { b, c });
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a });

set = new FIFOSet<UInt256>(10)
{
a,
b,
c
};
set.ExceptWith(new UInt256[] { a });
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c });

set = new FIFOSet<UInt256>(10)
{
a,
b,
c
};
set.ExceptWith(new UInt256[] { c });
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, b });
}
}
}
39 changes: 38 additions & 1 deletion neo/Helper.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using Microsoft.Extensions.Configuration;
using Neo.Plugins;
using Neo.IO.Caching;
using System;
using System.Collections.Generic;
using System.Globalization;
Expand Down Expand Up @@ -54,6 +54,43 @@ internal static int GetLowestSetBit(this BigInteger i)
throw new Exception();
}

internal static void Remove<T>(this HashSet<T> set, ISet<T> other)
{
if (set.Count > other.Count)
{
set.ExceptWith(other);
}
else
{
set.RemoveWhere(u => other.Contains(u));
}
}

internal static void Remove<T>(this HashSet<T> set, FIFOSet<T> other)
where T : IEquatable<T>
{
if (set.Count > other.Count)
{
set.ExceptWith(other);
}
else
{
set.RemoveWhere(u => other.Contains(u));
}
}

internal static void Remove<T, V>(this HashSet<T> set, IReadOnlyDictionary<T, V> other)
{
if (set.Count > other.Count)
{
set.ExceptWith(other.Keys);
}
else
{
set.RemoveWhere(u => other.ContainsKey(u));
}
}

internal static string GetVersion(this Assembly assembly)
{
CustomAttributeData attribute = assembly.CustomAttributes.FirstOrDefault(p => p.AttributeType == typeof(AssemblyInformationalVersionAttribute));
Expand Down
10 changes: 6 additions & 4 deletions neo/IO/Caching/FIFOSet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@

namespace Neo.IO.Caching
{
internal class FIFOSet<T> : IEnumerable<T> where T : IEquatable<T>
internal class FIFOSet<T> : IReadOnlyCollection<T> where T : IEquatable<T>
{
private readonly int maxCapacity;
private readonly int removeCount;
private readonly OrderedDictionary dictionary;

public int Count => dictionary.Count;

public FIFOSet(int maxCapacity, decimal batchSize = 0.1m)
{
if (maxCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(maxCapacity));
Expand Down Expand Up @@ -46,11 +48,11 @@ public bool Contains(T item)
return dictionary.Contains(item);
}

public void ExceptWith(IEnumerable<UInt256> hashes)
erikzhang marked this conversation as resolved.
Show resolved Hide resolved
public void ExceptWith(IEnumerable<T> entries)
{
foreach (var hash in hashes)
foreach (var entry in entries)
{
dictionary.Remove(hash);
dictionary.Remove(entry);
}
}

Expand Down
8 changes: 4 additions & 4 deletions neo/Network/P2P/TaskManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@ private void OnNewTasks(InvPayload payload)
return;
}
HashSet<UInt256> hashes = new HashSet<UInt256>(payload.Hashes);
hashes.ExceptWith(knownHashes);
hashes.Remove(knownHashes);
if (payload.Type == InventoryType.Block)
session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p)));

hashes.ExceptWith(globalTasks.Keys);
hashes.Remove(globalTasks);
if (hashes.Count == 0)
{
RequestTasks(session);
Expand Down Expand Up @@ -203,7 +203,7 @@ private void RequestTasks(TaskSession session)
if (session.HasTask) return;
if (session.AvailableTasks.Count > 0)
{
session.AvailableTasks.ExceptWith(knownHashes);
session.AvailableTasks.Remove(knownHashes);
session.AvailableTasks.RemoveWhere(p => Blockchain.Singleton.ContainsBlock(p));
HashSet<UInt256> hashes = new HashSet<UInt256>(session.AvailableTasks);
if (hashes.Count > 0)
Expand All @@ -213,7 +213,7 @@ private void RequestTasks(TaskSession session)
if (!IncrementGlobalTask(hash))
hashes.Remove(hash);
}
session.AvailableTasks.ExceptWith(hashes);
session.AvailableTasks.Remove(hashes);
foreach (UInt256 hash in hashes)
session.Tasks[hash] = DateTime.UtcNow;
foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray()))
Expand Down
2 changes: 2 additions & 0 deletions neo/UIntBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;

namespace Neo
{
Expand Down Expand Up @@ -110,6 +111,7 @@ void ISerializable.Serialize(BinaryWriter writer)
/// <summary>
/// Method ToArray() returns the byte array data_bytes, which stores the little-endian unsigned int
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte[] ToArray()
{
return data_bytes;
Expand Down