Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cache to native contract executions V2 #1766

Merged
merged 11 commits into from
Jul 17, 2020
Merged

Add cache to native contract executions V2 #1766

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8e160f6
cache v2
shargon Jul 14, 2020
a6c0354
More optimizations
shargon Jul 14, 2020
3f5f07d
Policy contract optimization
shargon Jul 14, 2020
a2cc20f
Remove interporable variable
shargon Jul 14, 2020
9f3a1b4
Two more
shargon Jul 14, 2020
554b44b
Add Set
shargon Jul 14, 2020
92ab0f5
Merge branch 'master' into native-cache-v2
shargon Jul 16, 2020
bcf453e
Optimizations
shargon Jul 16, 2020
62f1413
Optimize
erikzhang Jul 17, 2020
5da2463
Merge branch 'master' into native-cache-v2
erikzhang Jul 17, 2020
95c3494
Merge branch 'master' into native-cache-v2
shargon Jul 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 57 additions & 10 deletions src/neo/Ledger/StorageItem.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,45 +1,52 @@
using Neo.IO;
using Neo.SmartContract;
using System.Collections.Generic;
using System.IO;
using System.Numerics;

namespace Neo.Ledger
{
public class StorageItem : ICloneable<StorageItem>, ISerializable
{
private byte[] value;
private IInteroperable interoperable;
public bool IsConstant;
private object cached;

public int Size => Value.GetVarSize() + sizeof(bool);

public byte[] Value
{
get
{
if (value is null && interoperable != null)
if (value is null && cached is IInteroperable interoperable)
value = BinarySerializer.Serialize(interoperable.ToStackItem(null), 4096);
return value;
}
set
{
interoperable = null;
cached = null;
this.value = value;
}
}

public StorageItem()
{
}
public StorageItem() { }

public StorageItem(byte[] value, bool isConstant = false)
{
this.value = value;
this.IsConstant = isConstant;
}

public StorageItem(BigInteger value, bool isConstant = false)
{
this.value = value.ToByteArrayStandard();
this.cached = value;
this.IsConstant = isConstant;
}

public StorageItem(IInteroperable interoperable, bool isConstant = false)
{
this.interoperable = interoperable;
this.cached = interoperable;
this.IsConstant = isConstant;
}

Expand All @@ -58,6 +65,45 @@ public void Deserialize(BinaryReader reader)
IsConstant = reader.ReadBoolean();
}

public T[] GetSerializableArray<T>(int max = 0x1000000) where T : ISerializable, new()
{
if (cached is T[] value) return value;
erikzhang marked this conversation as resolved.
Show resolved Hide resolved

var ret = Value.AsSerializableArray<T>(max);
cached = ret;
return ret;
}

public T GetSerializable<T>() where T : ISerializable, new()
{
if (cached is T value) return value;

var ret = Value.AsSerializable<T>();
cached = ret;
return ret;
}

public BigInteger GetBigInteger()
{
if (cached is BigInteger value) return value;

var ret = new BigInteger(Value);
cached = ret;
return ret;
}

public void Set(BigInteger value)
{
this.value = value.ToByteArrayStandard();
this.cached = value;
}

public void Set<T>(IReadOnlyCollection<T> value) where T : ISerializable
{
this.value = value.ToByteArray();
this.cached = value;
}

void ICloneable<StorageItem>.FromReplica(StorageItem replica)
{
Value = replica.Value;
Expand All @@ -66,13 +112,14 @@ void ICloneable<StorageItem>.FromReplica(StorageItem replica)

public T GetInteroperable<T>() where T : IInteroperable, new()
{
if (interoperable is null)
if (cached is null)
{
interoperable = new T();
var interoperable = new T();
interoperable.FromStackItem(BinarySerializer.Deserialize(value, 16, 34));
cached = interoperable;
}
value = null;
return (T)interoperable;
return (T)cached;
}

public void Serialize(BinaryWriter writer)
Expand Down
10 changes: 5 additions & 5 deletions src/neo/SmartContract/Native/PolicyContract.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public long GetFeePerByte(StoreView snapshot)
[ContractMethod(0_01000000, CallFlags.AllowStates)]
public UInt160[] GetBlockedAccounts(StoreView snapshot)
{
return snapshot.Storages[CreateStorageKey(Prefix_BlockedAccounts)].Value.AsSerializableArray<UInt160>();
return snapshot.Storages[CreateStorageKey(Prefix_BlockedAccounts)].GetSerializableArray<UInt160>();
}

[ContractMethod(0_03000000, CallFlags.AllowModifyStates)]
Expand Down Expand Up @@ -129,10 +129,10 @@ private bool BlockAccount(ApplicationEngine engine, UInt160 account)
if (!CheckCommittees(engine)) return false;
StorageKey key = CreateStorageKey(Prefix_BlockedAccounts);
StorageItem storage = engine.Snapshot.Storages[key];
SortedSet<UInt160> accounts = new SortedSet<UInt160>(storage.Value.AsSerializableArray<UInt160>());
SortedSet<UInt160> accounts = new SortedSet<UInt160>(storage.GetSerializableArray<UInt160>());
if (!accounts.Add(account)) return false;
storage = engine.Snapshot.Storages.GetAndChange(key);
storage.Value = accounts.ToByteArray();
storage.Set(accounts);
return true;
}

Expand All @@ -142,10 +142,10 @@ private bool UnblockAccount(ApplicationEngine engine, UInt160 account)
if (!CheckCommittees(engine)) return false;
StorageKey key = CreateStorageKey(Prefix_BlockedAccounts);
StorageItem storage = engine.Snapshot.Storages[key];
SortedSet<UInt160> accounts = new SortedSet<UInt160>(storage.Value.AsSerializableArray<UInt160>());
SortedSet<UInt160> accounts = new SortedSet<UInt160>(storage.GetSerializableArray<UInt160>());
if (!accounts.Remove(account)) return false;
storage = engine.Snapshot.Storages.GetAndChange(key);
storage.Value = accounts.ToByteArray();
storage.Set(accounts);
return true;
}
}
Expand Down
14 changes: 7 additions & 7 deletions src/neo/SmartContract/Native/Tokens/NeoToken.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ protected override void OnBalanceChanging(ApplicationEngine engine, UInt160 acco
{
engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_Candidate).Add(state.VoteTo)).GetInteroperable<CandidateState>().Votes += amount;
StorageItem item = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_VotersCount));
BigInteger votersCount = new BigInteger(item.Value) + amount;
item.Value = votersCount.ToByteArray();
BigInteger votersCount = item.GetBigInteger() + amount;
item.Set(votersCount);
}
}

Expand Down Expand Up @@ -99,7 +99,7 @@ protected override void OnPersist(ApplicationEngine engine)
{
base.OnPersist(engine);
StorageItem storage = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_NextValidators), () => new StorageItem());
storage.Value = GetValidators(engine.Snapshot).ToByteArray();
storage.Set(GetValidators(engine.Snapshot));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'd be nice to optimize this one also while we're here.

If you're to look at preview2 state change dump, you'll see something like

 {
  "block": 3,
  "size": 1,
  "storage": [
   {
    "state": "Changed",
    "key": "ffffffff0e00000000000000000000000000000001",
    "value": "e807023e9b32ea89b94d066e649b124fd50e396ee91369e8e2a6ae1b11c170d022256d03009b7540e10f2562e5fd8fac9eaec25166a58b26e412348ff5a86927bfac22a202ba2c70f5996f357a43198705859fae2cfea13e1172962800772b3d588a9d4abd03408dcd416396f64783ac587ea1e1593c57d9fea880c8a6a1920e92a25947780602a7834be9b32e2981d157cb5bbd3acb42cfd11ea5c3b10224d7a44e98c5910f1b0214baf0ceea3a66f17e7e1e839ea25fd8bed6cd82e6bb6e68250189065f44ff01030205e9cefaea5a1dfc580af20c8d5aa2468bb0148f1a5e4605fc622c80e604ba00"
   }
  ]
 },

for every block and this value obviously doesn't change often. I guess it would be same even with an updated (post-preview2) governance system. But we shouldn't update storage at all if this value doesn't change, it's not hard to do, actually the only reason we didn't do it immediately in neo-go is that it would break state change dumps compatibility for us (even though technically storage contents would be the same for each block).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The problem is that TrackeableState is outside StorageItem, I know what you say, I will try to improve it in other PR.

}

[ContractMethod(0_03000000, CallFlags.AllowStates)]
Expand Down Expand Up @@ -150,12 +150,12 @@ private bool Vote(ApplicationEngine engine, UInt160 account, ECPoint voteTo)
if (state_account.VoteTo is null ^ voteTo is null)
{
StorageItem item = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_VotersCount));
BigInteger votersCount = new BigInteger(item.Value);
BigInteger votersCount = item.GetBigInteger();
if (state_account.VoteTo is null)
votersCount += state_account.Balance;
else
votersCount -= state_account.Balance;
item.Value = votersCount.ToByteArray();
item.Set(votersCount);
}
if (state_account.VoteTo != null)
{
Expand Down Expand Up @@ -210,7 +210,7 @@ public UInt160 GetCommitteeAddress(StoreView snapshot)

private IEnumerable<ECPoint> GetCommitteeMembers(StoreView snapshot)
{
decimal votersCount = (decimal)new BigInteger(snapshot.Storages[CreateStorageKey(Prefix_VotersCount)].Value);
decimal votersCount = (decimal)snapshot.Storages[CreateStorageKey(Prefix_VotersCount)].GetBigInteger();
decimal VoterTurnout = votersCount / (decimal)TotalAmount;
if (VoterTurnout < EffectiveVoterTurnout)
return Blockchain.StandbyCommittee;
Expand All @@ -225,7 +225,7 @@ public ECPoint[] GetNextBlockValidators(StoreView snapshot)
{
StorageItem storage = snapshot.Storages.TryGet(CreateStorageKey(Prefix_NextValidators));
if (storage is null) return Blockchain.StandbyValidators;
return storage.Value.AsSerializableArray<ECPoint>();
return storage.GetSerializableArray<ECPoint>();
}

public class NeoAccountState : AccountState
Expand Down
15 changes: 6 additions & 9 deletions src/neo/SmartContract/Native/Tokens/Nep5Token.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,10 @@ internal protected virtual void Mint(ApplicationEngine engine, UInt160 account,
TState state = storage.GetInteroperable<TState>();
OnBalanceChanging(engine, account, state, amount);
state.Balance += amount;
storage = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_TotalSupply), () => new StorageItem
{
Value = BigInteger.Zero.ToByteArrayStandard()
});
BigInteger totalSupply = new BigInteger(storage.Value);
storage = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_TotalSupply), () => new StorageItem(BigInteger.Zero));
BigInteger totalSupply = storage.GetBigInteger();
totalSupply += amount;
storage.Value = totalSupply.ToByteArrayStandard();
storage.Set(totalSupply);
engine.SendNotification(Hash, "Transfer", new Array { StackItem.Null, account.ToArray(), amount });
}

Expand All @@ -90,9 +87,9 @@ internal protected virtual void Burn(ApplicationEngine engine, UInt160 account,
else
state.Balance -= amount;
storage = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_TotalSupply));
BigInteger totalSupply = new BigInteger(storage.Value);
BigInteger totalSupply = storage.GetBigInteger();
totalSupply -= amount;
storage.Value = totalSupply.ToByteArrayStandard();
storage.Set(totalSupply);
engine.SendNotification(Hash, "Transfer", new Array { account.ToArray(), StackItem.Null, amount });
}

Expand All @@ -101,7 +98,7 @@ public virtual BigInteger TotalSupply(StoreView snapshot)
{
StorageItem storage = snapshot.Storages.TryGet(CreateStorageKey(Prefix_TotalSupply));
if (storage is null) return BigInteger.Zero;
return new BigInteger(storage.Value);
return storage.GetBigInteger();
}

[ContractMethod(0_01000000, CallFlags.AllowStates)]
Expand Down