Relocate transaction verification (#1507)

src/neo/Ledger/Blockchain.cs

@@ -26,6 +26,7 @@ public class Import { public IEnumerable<Block> Blocks; public bool Verify = tru
         public class ImportCompleted { }
         public class FillMemoryPool { public IEnumerable<Transaction> Transactions; }
         public class FillCompleted { }
+        internal class PreverifyCompleted { public Transaction Transaction; public VerifyResult Result; public bool Relay; }
         public class RelayResult { public IInventory Inventory; public VerifyResult Result; }
 
         public static readonly uint MillisecondsPerBlock = ProtocolSettings.Default.MillisecondsPerBlock;
@@ -58,6 +59,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu
         private const int MaxTxToReverifyPerIdle = 10;
         private static readonly object lockObj = new object();
         private readonly NeoSystem system;
+        private readonly IActorRef txrouter;
         private readonly List<UInt256> header_index = new List<UInt256>();
         private uint stored_header_count = 0;
         private readonly Dictionary<UInt256, Block> block_cache = new Dictionary<UInt256, Block>();
@@ -103,6 +105,7 @@ static Blockchain()
         public Blockchain(NeoSystem system, IStore store)
         {
             this.system = system;
+            this.txrouter = Context.ActorOf(TransactionRouter.Props(system));
             this.MemPool = new MemoryPool(system, ProtocolSettings.Default.MemoryPoolMaxTransactions);
             this.Store = store;
             this.View = new ReadOnlyView(store);
@@ -302,20 +305,15 @@ private void OnFillMemoryPool(IEnumerable<Transaction> transactions)
 
         private void OnInventory(IInventory inventory, bool relay = true)
         {
-            RelayResult rr = new RelayResult
+            VerifyResult result = inventory switch
             {
-                Inventory = inventory,
-                Result = inventory switch
-                {
-                    Block block => OnNewBlock(block),
-                    Transaction transaction => OnNewTransaction(transaction),
-                    _ => OnNewInventory(inventory)
-                }
+                Block block => OnNewBlock(block),
+                Transaction transaction => OnNewTransaction(transaction),
+                _ => OnNewInventory(inventory)
             };
-            if (relay && rr.Result == VerifyResult.Succeed)
+            if (relay && result == VerifyResult.Succeed)
                 system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = inventory });
-            Sender.Tell(rr);
-            Context.System.EventStream.Publish(rr);
+            SendRelayResult(inventory, result);
         }
 
         private VerifyResult OnNewBlock(Block block)
@@ -360,6 +358,14 @@ private VerifyResult OnNewTransaction(Transaction transaction)
             return MemPool.TryAdd(transaction, currentSnapshot);
         }
 
+        private void OnPreverifyCompleted(PreverifyCompleted task)
+        {
+            if (task.Result == VerifyResult.Succeed)
+                OnInventory(task.Transaction, task.Relay);
+            else
+                SendRelayResult(task.Transaction, task.Result);
+        }
+
         protected override void OnReceive(object message)
         {
             switch (message)
@@ -373,22 +379,34 @@ protected override void OnReceive(object message)
                 case Block block:
                     OnInventory(block, false);
                     break;
+                case Transaction tx:
+                    OnTransaction(tx, true);
+                    break;
                 case Transaction[] transactions:
-                    {
-                        // This message comes from a mempool's revalidation, already relayed
-                        foreach (var tx in transactions) OnInventory(tx, false);
-                        break;
-                    }
+                    // This message comes from a mempool's revalidation, already relayed
+                    foreach (var tx in transactions) OnTransaction(tx, false);
+                    break;
                 case IInventory inventory:
                     OnInventory(inventory);
                     break;
+                case PreverifyCompleted task:
+                    OnPreverifyCompleted(task);
+                    break;
                 case Idle _:
                     if (MemPool.ReVerifyTopUnverifiedTransactionsIfNeeded(MaxTxToReverifyPerIdle, currentSnapshot))
                         Self.Tell(Idle.Instance, ActorRefs.NoSender);
                     break;
             }
         }
 
+        private void OnTransaction(Transaction tx, bool relay)
+        {
+            if (ContainsTransaction(tx.Hash))
+                SendRelayResult(tx, VerifyResult.AlreadyExists);
+            else
+                txrouter.Tell(new TransactionRouter.Task { Transaction = tx, Relay = relay }, Sender);
+        }
+
         private void Persist(Block block)
         {
             using (SnapshotView snapshot = GetSnapshot())
@@ -507,6 +525,17 @@ private void SaveHeaderHashList(SnapshotView snapshot = null)
             }
         }
 
+        private void SendRelayResult(IInventory inventory, VerifyResult result)
+        {
+            RelayResult rr = new RelayResult
+            {
+                Inventory = inventory,
+                Result = result
+            };
+            Sender.Tell(rr);
+            Context.System.EventStream.Publish(rr);
+        }
+
         private void UpdateCurrentSnapshot()
         {
             Interlocked.Exchange(ref currentSnapshot, GetSnapshot())?.Dispose();

src/neo/Ledger/MemoryPool.cs

@@ -271,7 +271,7 @@ internal VerifyResult TryAdd(Transaction tx, StoreView snapshot)
             _txRwLock.EnterWriteLock();
             try
             {
-                VerifyResult result = tx.Verify(snapshot, VerificationContext);
+                VerifyResult result = tx.VerifyStateDependent(snapshot, VerificationContext);
                 if (result != VerifyResult.Succeed) return result;
 
                 _unsortedTransactions.Add(tx.Hash, poolItem);
@@ -425,7 +425,7 @@ private int ReverifyTransactions(SortedSet<PoolItem> verifiedSortedTxPool,
                 // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end.
                 foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count))
                 {
-                    if (item.Tx.VerifyForEachBlock(snapshot, VerificationContext) == VerifyResult.Succeed)
+                    if (item.Tx.VerifyStateDependent(snapshot, VerificationContext) == VerifyResult.Succeed)
                     {
                         reverifiedItems.Add(item);
                         VerificationContext.AddTransaction(item.Tx);

src/neo/Ledger/TransactionRouter.cs

@@ -0,0 +1,35 @@
+using Akka.Actor;
+using Akka.Routing;
+using Neo.Network.P2P.Payloads;
+using System;
+
+namespace Neo.Ledger
+{
+    internal class TransactionRouter : UntypedActor
+    {
+        public class Task { public Transaction Transaction; public bool Relay; }
+
+        private readonly IActorRef blockchain;
+
+        public TransactionRouter(NeoSystem system)
+        {
+            this.blockchain = system.Blockchain;
+        }
+
+        protected override void OnReceive(object message)
+        {
+            if (!(message is Task task)) return;
+            blockchain.Tell(new Blockchain.PreverifyCompleted
+            {
+                Transaction = task.Transaction,
+                Result = task.Transaction.VerifyStateIndependent(),
+                Relay = task.Relay
+            }, Sender);
+        }
+
+        internal static Props Props(NeoSystem system)
+        {
+            return Akka.Actor.Props.Create(() => new TransactionRouter(system)).WithRouter(new SmallestMailboxPool(Environment.ProcessorCount));
+        }
+    }
+}

src/neo/Network/P2P/Payloads/Transaction.cs

@@ -281,7 +281,7 @@ bool IInventory.Verify(StoreView snapshot)
             return Verify(snapshot, null) == VerifyResult.Succeed;
         }
 
-        public virtual VerifyResult VerifyForEachBlock(StoreView snapshot, TransactionVerificationContext context)
+        public virtual VerifyResult VerifyStateDependent(StoreView snapshot, TransactionVerificationContext context)
         {
             if (ValidUntilBlock <= snapshot.Height || ValidUntilBlock > snapshot.Height + MaxValidUntilBlockIncrement)
                 return VerifyResult.Expired;
@@ -294,24 +294,27 @@ public virtual VerifyResult VerifyForEachBlock(StoreView snapshot, TransactionVe
             foreach (TransactionAttribute attribute in Attributes)
                 if (!attribute.Verify(snapshot, this))
                     return VerifyResult.Invalid;
-            if (hashes.Length != Witnesses.Length) return VerifyResult.Invalid;
-            for (int i = 0; i < hashes.Length; i++)
-            {
-                if (Witnesses[i].VerificationScript.Length > 0) continue;
-                if (snapshot.Contracts.TryGet(hashes[i]) is null) return VerifyResult.Invalid;
-            }
+            long net_fee = NetworkFee - Size * NativeContract.Policy.GetFeePerByte(snapshot);
+            if (!this.VerifyWitnesses(snapshot, net_fee, WitnessFlag.StateDependent))
+                return VerifyResult.Invalid;
+            return VerifyResult.Succeed;
+        }
+
+        public virtual VerifyResult VerifyStateIndependent()
+        {
+            if (Size > MaxTransactionSize)
+                return VerifyResult.Invalid;
+            if (!this.VerifyWitnesses(null, NetworkFee, WitnessFlag.StateIndependent))
+                return VerifyResult.Invalid;
             return VerifyResult.Succeed;
         }
 
         public virtual VerifyResult Verify(StoreView snapshot, TransactionVerificationContext context)
         {
-            VerifyResult result = VerifyForEachBlock(snapshot, context);
+            VerifyResult result = VerifyStateIndependent();
             if (result != VerifyResult.Succeed) return result;
-            if (Size > MaxTransactionSize) return VerifyResult.Invalid;
-            long net_fee = NetworkFee - Size * NativeContract.Policy.GetFeePerByte(snapshot);
-            if (net_fee < 0) return VerifyResult.InsufficientFunds;
-            if (!this.VerifyWitnesses(snapshot, net_fee)) return VerifyResult.Invalid;
-            return VerifyResult.Succeed;
+            result = VerifyStateDependent(snapshot, context);
+            return result;
         }
 
         public StackItem ToStackItem(ReferenceCounter referenceCounter)

src/neo/Network/P2P/Payloads/Witness.cs

@@ -11,6 +11,8 @@ public class Witness : ISerializable
         public byte[] InvocationScript;
         public byte[] VerificationScript;
 
+        internal long GasConsumed { get; set; }
+
         private UInt160 _scriptHash;
         public virtual UInt160 ScriptHash
         {
@@ -24,6 +26,8 @@ public virtual UInt160 ScriptHash
             }
         }
 
+        public bool StateDependent => VerificationScript.Length == 0;
+
         public int Size => InvocationScript.GetVarSize() + VerificationScript.GetVarSize();
 
         void ISerializable.Deserialize(BinaryReader reader)

src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs

@@ -286,13 +286,19 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload)
 
         private void OnInventoryReceived(IInventory inventory)
         {
-            system.TaskManager.Tell(inventory);
-            if (inventory is Transaction transaction)
-                system.Consensus?.Tell(transaction);
-            system.Blockchain.Tell(inventory, ActorRefs.NoSender);
             pendingKnownHashes.Remove(inventory.Hash);
             knownHashes.Add(inventory.Hash);
-            if (inventory is Block b) UpdateLastBlockIndex(b.Index, false);
+            system.TaskManager.Tell(inventory);
+            system.Blockchain.Tell(inventory, ActorRefs.NoSender);
+            switch (inventory)
+            {
+                case Transaction transaction:
+                    system.Consensus?.Tell(transaction);
+                    break;
+                case Block block:
+                    UpdateLastBlockIndex(block.Index, false);
+                    break;
+            }
         }
 
         private void OnInvMessageReceived(InvPayload payload)

src/neo/SmartContract/Helper.cs

@@ -129,7 +129,7 @@ public static UInt160 ToScriptHash(this ReadOnlySpan<byte> script)
             return new UInt160(Crypto.Hash160(script));
         }
 
-        internal static bool VerifyWitnesses(this IVerifiable verifiable, StoreView snapshot, long gas)
+        internal static bool VerifyWitnesses(this IVerifiable verifiable, StoreView snapshot, long gas, WitnessFlag filter = WitnessFlag.All)
         {
             if (gas < 0) return false;
             if (gas > MaxVerificationGas) gas = MaxVerificationGas;
@@ -146,6 +146,14 @@ internal static bool VerifyWitnesses(this IVerifiable verifiable, StoreView snap
             if (hashes.Length != verifiable.Witnesses.Length) return false;
             for (int i = 0; i < hashes.Length; i++)
             {
+                WitnessFlag flag = verifiable.Witnesses[i].StateDependent ? WitnessFlag.StateDependent : WitnessFlag.StateIndependent;
+                if (!filter.HasFlag(flag))
+                {
+                    gas -= verifiable.Witnesses[i].GasConsumed;
+                    if (gas < 0) return false;
+                    continue;
+                }
+
                 int offset;
                 ContractMethodDescriptor init = null;
                 byte[] verification = verifiable.Witnesses[i].VerificationScript;
@@ -164,14 +172,16 @@ internal static bool VerifyWitnesses(this IVerifiable verifiable, StoreView snap
                     if (hashes[i] != verifiable.Witnesses[i].ScriptHash) return false;
                     offset = 0;
                 }
-                using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.Verification, verifiable, snapshot.Clone(), gas))
+                using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.Verification, verifiable, snapshot?.Clone(), gas))
                 {
-                    ExecutionContext context = engine.LoadScript(verification, CallFlags.None, offset);
+                    CallFlags callFlags = verifiable.Witnesses[i].StateDependent ? CallFlags.AllowStates : CallFlags.None;
+                    ExecutionContext context = engine.LoadScript(verification, callFlags, offset);
                     if (init != null) engine.LoadContext(context.Clone(init.Offset), false);
                     engine.LoadScript(verifiable.Witnesses[i].InvocationScript, CallFlags.None);
                     if (engine.Execute() == VMState.FAULT) return false;
                     if (engine.ResultStack.Count != 1 || !engine.ResultStack.Pop().GetBoolean()) return false;
                     gas -= engine.GasConsumed;
+                    verifiable.Witnesses[i].GasConsumed = engine.GasConsumed;
                 }
             }
             return true;

src/neo/SmartContract/WitnessFlag.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace Neo.SmartContract
+{
+    [Flags]
+    internal enum WitnessFlag : byte
+    {
+        None = 0,
+
+        StateIndependent = 0b00000001,
+        StateDependent = 0b00000010,
+
+        All = StateIndependent | StateDependent
+    }
+}