Merge pull request #420 from longfin/feature/boost-ibd

Boost PreloadAsync()

CHANGES.md

@@ -8,6 +8,13 @@ To be released.
 
 ### Backward-incompatible interface changes
 
+ -  `IStore.StoreStateReference<T>(string, IImmutableSet<Address>, Block<T>)`
+    method became replaced by
+    `StoreStateReference(string, IImmutableSet<Address>, HashDigest<SHA256>, long)`
+    method so that it takes hash and index of a block instead of an entire
+    block.  [[#420]]
+ -  Added `IStore.ForkBlockIndexes()` method.  [[#420]]
+
 ### Added interfaces
 
 ### Behavioral changes
@@ -17,6 +24,7 @@ To be released.
  -  Fixed a bug that `Swarm<T>` hadn't released its TURN releated resources on
     `Swarm<T>.StopAsync()`.  [[#450]]
 
+[#420]: https://github.com/planetarium/libplanet/pull/420
 [#450]: https://github.com/planetarium/libplanet/pull/450
 
 

Libplanet.Tests/Blockchain/BlockChainTest.cs

@@ -1392,7 +1392,8 @@ void BuildIndex(Guid id, Block<DumbAction> block)
                     store.StoreStateReference(
                         chainId.ToString(),
                         dirty.Keys.ToImmutableHashSet(),
-                        b
+                        b.Hash,
+                        b.Index
                     );
                     BuildIndex(chainId, b);
                 }

Libplanet.Tests/Store/StoreExtensionTest.cs

@@ -47,7 +47,12 @@ public void LookupStateReference(StoreFixture fx)
             Assert.Null(fx.Store.LookupStateReference(fx.StoreNamespace, address, block5));
             Assert.Null(fx.Store.LookupStateReference(fx.StoreNamespace, address, block6));
 
-            fx.Store.StoreStateReference(fx.StoreNamespace, tx4.UpdatedAddresses, block4);
+            fx.Store.StoreStateReference(
+                fx.StoreNamespace,
+                tx4.UpdatedAddresses,
+                block4.Hash,
+                block4.Index
+            );
             Assert.Null(fx.Store.LookupStateReference(fx.StoreNamespace, address, fx.Block3));
             Assert.Equal(
                 Tuple.Create(block4.Hash, block4.Index),
@@ -62,7 +67,12 @@ public void LookupStateReference(StoreFixture fx)
                 fx.Store.LookupStateReference(fx.StoreNamespace, address, block6)
             );
 
-            fx.Store.StoreStateReference(fx.StoreNamespace, tx5.UpdatedAddresses, block5);
+            fx.Store.StoreStateReference(
+                fx.StoreNamespace,
+                tx5.UpdatedAddresses,
+                block5.Hash,
+                block5.Index
+            );
             Assert.Null(fx.Store.LookupStateReference(
                 fx.StoreNamespace, address, fx.Block3));
             Assert.Equal(

Libplanet.Tests/Store/StoreTest.cs

@@ -51,7 +51,8 @@ public void DeleteNamespace()
             Fx.Store.StoreStateReference(
                 Fx.StoreNamespace,
                 addresses.Take(3).ToImmutableHashSet(),
-                block1
+                block1.Hash,
+                block1.Index
             );
             Fx.Store.IncreaseTxNonce(Fx.StoreNamespace, Fx.Transaction1.Signer);
 
@@ -87,7 +88,8 @@ public void ListAddresses()
             Fx.Store.StoreStateReference(
                 Fx.StoreNamespace,
                 addresses.Take(3).ToImmutableHashSet(),
-                Fx.Block1
+                Fx.Block1.Hash,
+                Fx.Block1.Index
             );
             Assert.Equal(
                 addresses.Take(3).ToImmutableHashSet(),
@@ -96,7 +98,8 @@ public void ListAddresses()
             Fx.Store.StoreStateReference(
                 Fx.StoreNamespace,
                 addresses.Skip(2).Take(3).ToImmutableHashSet(),
-                Fx.Block2
+                Fx.Block2.Hash,
+                Fx.Block2.Index
             );
             Assert.Equal(
                 addresses.Take(5).ToImmutableHashSet(),
@@ -105,7 +108,8 @@ public void ListAddresses()
             Fx.Store.StoreStateReference(
                 Fx.StoreNamespace,
                 addresses.Skip(5).Take(3).ToImmutableHashSet(),
-                Fx.Block3
+                Fx.Block3.Hash,
+                Fx.Block3.Index
             );
             Assert.Equal(
                 addresses.ToImmutableHashSet(),
@@ -343,13 +347,23 @@ public void IterateStateReferences()
 
             Assert.Empty(this.Fx.Store.IterateStateReferences(this.Fx.StoreNamespace, address));
 
-            Fx.Store.StoreStateReference(Fx.StoreNamespace, tx4.UpdatedAddresses, block4);
+            Fx.Store.StoreStateReference(
+                Fx.StoreNamespace,
+                tx4.UpdatedAddresses,
+                block4.Hash,
+                block4.Index
+            );
             Assert.Equal(
                 new[] { Tuple.Create(block4.Hash, block4.Index) },
                 this.Fx.Store.IterateStateReferences(this.Fx.StoreNamespace, address)
             );
 
-            Fx.Store.StoreStateReference(Fx.StoreNamespace, tx5.UpdatedAddresses, block5);
+            Fx.Store.StoreStateReference(
+                Fx.StoreNamespace,
+                tx5.UpdatedAddresses,
+                block5.Hash,
+                block5.Index
+            );
             Assert.Equal(
                 new[]
                 {
@@ -392,15 +406,23 @@ public void ForkStateReferences(int branchPointIndex)
             {
                 updatedAddresses = new HashSet<Address> { address1 };
                 Fx.Store.StoreStateReference(
-                    Fx.StoreNamespace, updatedAddresses.ToImmutableHashSet(), block);
+                    Fx.StoreNamespace,
+                    updatedAddresses.ToImmutableHashSet(),
+                    block.Hash,
+                    block.Index
+                );
             }
 
             var txs2 = new[] { tx2 };
             blocks.Add(TestUtils.MineNext(blocks[2], txs2));
 
             updatedAddresses = new HashSet<Address> { address2 };
             Fx.Store.StoreStateReference(
-                Fx.StoreNamespace, updatedAddresses.ToImmutableHashSet(), blocks[3]);
+                Fx.StoreNamespace,
+                updatedAddresses.ToImmutableHashSet(),
+                blocks[3].Hash,
+                blocks[3].Index
+            );
 
             var branchPoint = blocks[branchPointIndex];
             var addressesToStrip = new[] { address1, address2 }.ToImmutableHashSet();

Libplanet.Tests/Store/StoreTracker.cs

@@ -176,15 +176,15 @@ public IEnumerable<Tuple<HashDigest<SHA256>, long>> IterateStateReferences(
             return _store.IterateStateReferences(@namespace, address);
         }
 
-        public void StoreStateReference<T>(
+        public void StoreStateReference(
             string @namespace,
             IImmutableSet<Address> addresses,
-            Block<T> block)
-            where T : IAction, new()
+            HashDigest<SHA256> blockHash,
+            long blockIndex)
         {
             // FIXME: Log arguments properly (including @namespace).
-            _logs.Add((nameof(StoreStateReference), block.Hash, null));
-            _store.StoreStateReference(@namespace, addresses, block);
+            _logs.Add((nameof(StoreStateReference), blockHash, null));
+            _store.StoreStateReference(@namespace, addresses, blockHash, blockIndex);
         }
 
         public void ForkStateReferences<T>(
@@ -200,6 +200,15 @@ public void ForkStateReferences<T>(
                 sourceNamespace, destinationNamespace, branchPoint, addressesToStrip);
         }
 
+        public void ForkBlockIndexes(
+            string sourceNamespace,
+            string destinationNamespace,
+            HashDigest<SHA256> branchPoint)
+        {
+            _logs.Add((nameof(ForkBlockIndexes), null, null));
+            _store.ForkBlockIndexes(sourceNamespace, destinationNamespace, branchPoint);
+        }
+
         public IEnumerable<KeyValuePair<Address, long>> ListTxNonces(string @namespace)
         {
             _logs.Add((nameof(ListTxNonces), @namespace, null));

Libplanet/Blockchain/BlockChain.cs

@@ -815,15 +815,8 @@ internal BlockChain<T> Fork(HashDigest<SHA256> point)
             try
             {
                 _rwlock.EnterReadLock();
-                foreach (var index in Store.IterateIndex(id))
-                {
-                    Store.AppendIndex(forkedId, index);
-                    if (index.Equals(point))
-                    {
-                        break;
-                    }
-                }
 
+                Store.ForkBlockIndexes(id, forkedId, point);
                 Block<T> pointBlock = Blocks[point];
 
                 var addressesToStrip = new HashSet<Address>();
@@ -1080,7 +1073,12 @@ bool buildStateReferences
 
             if (buildStateReferences)
             {
-                Store.StoreStateReference(Id.ToString(), updatedAddresses, block);
+                Store.StoreStateReference(
+                    Id.ToString(),
+                    updatedAddresses,
+                    block.Hash,
+                    block.Index
+                );
             }
         }
     }

Libplanet/Net/StateReferenceDownloadState.cs

@@ -17,11 +17,6 @@ public class StateReferenceDownloadState : PreloadState
         /// </summary>
         public int ReceivedStateReferenceCount { get; internal set; }
 
-        /// <summary>
-        /// The address of the state references just received.
-        /// </summary>
-        public Address ReceivedAddress { get; internal set; }
-
         /// <inheritdoc />
         public override int CurrentPhase => 2;
     }

Libplanet/Net/Swarm.cs

@@ -1042,21 +1042,37 @@ await socket.SendMultipartMessageAsync(
 
                             int count = 0, totalCount = recentStates.StateReferences.Count;
                             _logger.Debug("Starts to store state refs received from {0}.", peer);
+
+                            var d = new Dictionary<HashDigest<SHA256>, ISet<Address>>();
                             foreach (var pair in recentStates.StateReferences)
                             {
                                 cancellationToken.ThrowIfCancellationRequested();
                                 IImmutableSet<Address> address = ImmutableHashSet.Create(pair.Key);
                                 foreach (HashDigest<SHA256> bHash in pair.Value)
                                 {
-                                    Block<T> block = store.GetBlock<T>(bHash);
-                                    store.StoreStateReference(ns, address, block);
+                                    if (!d.ContainsKey(bHash))
+                                    {
+                                        d[bHash] = new HashSet<Address>();
+                                    }
+
+                                    d[bHash].UnionWith(address);
+                                }
+                            }
+
+                            totalCount = d.Count;
+                            foreach (KeyValuePair<HashDigest<SHA256>, ISet<Address>> pair in d)
+                            {
+                                HashDigest<SHA256> hash = pair.Key;
+                                IImmutableSet<Address> addresses = pair.Value.ToImmutableHashSet();
+                                if (store.GetBlockIndex(hash) is long index)
+                                {
+                                    store.StoreStateReference(ns, addresses, hash, index);
                                 }
 
                                 progress?.Report(new StateReferenceDownloadState()
                                 {
                                     TotalStateReferenceCount = totalCount,
                                     ReceivedStateReferenceCount = ++count,
-                                    ReceivedAddress = pair.Key,
                                 });
                             }
 

Libplanet/Store/BaseStore.cs

@@ -38,6 +38,12 @@ public abstract long AppendIndex(
             HashDigest<SHA256> hash
         );
 
+        /// <inheritdoc />
+        public abstract void ForkBlockIndexes(
+            string sourceNamespace,
+            string destinationNamespace,
+            HashDigest<SHA256> branchPoint);
+
         /// <inheritdoc />
         public abstract IEnumerable<Address> ListAddresses(string @namespace);
 
@@ -119,12 +125,11 @@ AddressStateMap states
         public abstract IEnumerable<Tuple<HashDigest<SHA256>, long>> IterateStateReferences(
             string @namespace, Address address);
 
-        /// <inheritdoc />
-        public abstract void StoreStateReference<T>(
+        public abstract void StoreStateReference(
             string @namespace,
             IImmutableSet<Address> addresses,
-            Block<T> block)
-            where T : IAction, new();
+            HashDigest<SHA256> hashDigest,
+            long index);
 
         /// <inheritdoc />
         public abstract void ForkStateReferences<T>(

Libplanet/Store/IStore.cs

@@ -53,6 +53,26 @@ IEnumerable<HashDigest<SHA256>> IterateIndex(
 
         bool DeleteIndex(string @namespace, HashDigest<SHA256> hash);
 
+        /// <summary>
+        /// Forks block indexes from
+        /// <paramref name="sourceNamespace"/> to
+        /// <paramref name="destinationNamespace"/>.
+        /// </summary>
+        /// <param name="sourceNamespace">The namespace of block indexes to
+        /// fork.</param>
+        /// <param name="destinationNamespace">The namespace of destination
+        /// block indexes.</param>
+        /// <param name="branchPoint">The branch point <see cref="Block{T}"/>
+        /// to fork.</param>
+        /// <exception cref="NamespaceNotFoundException">Thrown when the given
+        /// <paramref name="sourceNamespace"/> does not exist.</exception>
+        /// <seealso cref="IterateIndex(string, int, int?)"/>
+        /// <seealso cref="AppendIndex(string, HashDigest{SHA256})"/>
+        void ForkBlockIndexes(
+            string sourceNamespace,
+            string destinationNamespace,
+            HashDigest<SHA256> branchPoint);
+
         /// <summary>
         /// Deletes an index, tx nonces, and state references in the given
         /// <paramref name="namespace"/>.
@@ -175,6 +195,7 @@ void SetBlockStates(
             AddressStateMap states
         );
 
+        #pragma warning disable MEN002
         /// <summary>
         /// Gets pairs of a state reference and a corresponding <see cref="Block{T}.Index"/> of
         /// the requested <paramref name="address"/> in the specified <paramref name="namespace"/>.
@@ -184,7 +205,8 @@ AddressStateMap states
         /// <returns><em>Ordered</em> pairs of a state reference and a corresponding
         /// <see cref="Block{T}.Index"/>.  The highest index (i.e., the closest to the tip) goes
         /// first and the lowest index (i.e., the closest to the genesis) goes last.</returns>
-        /// <seealso cref="StoreStateReference{T}(string, IImmutableSet{Address}, Block{T})"/>
+        /// <seealso cref="StoreStateReference(string, IImmutableSet{Address}, HashDigest{SHA256}, long)"/>
+        #pragma warning restore MEN002 // Line is too long
         IEnumerable<Tuple<HashDigest<SHA256>, long>> IterateStateReferences(
             string @namespace,
             Address address);
@@ -193,25 +215,24 @@ IEnumerable<Tuple<HashDigest<SHA256>, long>> IterateStateReferences(
         /// Stores a state reference, which is a <see cref="Block{T}.Hash"/>
         /// that has the state of the <see cref="Address"/> for each updated
         /// <see cref="Address"/>es by the <see cref="Transaction{T}"/>s in the
-        /// <paramref name="block"/>.</summary>
+        /// <see cref="Block{T}" />.</summary>
         /// <param name="namespace">The namespace to store a state reference.
         /// </param>
         /// <param name="addresses">The <see cref="Address"/>es to store state
         /// reference.</param>
-        /// <param name="block">The <see cref="Block{T}"/> which has the state
+        /// <param name="blockHash">The <see cref="Block{T}.Hash"/> which has the state
         /// of the <see cref="Address"/>.</param>
-        /// <typeparam name="T">An <see cref="IAction"/> class used with
-        /// <paramref name="block"/>.</typeparam>
-        /// <remarks>State reference must be stored in the same order as the blocks. For now,
-        /// it is assumed that this is only called by
-        /// <see cref="BlockChain{T}.Append(Block{T})"/> method.</remarks>
+        /// <param name="blockIndex">The <see cref="Block{T}.Index"/> which has the state
+        /// of the <see cref="Address"/>. This must refer to the same block
+        /// that <paramref name="blockHash"/> refers to.</param>
         /// <seealso cref="IterateStateReferences(string, Address)"/>
-        void StoreStateReference<T>(
+        void StoreStateReference(
             string @namespace,
             IImmutableSet<Address> addresses,
-            Block<T> block)
-            where T : IAction, new();
+            HashDigest<SHA256> blockHash,
+            long blockIndex);
 
+        #pragma warning disable MEN002
         /// <summary>
         /// Forks state references, which are <see cref="Block{T}.Hash"/>es that
         /// have the state of the <see cref="Address"/>es, from
@@ -236,7 +257,8 @@ void StoreStateReference<T>(
         /// <exception cref="NamespaceNotFoundException">Thrown when the given
         /// <paramref name="sourceNamespace"/> does not exist.</exception>
         /// <seealso cref="IterateStateReferences(string, Address)"/>
-        /// <seealso cref="StoreStateReference{T}(string, IImmutableSet{Address}, Block{T})"/>
+        /// <seealso cref="StoreStateReference(string, IImmutableSet{Address}, HashDigest{SHA256}, long)"/>
+        #pragma warning restore MEN002
         void ForkStateReferences<T>(
             string sourceNamespace,
             string destinationNamespace,