Feature/javascript tracers

scripts/deployment/build-runner.sh

@@ -22,7 +22,6 @@ do
     -p:IncludeAllContentForSelfExtract=true \
     -p:PublishSingleFile=true
 
-  cp -r configs $output_path/$rid
   mkdir $output_path/$rid/keystore
 
   # A temporary symlink for Linux and macOS to support existing scripts if any

src/Nethermind/Directory.Packages.props

@@ -32,6 +32,12 @@
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="7.0.9" />
     <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
     <PackageVersion Include="Microsoft.AspNetCore.WebSockets" Version="2.2.1" />
+    <PackageVersion Include="Microsoft.ClearScript.V8" Version="7.4.3" />
+    <PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-arm64" Version="7.4.3" />
+    <PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-x64" Version="7.4.3" />
+    <PackageVersion Include="Microsoft.ClearScript.V8.Native.osx-arm64" Version="7.4.3" />
+    <PackageVersion Include="Microsoft.ClearScript.V8.Native.osx-x64" Version="7.4.3" />
+    <PackageVersion Include="Microsoft.ClearScript.V8.Native.win-x64" Version="7.4.3" />
     <PackageVersion Include="Microsoft.Extensions.CommandLineUtils" Version="1.1.1" />
     <PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="7.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="7.0.0" />

src/Nethermind/Ethereum.Blockchain.Test/MetaTests.cs

@@ -21,7 +21,8 @@ public class MetaTests
             "runtimes",
             "ref",
             "TestFiles",
-            "Blockhash"
+            "Blockhash",
+            "Data"
         };
 
         [Test]

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -314,12 +314,10 @@ private void ApplyMinerRewards(Block block, IBlockTracer tracer, IReleaseSpec sp
         {
             BlockReward reward = rewards[i];
 
-            ITxTracer txTracer = NullTxTracer.Instance;
-            if (tracer.IsTracingRewards)
-            {
-                // we need this tracer to be able to track any potential miner account creation
-                txTracer = tracer.StartNewTxTrace(null);
-            }
+            using ITxTracer txTracer = tracer.IsTracingRewards
+                ? // we need this tracer to be able to track any potential miner account creation
+                tracer.StartNewTxTrace(null)
+                : NullTxTracer.Instance;
 
             ApplyMinerReward(block, reward, spec);
 

src/Nethermind/Nethermind.Consensus/Processing/TransactionProcessorAdapterExtensions.cs

@@ -20,10 +20,10 @@ public static void ProcessTransaction(this ITransactionProcessorAdapter transact
     {
         if (processingOptions.ContainsFlag(ProcessingOptions.DoNotVerifyNonce))
         {
-            currentTx.Nonce = stateProvider.GetNonce(currentTx.SenderAddress);
+            currentTx.Nonce = stateProvider.GetNonce(currentTx.SenderAddress!);
         }
 
-        receiptsTracer.StartNewTxTrace(currentTx);
+        using ITxTracer tracer = receiptsTracer.StartNewTxTrace(currentTx);
         transactionProcessor.Execute(currentTx, blkCtx, receiptsTracer);
         receiptsTracer.EndTxTrace();
     }

src/Nethermind/Nethermind.Consensus/Tracing/GethStyleTracer.cs

@@ -12,32 +12,40 @@
 using Nethermind.Consensus.Processing;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
+using Nethermind.Core.Specs;
 using Nethermind.Crypto;
 using Nethermind.Evm.Tracing;
 using Nethermind.Evm.Tracing.GethStyle;
+using Nethermind.Evm.Tracing.GethStyle.JavaScript;
 using Nethermind.Evm.TransactionProcessing;
 using Nethermind.Serialization.Rlp;
+using Nethermind.State;
 
 namespace Nethermind.Consensus.Tracing;
 
 public class GethStyleTracer : IGethStyleTracer
 {
     private readonly IBlockTree _blockTree;
+    private readonly ISpecProvider _specProvider;
     private readonly ChangeableTransactionProcessorAdapter _transactionProcessorAdapter;
     private readonly IBlockchainProcessor _processor;
+    private readonly IWorldState _worldState;
     private readonly IReceiptStorage _receiptStorage;
     private readonly IFileSystem _fileSystem;
 
-    public GethStyleTracer(
-        IBlockchainProcessor processor,
+    public GethStyleTracer(IBlockchainProcessor processor,
+        IWorldState worldState,
         IReceiptStorage receiptStorage,
         IBlockTree blockTree,
+        ISpecProvider specProvider,
         ChangeableTransactionProcessorAdapter transactionProcessorAdapter,
         IFileSystem fileSystem)
     {
         _processor = processor ?? throw new ArgumentNullException(nameof(processor));
+        _worldState = worldState;
         _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage));
         _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree));
+        _specProvider = specProvider;
         _transactionProcessorAdapter = transactionProcessorAdapter;
         _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
     }
@@ -110,18 +118,19 @@ public GethLikeTxTrace Trace(Hash256 blockHash, int txIndex, GethTraceOptions op
         if (tx.Hash is null) throw new InvalidOperationException("Cannot trace transactions without tx hash set.");
 
         block = block.WithReplacedBodyCloned(BlockBody.WithOneTransactionOnly(tx));
-        GethLikeBlockMemoryTracer blockTracer = new(options with { TxHash = tx.Hash });
+        IBlockTracer<GethLikeTxTrace> blockTracer = CreateOptionsTracer(block.Header, options with { TxHash = tx.Hash });
         _processor.Process(block, ProcessingOptions.Trace, blockTracer.WithCancellation(cancellationToken));
         return blockTracer.BuildResult().SingleOrDefault();
     }
-    public GethLikeTxTrace[] TraceBlock(BlockParameter blockParameter, GethTraceOptions options, CancellationToken cancellationToken)
+
+    public IReadOnlyCollection<GethLikeTxTrace> TraceBlock(BlockParameter blockParameter, GethTraceOptions options, CancellationToken cancellationToken)
     {
         var block = _blockTree.FindBlock(blockParameter);
 
         return TraceBlock(block, options, cancellationToken);
     }
 
-    public GethLikeTxTrace[] TraceBlock(Rlp blockRlp, GethTraceOptions options, CancellationToken cancellationToken)
+    public IReadOnlyCollection<GethLikeTxTrace> TraceBlock(Rlp blockRlp, GethTraceOptions options, CancellationToken cancellationToken)
     {
         return TraceBlock(GetBlockToTrace(blockRlp), options, cancellationToken);
     }
@@ -152,14 +161,19 @@ public IEnumerable<string> TraceBlockToFile(Hash256 blockHash, GethTraceOptions
     {
         ArgumentNullException.ThrowIfNull(txHash);
 
-        var tracer = new GethLikeBlockMemoryTracer(options with { TxHash = txHash });
+        IBlockTracer<GethLikeTxTrace> tracer = CreateOptionsTracer(block.Header, options with { TxHash = txHash });
 
         _processor.Process(block, ProcessingOptions.Trace, tracer.WithCancellation(cancellationToken));
 
         return tracer.BuildResult().SingleOrDefault();
     }
 
-    private GethLikeTxTrace[] TraceBlock(Block? block, GethTraceOptions options, CancellationToken cancellationToken)
+    private IBlockTracer<GethLikeTxTrace> CreateOptionsTracer(BlockHeader block, GethTraceOptions options) =>
+        !string.IsNullOrEmpty(options.Tracer)
+            ? new GethLikeBlockJavaScriptTracer(_worldState, _specProvider.GetSpec(block), options)
+            : new GethLikeBlockMemoryTracer(options);
+
+    private IReadOnlyCollection<GethLikeTxTrace> TraceBlock(Block? block, GethTraceOptions options, CancellationToken cancellationToken)
     {
         ArgumentNullException.ThrowIfNull(block);
 
@@ -174,9 +188,9 @@ private GethLikeTxTrace[] TraceBlock(Block? block, GethTraceOptions options, Can
             if (!_blockTree.IsMainChain(parent.Hash)) throw new InvalidOperationException("Cannot trace orphaned blocks");
         }
 
-        GethLikeBlockMemoryTracer tracer = new(options);
+        IBlockTracer<GethLikeTxTrace> tracer = CreateOptionsTracer(block.Header, options);
         _processor.Process(block, ProcessingOptions.Trace, tracer.WithCancellation(cancellationToken));
-        return tracer.BuildResult().ToArray();
+        return tracer.BuildResult();
     }
 
     private static Block GetBlockToTrace(Rlp blockRlp)

src/Nethermind/Nethermind.Consensus/Tracing/IGethStyleTracer.cs

@@ -19,7 +19,7 @@ public interface IGethStyleTracer
     GethLikeTxTrace Trace(Hash256 blockHash, int txIndex, GethTraceOptions options, CancellationToken cancellationToken);
     GethLikeTxTrace Trace(Rlp blockRlp, Hash256 txHash, GethTraceOptions options, CancellationToken cancellationToken);
     GethLikeTxTrace? Trace(BlockParameter blockParameter, Transaction tx, GethTraceOptions options, CancellationToken cancellationToken);
-    GethLikeTxTrace[] TraceBlock(BlockParameter blockParameter, GethTraceOptions options, CancellationToken cancellationToken);
-    GethLikeTxTrace[] TraceBlock(Rlp blockRlp, GethTraceOptions options, CancellationToken cancellationToken);
+    IReadOnlyCollection<GethLikeTxTrace> TraceBlock(BlockParameter blockParameter, GethTraceOptions options, CancellationToken cancellationToken);
+    IReadOnlyCollection<GethLikeTxTrace> TraceBlock(Rlp blockRlp, GethTraceOptions options, CancellationToken cancellationToken);
     IEnumerable<string> TraceBlockToFile(Hash256 blockHash, GethTraceOptions options, CancellationToken cancellationToken);
 }

src/Nethermind/Nethermind.Core.Test/Json/UInt256ConverterTests.cs

@@ -16,6 +16,7 @@ public class UInt256ConverterTests : ConverterTestBase<UInt256>
     {
         [TestCase(NumberConversion.Hex)]
         [TestCase(NumberConversion.Decimal)]
+        [TestCase(NumberConversion.Raw)]
         public void Test_roundtrip(NumberConversion numberConversion)
         {
             UInt256Converter converter = new(numberConversion);
@@ -25,8 +26,7 @@ public void Test_roundtrip(NumberConversion numberConversion)
         }
 
         [TestCase((NumberConversion)99)]
-        [TestCase(NumberConversion.Raw)]
-        public void Raw_not_supported(NumberConversion notSupportedConversion)
+        public void Undefined_not_supported(NumberConversion notSupportedConversion)
         {
             UInt256Converter converter = new(notSupportedConversion);
             Assert.Throws<NotSupportedException>(

src/Nethermind/Nethermind.Core/Address.cs

@@ -61,14 +61,17 @@ public static bool IsValidAddress(string hexString, bool allowPrefix)
 
         public Address(string hexString) : this(Extensions.Bytes.FromHexString(hexString)) { }
 
+        /// <summary>
+        /// Parses string value to Address. String has to be exactly 20 bytes long.
+        /// </summary>
         public static bool TryParse(string? value, out Address? address)
         {
             if (value is not null)
             {
                 try
                 {
                     byte[] bytes = Extensions.Bytes.FromHexString(value);
-                    if (bytes?.Length == Size)
+                    if (bytes.Length == Size)
                     {
                         address = new Address(bytes);
                         return true;
@@ -81,6 +84,25 @@ public static bool TryParse(string? value, out Address? address)
             return false;
         }
 
+        /// <summary>
+        /// Parses string value to Address. String can be shorter than 20 bytes long, it is padded with leading 0's then.
+        /// </summary>
+        public static bool TryParseVariableLength(string? value, out Address? address)
+        {
+            if (value is not null)
+            {
+                try
+                {
+                    address = new Address(Extensions.Bytes.FromHexString(value, Size));
+                    return true;
+                }
+                catch (IndexOutOfRangeException) { }
+            }
+
+            address = default;
+            return false;
+        }
+
         public Address(byte[] bytes)
         {
             if (bytes is null)

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -685,24 +685,24 @@ private static string ByteArrayToHexViaLookup32(byte[] bytes, bool withZeroX, bo
             return withEip55Checksum
                 ? ByteArrayToHexViaLookup32Checksum(length, stateToPass)
                 : string.Create(length, stateToPass, static (chars, state) =>
-            {
-                int skip = state.LeadingZeros / 2;
-                byte[] bytes = state.Bytes;
-                if (bytes.Length == 0)
                 {
-                    if (state.WithZeroX)
+                    int skip = state.LeadingZeros / 2;
+                    byte[] bytes = state.Bytes;
+                    if (bytes.Length == 0)
                     {
-                        chars[1] = 'x';
-                        chars[0] = '0';
-                    }
+                        if (state.WithZeroX)
+                        {
+                            chars[1] = 'x';
+                            chars[0] = '0';
+                        }
 
-                    return;
-                }
+                        return;
+                    }
 
-                ref byte input = ref Unsafe.Add(ref bytes[0], skip);
-                ref char charsRef = ref MemoryMarshal.GetReference(chars);
-                OutputBytesToCharHex(ref input, state.Bytes.Length, ref charsRef, state.WithZeroX, state.LeadingZeros);
-            });
+                    ref byte input = ref Unsafe.Add(ref bytes[0], skip);
+                    ref char charsRef = ref MemoryMarshal.GetReference(chars);
+                    OutputBytesToCharHex(ref input, state.Bytes.Length, ref charsRef, state.WithZeroX, state.LeadingZeros);
+                });
         }
 
         internal static void OutputBytesToCharHex(ref byte input, int length, ref char charsRef, bool withZeroX, int leadingZeros)
@@ -737,9 +737,9 @@ internal static void OutputBytesToCharHex(ref byte input, int length, ref char c
                     0xFF, 0xFF, 2, 0xFF, 0xFF, 0xFF, 3, 0xFF);
 
                 Vector128<byte> asciiTable = Vector128.Create((byte)'0', (byte)'1', (byte)'2', (byte)'3',
-                                     (byte)'4', (byte)'5', (byte)'6', (byte)'7',
-                                     (byte)'8', (byte)'9', (byte)'a', (byte)'b',
-                                     (byte)'c', (byte)'d', (byte)'e', (byte)'f');
+                    (byte)'4', (byte)'5', (byte)'6', (byte)'7',
+                    (byte)'8', (byte)'9', (byte)'a', (byte)'b',
+                    (byte)'c', (byte)'d', (byte)'e', (byte)'f');
 
                 nuint pos = 0;
                 Debug.Assert(toProcess >= 4);
@@ -768,6 +768,7 @@ static Vector128<byte> Shuffle(Vector128<byte> value, Vector128<byte> mask)
                         {
                             ThrowHelper.ThrowNotSupportedException();
                         }
+
                         return AdvSimd.Arm64.VectorTableLookup(value, mask);
                     }
 
@@ -777,9 +778,7 @@ static Vector128<byte> Shuffle(Vector128<byte> value, Vector128<byte> mask)
 
                     // ExtractVector128 is not entirely the same as ShiftRightLogical128BitLane, but it works here since
                     // first two bytes in lowNibbles are guaranteed to be zeros
-                    Vector128<byte> shifted = Sse2.IsSupported ?
-                        Sse2.ShiftRightLogical128BitLane(lowNibbles, 2) :
-                        AdvSimd.ExtractVector128(lowNibbles, lowNibbles, 2);
+                    Vector128<byte> shifted = Sse2.IsSupported ? Sse2.ShiftRightLogical128BitLane(lowNibbles, 2) : AdvSimd.ExtractVector128(lowNibbles, lowNibbles, 2);
 
                     Vector128<byte> highNibbles = Vector128.ShiftRightLogical(shifted.AsInt32(), 4).AsByte();
 
@@ -803,7 +802,6 @@ static Vector128<byte> Shuffle(Vector128<byte> value, Vector128<byte> mask)
                     {
                         pos = lengthSubVector128;
                     }
-
                 } while (true);
             }
             else
@@ -934,23 +932,58 @@ public static byte[] FromUtf8HexString(ReadOnlySpan<byte> hexString)
         }
 
         [DebuggerStepThrough]
-        public static byte[] FromHexString(string hexString)
+        public static byte[] FromHexString(string hexString, int length) =>
+            hexString is null ? throw new ArgumentNullException(nameof(hexString)) : FromHexString(hexString.AsSpan(), length);
+
+        [DebuggerStepThrough]
+        private static byte[] FromHexString(ReadOnlySpan<char> hexString, int length)
         {
-            if (hexString is null)
+            int start = hexString is ['0', 'x', ..] ? 2 : 0;
+            ReadOnlySpan<char> chars = hexString.Slice(start);
+
+            if (chars.Length == 0)
             {
-                throw new ArgumentNullException(nameof(hexString));
+                return Array.Empty<byte>();
             }
 
+            int oddMod = hexString.Length % 2;
+            int actualLength = (chars.Length >> 1) + oddMod;
+            byte[] result = GC.AllocateArray<byte>(length);
+            Span<byte> writeToSpan = result.AsSpan(length - actualLength);
+
+            bool isSuccess;
+            if (oddMod == 0 &&
+                BitConverter.IsLittleEndian && (Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) &&
+                chars.Length >= Vector128<ushort>.Count * 2)
+            {
+                isSuccess = HexConverter.TryDecodeFromUtf16_Vector128(chars, writeToSpan);
+            }
+            else
+            {
+                isSuccess = HexConverter.TryDecodeFromUtf16(chars, writeToSpan, oddMod == 1);
+            }
+
+            return isSuccess ? result : throw new FormatException("Incorrect hex string");
+        }
+
+        [DebuggerStepThrough]
+        public static byte[] FromHexString(string hexString) =>
+            hexString is null ? throw new ArgumentNullException(nameof(hexString)) : FromHexString(hexString.AsSpan());
+
+        [DebuggerStepThrough]
+        private static byte[] FromHexString(ReadOnlySpan<char> hexString)
+        {
             int start = hexString is ['0', 'x', ..] ? 2 : 0;
-            ReadOnlySpan<char> chars = hexString.AsSpan(start);
+            ReadOnlySpan<char> chars = hexString.Slice(start);
 
             if (chars.Length == 0)
             {
                 return Array.Empty<byte>();
             }
 
             int oddMod = hexString.Length % 2;
-            byte[] result = GC.AllocateUninitializedArray<byte>((chars.Length >> 1) + oddMod);
+            int actualLength = (chars.Length >> 1) + oddMod;
+            byte[] result = GC.AllocateUninitializedArray<byte>(actualLength);
 
             bool isSuccess;
             if (oddMod == 0 &&

src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs

@@ -60,6 +60,11 @@ private static string ToHexViaLookup(ReadOnlySpan<byte> bytes, bool withZeroX, b
             int leadingZeros = skipLeadingZeros ? Bytes.CountLeadingZeros(bytes) : 0;
             int length = bytes.Length * 2 + (withZeroX ? 2 : 0) - leadingZeros;
 
+            if (skipLeadingZeros && length == (withZeroX ? 2 : 0))
+            {
+                return withZeroX ? "0x0" : "0";
+            }
+
             char[] charArray = ArrayPool<char>.Shared.Rent(length);
 
             ref byte input = ref Unsafe.Add(ref MemoryMarshal.GetReference(bytes), leadingZeros / 2);