Merge branch 'master' into call-fixes-precompiles

GeneralStateTests.md

@@ -320,7 +320,7 @@ OK: 0/58 Fail: 15/58 Skip: 43/58
 OK: 0/58 Fail: 58/58 Skip: 0/58
 ## stChangedEIP150
 ```diff
- Call1024BalanceTooLow.json Skip
+- Call1024BalanceTooLow.json Fail
 - Call1024PreCalls.json Fail
 - Callcode1024BalanceTooLow.json Fail
 + callcall_00_OOGE_1.json OK
@@ -351,12 +351,12 @@ OK: 0/58 Fail: 58/58 Skip: 0/58
 + contractCreationMakeCallThatAskMoreGasThenTransactionProvided.jsonOK
 - createInitFail_OOGduringInit.json Fail
 ```
-OK: 4/30 Fail: 24/30 Skip: 2/30
+OK: 4/30 Fail: 25/30 Skip: 1/30
 ## stCodeCopyTest
 ```diff
- ExtCodeCopyTests.json Skip
+- ExtCodeCopyTests.json Fail
 ```
-OK: 0/1 Fail: 0/1 Skip: 1/1
+OK: 0/1 Fail: 1/1 Skip: 0/1
 ## stCodeSizeLimit
 ```diff
 - codesizeInit.json Fail
@@ -400,7 +400,7 @@ OK: 2/3 Fail: 1/3 Skip: 0/3
 OK: 0/30 Fail: 22/30 Skip: 8/30
 ## stDelegatecallTestHomestead
 ```diff
- Call1024BalanceTooLow.json Skip
+- Call1024BalanceTooLow.json Fail
 - Call1024OOG.json Fail
 - Call1024PreCalls.json Fail
 - CallLoseGasOOG.json Fail

nimbus/config.nim

@@ -7,8 +7,9 @@
 # This file may not be copied, modified, or distributed except according to
 # those terms.
 
-import parseopt, strutils, macros, os
-import asyncdispatch2, eth_keys, eth_p2p, eth_common, chronicles
+import
+ parseopt, strutils, macros, os,
+ asyncdispatch2, eth_keys, eth_p2p, eth_common, chronicles, nimcrypto/hash
 
 const
  NimbusName* = "Nimbus"

nimbus/constants.nim

@@ -1,6 +1,5 @@
-
 import
- math, strutils, eth_common
+ math, strutils, eth_common, nimcrypto/hash
 
 proc default(t: typedesc): t = discard
 

nimbus/db/backends/caching_backend.nim

@@ -1,5 +1,7 @@
-import ranges, eth_trie, tables, sets
-import ../storage_types
+import
+ ranges, tables, sets,
+ eth_trie/db,
+ ../storage_types
 
 type
  CachingDB* = ref object of RootObj

nimbus/db/backends/rocksdb_backend.nim

@@ -1,4 +1,4 @@
-import os, rocksdb, ranges, eth_trie/db_tracing
+import os, rocksdb, ranges, eth_trie/[defs, db_tracing]
 import ../storage_types
 
 type
@@ -7,18 +7,6 @@ type
 
  ChainDB* = RocksChainDB
 
-proc newChainDB*(basePath: string): ChainDB =
- result.new()
- let
- dataDir = basePath / "data"
- backupsDir = basePath / "backups"
-
- createDir(dataDir)
- createDir(backupsDir)
-
- let s = result.store.init(dataDir, backupsDir)
- if not s.ok: raiseStorageInitError()
-
 proc get*(db: ChainDB, key: openarray[byte]): seq[byte] =
  let s = db.store.getBytes(key)
  if s.ok:
@@ -46,3 +34,18 @@ proc del*(db: ChainDB, key: openarray[byte]) =
 
 proc close*(db: ChainDB) =
  db.store.close
+
+proc newChainDB*(basePath: string): ChainDB =
+ result.new()
+ let
+ dataDir = basePath / "data"
+ backupsDir = basePath / "backups"
+
+ createDir(dataDir)
+ createDir(backupsDir)
+
+ let s = result.store.init(dataDir, backupsDir)
+ if not s.ok: raiseStorageInitError()
+
+ put(result, emptyRlpHash.data, emptyRlp)
+

nimbus/db/backends/sqlite_backend.nim

@@ -1,5 +1,5 @@
 import
- os, sqlite3, ranges, ranges/ptr_arith, eth_trie/db_tracing,
+ os, sqlite3, ranges, ranges/ptr_arith, eth_trie/[db_tracing, defs],
  ../storage_types
 
 type
@@ -9,6 +9,8 @@ type
 
  ChainDB* = SqliteChainDB
 
+proc put*(db: ChainDB, key, value: openarray[byte])
+
 proc newChainDB*(basePath: string, inMemory = false): ChainDB =
  result.new()
  let dbPath = if inMemory: ":memory:" else: basePath / "nimbus.db"
@@ -54,6 +56,8 @@ proc newChainDB*(basePath: string, inMemory = false): ChainDB =
 
  result.deleteStmt = prepare "DELETE FROM trie_nodes WHERE key = ?;"
 
+ put(result, emptyRlpHash.data, emptyRlp)
+
 proc bindBlob(s: Pstmt, n: int, blob: openarray[byte]): int32 =
  sqlite3.bind_blob(s, n.int32, blob.baseAddr, blob.len.int32, nil)
 

nimbus/db/db_chain.nim

@@ -7,7 +7,7 @@
 
 import
  tables, sequtils, algorithm,
- rlp, ranges, state_db, nimcrypto, eth_trie/[types, hexary], eth_common, byteutils,
+ rlp, ranges, state_db, nimcrypto, eth_trie/[hexary, db], eth_common, byteutils,
  ../errors, ../block_types, ../utils/header, ../constants, ./storage_types.nim
 
 type

nimbus/db/state_db.nim

@@ -7,8 +7,8 @@
 
 import
  sequtils, strformat, tables,
- chronicles, eth_common, nimcrypto, rlp, eth_trie/[hexary, memdb],
- ../constants, ../errors, ../validation, ../account
+ chronicles, eth_common, nimcrypto, rlp, eth_trie/[hexary, db],
+ ../constants, ../errors, ../validation
 
 logScope:
  topics = "state_db"

nimbus/genesis.nim

@@ -1,6 +1,8 @@
-import db/[db_chain, state_db], genesis_alloc, eth_common, tables, stint,
- byteutils, times, config, rlp, ranges, block_types, eth_trie,
- eth_trie/memdb, account, constants, nimcrypto, chronicles
+import
+ times, tables,
+ eth_common, stint, byteutils, rlp, ranges, block_types, nimcrypto,
+ chronicles, eth_trie, eth_trie/db,
+ db/[db_chain, state_db], genesis_alloc, config, constants
 
 type
  Genesis* = object

nimbus/nimbus.nim

@@ -10,9 +10,10 @@
 import
  os, strutils, net, eth_common, db/[storage_types, db_chain],
  asyncdispatch2, json_rpc/rpcserver, eth_keys,
- eth_p2p, eth_p2p/rlpx_protocols/[eth],
+ eth_p2p, eth_p2p/rlpx_protocols/[eth_protocol, les_protocol],
+ eth_p2p/blockchain_sync,
  config, genesis, rpc/[common, p2p], p2p/chain,
- eth_trie
+ eth_trie/db
 
 const UseSqlite = false
 

nimbus/p2p/chain.nim

@@ -1,10 +1,9 @@
 import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges,
  ../vm/[computation, interpreter_dispatch, message], ../constants, stint, nimcrypto,
  ../vm_state_transactions,
- eth_trie/memdb, eth_trie, rlp,
+ eth_trie/db, eth_trie, rlp,
  sugar
 
-
 type
  Chain* = ref object of AbstractChainDB
  db: BaseChainDB
@@ -101,7 +100,7 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre
  return gasUsed.u256 * t.gasPrice.u256
 
 proc calcTxRoot(transactions: openarray[Transaction]): Hash256 =
- var tr = initHexaryTrie(trieDB(newMemDB()))
+ var tr = initHexaryTrie(newMemoryDB())
  for i, t in transactions:
  tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange)
  return tr.rootHash
@@ -112,6 +111,9 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
 
  let blockReward = 5.u256 * pow(10.u256, 18) # 5 ETH
 
+ let transaction = c.db.db.beginTransaction()
+ defer: transaction.dispose()
+
  echo "Persisting blocks: ", headers[0].blockNumber, " - ", headers[^1].blockNumber
  for i in 0 ..< headers.len:
  let head = c.db.getCanonicalHead()
@@ -158,6 +160,8 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
  echo "Wrong state root in block ", headers[i].blockNumber, ". Expected: ", headers[i].stateRoot, ", Actual: ", stateDb.rootHash, " arrived from ", c.db.getCanonicalHead().stateRoot
  assert(headers[i].stateRoot == stateDb.rootHash)
 
-
  discard c.db.persistHeaderToDb(headers[i])
  assert(c.db.getCanonicalHead().blockHash == headers[i].blockHash)
+
+ transaction.commit()
+

nimbus/rpc/common.nim

@@ -6,10 +6,11 @@
 # at your option.
 # This file may not be copied, modified, or distributed except according to
 # those terms.
-import strutils, nimcrypto, eth_common, stint, eth_trie/[memdb, types]
+
 import
- json_rpc/server, ../vm_state, ../db/[db_chain, state_db],
- ../constants, ../config, hexstrings
+ strutils,
+ nimcrypto, eth_common, stint, json_rpc/server,
+ ../vm_state, ../db/[db_chain, state_db], ../constants, ../config, hexstrings
 
 proc setupCommonRPC*(server: RpcServer) =
  server.rpc("web3_clientVersion") do() -> string:

nimbus/rpc/p2p.nim

@@ -6,11 +6,14 @@
 # at your option.
 # This file may not be copied, modified, or distributed except according to
 # those terms.
+
 import
- nimcrypto, json_rpc/rpcserver, eth_p2p, hexstrings, strutils, stint,
- ../config, ../vm_state, ../constants, eth_trie/[memdb, types], eth_keys,
- ../db/[db_chain, state_db, storage_types], eth_common, rpc_types, byteutils,
- ranges/typedranges, times, ../utils/header, rlp, ../transaction
+ strutils, times, options,
+ nimcrypto, json_rpc/rpcserver, hexstrings, stint, byteutils, ranges/typedranges,
+ eth_common, eth_p2p, eth_keys, eth_trie/db, rlp,
+ ../utils/header, ../transaction, ../config, ../vm_state, ../constants,
+ ../db/[db_chain, state_db, storage_types],
+ rpc_types
 
 #[
  Note:
@@ -52,14 +55,14 @@ func headerFromTag(chain:BaseChainDB, blockTag: string): BlockHeader =
  result = chain.getBlockHeader(blockNum)
 
 proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
- 
- func getAccountDb(header: BlockHeader, readOnly = true): AccountStateDb = 
+
+ func getAccountDb(header: BlockHeader, readOnly = true): AccountStateDb =
  ## Retrieves the account db from canonical head
  let vmState = newBaseVMState(header, chain)
  result = vmState.chaindb.getStateDb(vmState.blockHeader.hash, readOnly)
 
  func accountDbFromTag(tag: string, readOnly = true): AccountStateDb =
- result = getAccountDb(chain.headerFromTag(tag)) 
+ result = getAccountDb(chain.headerFromTag(tag))
 
  proc getBlockBody(hash: KeccakHash): BlockBody =
  if not chain.getBlockBody(hash, result):
@@ -68,7 +71,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  rpcsrv.rpc("net_version") do() -> uint:
  let conf = getConfiguration()
  result = conf.net.networkId
- 
+
  rpcsrv.rpc("eth_syncing") do() -> JsonNode:
  ## Returns SyncObject or false when not syncing.
  # TODO: Requires PeerPool to check sync state.
@@ -84,7 +87,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  result = %sync
  else:
  result = newJBool(false)
- 
+
  rpcsrv.rpc("eth_coinbase") do() -> EthAddress:
  ## Returns the current coinbase address.
  result = chain.getCanonicalHead().coinbase
@@ -238,7 +241,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  ## Generates and returns an estimate of how much gas is necessary to allow the transaction to complete.
  ## The transaction will not be added to the blockchain. Note that the estimate may be significantly more than
  ## the amount of gas actually used by the transaction, for a variety of reasons including EVM mechanics and node performance.
- ## 
+ ##
  ## call: the transaction call object.
  ## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
  ## Returns the amount of gas used.
@@ -256,7 +259,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  startIdx = 0
  for i in 0 ..< blockBody.uncles.len:
  rawData[startIdx .. startIdx + 32] = blockBody.uncles[i].hash.data
- startIdx += 32 
+ startIdx += 32
  result.sha3Uncles = keccak256.digest(rawData)
 
  result.logsBloom = some(header.bloom)
@@ -317,7 +320,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  result.gasPrice = transaction.gasPrice
  result.gas = accountGas
  result.input = transaction.payload
- 
+
  rpcsrv.rpc("eth_getTransactionByHash") do(data: HexDataStr) -> TransactionObject:
  ## Returns the information about a transaction requested by transaction hash.
  ##
@@ -393,7 +396,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  idx.inc
 
  rpcsrv.rpc("eth_getUncleByBlockHashAndIndex") do(data: HexDataStr, quantity: int) -> Option[BlockObject]:
- ## Returns information about a uncle of a block by hash and uncle index position. 
+ ## Returns information about a uncle of a block by hash and uncle index position.
  ##
  ## data: hash of block.
  ## quantity: the uncle's index position.
@@ -428,12 +431,12 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  ## [A] "A in first position (and anything after)"
  ## [null, B] "anything in first position AND B in second position (and anything after)"
  ## [A, B] "A in first position AND B in second position (and anything after)"
- ## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)" 
+ ## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)"
  ##
  ## filterOptions: settings for this filter.
  ## Returns integer filter id.
  discard
- 
+
  rpcsrv.rpc("eth_newBlockFilter") do() -> int:
  ## Creates a filter in the node, to notify when a new block arrives.
  ## To check if the state has changed, call eth_getFilterChanges.
@@ -449,7 +452,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
  discard
 
  rpcsrv.rpc("eth_uninstallFilter") do(filterId: int) -> bool:
- ## Uninstalls a filter with given id. Should always be called when watch is no longer needed. 
+ ## Uninstalls a filter with given id. Should always be called when watch is no longer needed.
  ## Additonally Filters timeout when they aren't requested with eth_getFilterChanges for a period of time.
  ##
  ## filterId: The filter id.

nimbus/rpc/rpc_types.nim

@@ -28,7 +28,7 @@ type
  data*: EthHashStr # TODO: Support more data. The compiled code of a contract OR the hash of the invoked method signature and encoded parameters. For details see Ethereum Contract ABI.
  nonce*: int # (optional) integer of a nonce. This allows to overwrite your own pending transactions that use the same nonce 
 
- EthCall* = object 
+ EthCall* = object
  # Parameter from user
  source*: EthAddressStr # (optional) The address the transaction is send from.
  to*: EthAddressStr # The address the transaction is directed to.