From 81d37fc5f980e72bc54a2dd006a59f13ce0f8c39 Mon Sep 17 00:00:00 2001 From: vub Date: Tue, 12 Jan 2016 05:30:14 -0500 Subject: [PATCH] Added tree split/merge and made next/prev work --- ethereum/blocks.py | 2 +- ethereum/tests/test_blocks.py | 1 + ethereum/trie.py | 163 +++++++++++++++++++++++++++++++--- fixtures | 2 +- 4 files changed, 154 insertions(+), 14 deletions(-) diff --git a/ethereum/blocks.py b/ethereum/blocks.py index 0cc8b751d..097698798 100644 --- a/ethereum/blocks.py +++ b/ethereum/blocks.py @@ -38,7 +38,6 @@ # Difficulty adjustment algo def calc_difficulty(parent, timestamp): config = parent.config - print 'cutoff', config['HOMESTEAD_FORK_BLKNUM'] offset = parent.difficulty // config['BLOCK_DIFF_FACTOR'] if parent.number >= (config['HOMESTEAD_FORK_BLKNUM'] - 1): sign = max(1 - ((timestamp - parent.timestamp) // config['HOMESTEAD_DIFF_ADJUSTMENT_CUTOFF']), -99) @@ -51,6 +50,7 @@ def calc_difficulty(parent, timestamp): period_count = (parent.number + 1) // config['EXPDIFF_PERIOD'] if period_count >= config['EXPDIFF_FREE_PERIODS']: o = max(o + 2**(period_count - config['EXPDIFF_FREE_PERIODS']), config['MIN_DIFF']) + # print 'Calculating difficulty of block %d, timestamp difference %d, parent diff %d, child diff %d' % (parent.number + 1, timestamp - parent.timestamp, parent.difficulty, o) return o diff --git a/ethereum/tests/test_blocks.py b/ethereum/tests/test_blocks.py index dcaa06502..9deefc0c5 100644 --- a/ethereum/tests/test_blocks.py +++ b/ethereum/tests/test_blocks.py @@ -91,6 +91,7 @@ def run_block_test(params, config_overrides = {}): assert b2.validate_uncles() blockmap[b2.hash] = b2 env.db.put(b2.hash, rlp.encode(b2)) + print 'Block %d with state root %s' % (b2.number, b2.state.root_hash.encode('hex')) # blkdict = b.to_dict(False, True, False, True) # assert blk["blockHeader"] == \ # translate_keys(blkdict["header"], translator_list, lambda y, x: x, []) diff --git a/ethereum/trie.py b/ethereum/trie.py index cbf05e6a5..a7f476fee 100644 --- a/ethereum/trie.py +++ b/ethereum/trie.py @@ -460,84 +460,223 @@ def _update_kv_node(self, node, key, value): return new_node def _getany(self, node, reverse=False, path=[]): + # print 'getany', node, 'reverse=', reverse, path node_type = self._get_node_type(node) if node_type == NODE_TYPE_BLANK: return None if node_type == NODE_TYPE_BRANCH: - if node[16]: + if node[16] and not reverse: + # print 'found!', [16], path return [16] scan_range = list(range(16)) if reverse: scan_range.reverse() for i in scan_range: - o = self._getany(self._decode_to_node(node[i]), path=path + [i]) - if o: + o = self._getany(self._decode_to_node(node[i]), reverse=reverse, path=path + [i]) + if o is not None: + # print 'found@', [i] + o, path return [i] + o + if node[16] and reverse: + # print 'found!', [16], path + return [16] return None curr_key = without_terminator(unpack_to_nibbles(node[0])) if node_type == NODE_TYPE_LEAF: + # print 'found#', curr_key, path return curr_key if node_type == NODE_TYPE_EXTENSION: curr_key = without_terminator(unpack_to_nibbles(node[0])) sub_node = self._decode_to_node(node[1]) - return self._getany(sub_node, path=path + curr_key) + return curr_key + self._getany(sub_node, reverse=reverse, path=path + curr_key) + + def _split(self, node, key): + node_type = self._get_node_type(node) + if node_type == NODE_TYPE_BLANK: + return BLANK_NODE, BLANK_NODE + elif not key: + return BLANK_NODE, node + elif node_type == NODE_TYPE_BRANCH: + b1 = node[:key[0]] + b1 += [''] * (17 - len(b1)) + b2 = node[key[0]+1:] + b2 = [''] * (17 - len(b2)) + b2 + b1[16], b2[16] = b2[16], b1[16] + sub = self._decode_to_node(node[key[0]]) + sub1, sub2 = self._split(sub, key[1:]) + b1[key[0]] = self._encode_node(sub1) if sub1 else '' + b2[key[0]] = self._encode_node(sub2) if sub2 else '' + return self._normalize_branch_node(b1) if len([x for x in b1 if x]) else BLANK_NODE, \ + self._normalize_branch_node(b2) if len([x for x in b2 if x]) else BLANK_NODE + + descend_key = without_terminator(unpack_to_nibbles(node[0])) + if node_type == NODE_TYPE_LEAF: + if descend_key < key: + return node, BLANK_NODE + else: + return BLANK_NODE, node + elif node_type == NODE_TYPE_EXTENSION: + sub_node = self._decode_to_node(node[1]) + sub_key = key[len(descend_key):] + if starts_with(key, descend_key): + sub1, sub2 = self._split(sub_node, sub_key) + subtype1 = self._get_node_type(sub1) + subtype2 = self._get_node_type(sub2) + if not sub1: + o1 = BLANK_NODE + elif subtype1 in (NODE_TYPE_LEAF, NODE_TYPE_EXTENSION): + new_key = key[:len(descend_key)] + unpack_to_nibbles(sub1[0]) + o1 = [pack_nibbles(new_key), sub1[1]] + else: + o1 = [pack_nibbles(key[:len(descend_key)]), self._encode_node(sub1)] + if not sub2: + o2 = BLANK_NODE + elif subtype2 in (NODE_TYPE_LEAF, NODE_TYPE_EXTENSION): + new_key = key[:len(descend_key)] + unpack_to_nibbles(sub2[0]) + o2 = [pack_nibbles(new_key), sub2[1]] + else: + o2 = [pack_nibbles(key[:len(descend_key)]), self._encode_node(sub2)] + return o1, o2 + elif descend_key < key[:len(descend_key)]: + return node, BLANK_NODE + elif descend_key > key[:len(descend_key)]: + return BLANK_NODE, node + else: + return BLANK_NODE, BLANK_NODE + + def split(self, key): + key = bin_to_nibbles(key) + r1, r2 = self._split(self.root_node, key) + t1, t2 = Trie(self.db), Trie(self.db) + t1.root_node, t2.root_node = r1, r2 + return t1, t2 + + def _merge(self, node1, node2): + assert isinstance(node1, list) or not node1 + assert isinstance(node2, list) or not node2 + node_type1 = self._get_node_type(node1) + node_type2 = self._get_node_type(node2) + if not node1: + return node2 + if not node2: + return node1 + if node_type1 != NODE_TYPE_BRANCH and node_type2 != NODE_TYPE_BRANCH: + descend_key1 = unpack_to_nibbles(node1[0]) + descend_key2 = unpack_to_nibbles(node2[0]) + # find longest common prefix + prefix_length = 0 + for i in range(min(len(descend_key1), len(descend_key2))): + if descend_key1[i] != descend_key2[i]: + break + prefix_length = i + 1 + if prefix_length: + sub1 = self._decode_to_node(node1[1]) if node_type1 == NODE_TYPE_EXTENSION else node1[1] + new_sub1 = [ + pack_nibbles(descend_key1[prefix_length:]), + sub1 + ] if descend_key1[prefix_length:] else sub1 + sub2 = self._decode_to_node(node2[1]) if node_type2 == NODE_TYPE_EXTENSION else node2[1] + new_sub2 = [ + pack_nibbles(descend_key2[prefix_length:]), + sub2 + ] if descend_key2[prefix_length:] else sub2 + return [pack_nibbles(descend_key1[:prefix_length]), + self._encode_node(self._merge(new_sub1, new_sub2))] + + nodes = [[node1], [node2]] + for (node, node_type) in zip(nodes, [node_type1, node_type2]): + if node_type != NODE_TYPE_BRANCH: + new_node = [BLANK_NODE] * 17 + curr_key = unpack_to_nibbles(node[0][0]) + new_node[curr_key[0]] = self._encode_node([ + pack_nibbles(curr_key[1:]), + node[0][1] + ]) if curr_key[0] < 16 and curr_key[1:] else node[0][1] + node[0] = new_node + node1, node2 = nodes[0][0], nodes[1][0] + assert len([i for i in range(17) if node1[i] and node2[i]]) <= 1 + new_node = [self._encode_node(self._merge(self._decode_to_node(node1[i]), self._decode_to_node(node2[i]))) if node1[i] and node2[i] else node1[i] or node2[i] for i in range(17)] + return new_node + + @classmethod + def unsafe_merge(cls, trie1, trie2): + t = Trie(trie1.db) + t.root_node = t._merge(trie1.root_node, trie2.root_node) + return t def _iter(self, node, key, reverse=False, path=[]): + # print 'iter', node, key, 'reverse =', reverse, 'path =', path node_type = self._get_node_type(node) if node_type == NODE_TYPE_BLANK: return None elif node_type == NODE_TYPE_BRANCH: + # print 'b' if len(key): sub_node = self._decode_to_node(node[key[0]]) o = self._iter(sub_node, key[1:], reverse, path + [key[0]]) - if o: + if o is not None: + # print 'returning', [key[0]] + o, path return [key[0]] + o if reverse: - scan_range = list(range(key[0] if len(key) else 0)) + scan_range = reversed(list(range(key[0] if len(key) else 0))) else: scan_range = list(range(key[0] + 1 if len(key) else 0, 16)) for i in scan_range: sub_node = self._decode_to_node(node[i]) + # print 'prelim getany', path+[i] o = self._getany(sub_node, reverse, path + [i]) - if o: + if o is not None: + # print 'returning', [i] + o, path return [i] + o - if reverse and node[16]: + if reverse and key and node[16]: + # print 'o' return [16] return None descend_key = without_terminator(unpack_to_nibbles(node[0])) if node_type == NODE_TYPE_LEAF: if reverse: + # print 'L', descend_key, key, descend_key if descend_key < key else None, path return descend_key if descend_key < key else None else: + # print 'L', descend_key, key, descend_key if descend_key > key else None, path return descend_key if descend_key > key else None if node_type == NODE_TYPE_EXTENSION: # traverse child nodes sub_node = self._decode_to_node(node[1]) sub_key = key[len(descend_key):] + # print 'amhere', key, descend_key, descend_key > key[:len(descend_key)] if starts_with(key, descend_key): o = self._iter(sub_node, sub_key, reverse, path + descend_key) elif descend_key > key[:len(descend_key)] and not reverse: - o = self._getany(sub_node, sub_key, False, path + descend_key) + # print 1 + # print 'prelim getany', path+descend_key + o = self._getany(sub_node, False, path + descend_key) elif descend_key < key[:len(descend_key)] and reverse: - o = self._getany(sub_node, sub_key, True, path + descend_key) + # print 2 + # print 'prelim getany', path+descend_key + o = self._getany(sub_node, True, path + descend_key) else: o = None + # print 'returning@', descend_key + o if o else None, path return descend_key + o if o else None def next(self, key): + # print 'nextting' key = bin_to_nibbles(key) o = self._iter(self.root_node, key) - return nibbles_to_bin(o) if o else None + # print 'answer', o + return nibbles_to_bin(without_terminator(o)) if o else None def prev(self, key): + # print 'prevving' key = bin_to_nibbles(key) o = self._iter(self.root_node, key, reverse=True) - return nibbles_to_bin(o) if o else None + # print 'answer', o + return nibbles_to_bin(without_terminator(o)) if o else None def _delete_node_storage(self, node): '''delete storage diff --git a/fixtures b/fixtures index f82762ebd..0c020f75f 160000 --- a/fixtures +++ b/fixtures @@ -1 +1 @@ -Subproject commit f82762ebd03a0e25bdf05159a5a621c6b1813a7f +Subproject commit 0c020f75f6ad2be5816bb3445253b7aaf8788e03