From d5aa19813c32eff02d4b65ac9a4131f3eed04825 Mon Sep 17 00:00:00 2001 From: Doug Date: Tue, 20 Dec 2016 19:25:24 -0800 Subject: [PATCH 1/2] Allow linearization scripts to support hash byte reversal Currently, the linearization scripts require input hashes to be in one endian form. Add support for byte reversal. --- contrib/linearize/README.md | 40 +++++++++++++++---------- contrib/linearize/example-linearize.cfg | 4 ++- contrib/linearize/linearize-data.py | 19 ++++++++++-- contrib/linearize/linearize-hashes.py | 14 ++++++++- 4 files changed, 58 insertions(+), 19 deletions(-) diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 06f278f3b3c71..07597481e0604 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,33 +1,43 @@ # Linearize -Construct a linear, no-fork, best version of the blockchain. +Construct a linear, no-fork, best version of the Bitcoin blockchain. ## Step 1: Download hash list $ ./linearize-hashes.py linearize.cfg > hashlist.txt Required configuration file settings for linearize-hashes: -* RPC: rpcuser, rpcpassword +* RPC: `rpcuser`, `rpcpassword` Optional config file setting for linearize-hashes: -* RPC: host, port -* Block chain: min_height, max_height +* RPC: `host`, `port` (Default: `127.0.0.1:8332`) +* Blockchain: `min_height`, `max_height` +* `rev_hash_bytes`: If true, the written block hash list will be +byte-reversed. (In other words, the hash returned by getblockhash will have its +bytes reversed.) False by default. Intended for generation of +standalone hash lists but safe to use with linearize-data.py, which will output +the same data no matter which byte format is chosen. ## Step 2: Copy local block data $ ./linearize-data.py linearize.cfg Required configuration file settings: -* "input": bitcoind blocks/ directory containing blkNNNNN.dat -* "hashlist": text file containing list of block hashes, linearized-hashes.py -output. -* "output_file": bootstrap.dat +* `output_file`: The file that will contain the final blockchain. or -* "output": output directory for linearized blocks/blkNNNNN.dat output +* `output`: Output directory for linearized blocks/blkNNNNN.dat output. Optional config file setting for linearize-data: -* "netmagic": network magic number -* "max_out_sz": maximum output file size (default `1000*1000*1000`) -* "split_timestamp": Split files when a new month is first seen, in addition to -reaching a maximum file size. -* "file_timestamp": Set each file's last-modified time to that of the -most recent block in that file. +* `file_timestamp`: Set each file's last-modified time to that of the most +recent block in that file. +* `genesis`: The hash of the genesis block in the blockchain. +* `input: bitcoind blocks/ directory containing blkNNNNN.dat +* `hashlist`: text file containing list of block hashes created by +linearize-hashes.py. +* `max_out_sz`: Maximum size for files created by the `output_file` option. +(Default: `1000*1000*1000 bytes`) +* `netmagic`: Network magic number. +* `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py +will be byte-reversed when read by linearize-data.py. See the linearize-hashes +entry for more information. +* `split_timestamp`: Split blockchain files when a new month is first seen, in +addition to reaching a maximum file size (`max_out_sz`). diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 38da02e66c1e7..cccdd792131ae 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -23,7 +23,9 @@ input=/home/example/.bitcoin/blocks output_file=/home/example/Downloads/bootstrap.dat hashlist=hashlist.txt -split_year=1 # Maxmimum size in bytes of out-of-order blocks cache in memory out_of_order_cache_sz = 100000000 + +# Do we want the reverse the hash bytes coming from getblockhash? +rev_hash_bytes = False diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index d0417748fc072..adea35dbd5588 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -23,6 +23,12 @@ settings = {} +##### Switch endian-ness ##### +def hex_switchEndian(s): + """ Switches the endianness of a hex string (in pairs of hex chars) """ + pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] + return ''.join(pairList[::-1]) + def uint32(x): return x & 0xffffffffL @@ -69,17 +75,21 @@ def get_blk_dt(blk_hdr): dt_ym = datetime.datetime(dt.year, dt.month, 1) return (dt_ym, nTime) +# When getting the list of block hashes, undo any byte reversals. def get_block_hashes(settings): blkindex = [] f = open(settings['hashlist'], "r") for line in f: line = line.rstrip() + if settings['rev_hash_bytes'] == 'true': + line = hex_switchEndian(line) blkindex.append(line) print("Read " + str(len(blkindex)) + " hashes") return blkindex +# The block map shouldn't give or receive byte-reversed hashes. def mkblockmap(blkindex): blkmap = {} for height,hash in enumerate(blkindex): @@ -265,6 +275,12 @@ def run(self): settings[m.group(1)] = m.group(2) f.close() + # Force hash byte format setting to be lowercase to make comparisons easier. + # Also place upfront in case any settings need to know about it. + if 'rev_hash_bytes' not in settings: + settings['rev_hash_bytes'] = 'false' + settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + if 'netmagic' not in settings: settings['netmagic'] = 'f9beb4d9' if 'genesis' not in settings: @@ -295,9 +311,8 @@ def run(self): blkindex = get_block_hashes(settings) blkmap = mkblockmap(blkindex) + # Block hash map won't be byte-reversed. Neither should the genesis hash. if not settings['genesis'] in blkmap: print("Genesis block not found in hashlist") else: BlockDataCopier(settings, blkindex, blkmap).run() - - diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index ba8d6f8a044c8..9efdf13255030 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -17,6 +17,12 @@ settings = {} +##### Switch endian-ness ##### +def hex_switchEndian(s): + """ Switches the endianness of a hex string (in pairs of hex chars) """ + pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] + return ''.join(pairList[::-1]) + class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) @@ -70,6 +76,8 @@ def get_block_hashes(settings, max_blocks_per_call=10000): print('JSON-RPC: error at height', height+x, ': ', resp_obj['error'], file=sys.stderr) exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence + if settings['rev_hash_bytes'] == 'true': + resp_obj['result'] = hex_switchEndian(resp_obj['result']) print(resp_obj['result']) height += num_blocks @@ -101,6 +109,8 @@ def get_block_hashes(settings, max_blocks_per_call=10000): settings['min_height'] = 0 if 'max_height' not in settings: settings['max_height'] = 313000 + if 'rev_hash_bytes' not in settings: + settings['rev_hash_bytes'] = 'false' if 'rpcuser' not in settings or 'rpcpassword' not in settings: print("Missing username and/or password in cfg file", file=stderr) sys.exit(1) @@ -109,5 +119,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): settings['min_height'] = int(settings['min_height']) settings['max_height'] = int(settings['max_height']) - get_block_hashes(settings) + # Force hash byte format setting to be lowercase to make comparisons easier. + settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + get_block_hashes(settings) From 3c8f63ba7c7be62d462727f4d67633e1064f0f79 Mon Sep 17 00:00:00 2001 From: Doug Date: Tue, 20 Dec 2016 19:27:01 -0800 Subject: [PATCH 2/2] Make linearize scripts Python 3-compatible. --- contrib/linearize/README.md | 13 +++++++---- contrib/linearize/linearize-data.py | 28 +++++++++++++---------- contrib/linearize/linearize-hashes.py | 32 ++++++++++++++++++--------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 07597481e0604..adc9a559cc34d 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,5 +1,6 @@ # Linearize -Construct a linear, no-fork, best version of the Bitcoin blockchain. +Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts +run using Python 3 but are compatible with Python 2. ## Step 1: Download hash list @@ -9,7 +10,8 @@ Required configuration file settings for linearize-hashes: * RPC: `rpcuser`, `rpcpassword` Optional config file setting for linearize-hashes: -* RPC: `host`, `port` (Default: `127.0.0.1:8332`) +* RPC: `host` (Default: `127.0.0.1`) +* RPC: `port` (Default: `8332`) * Blockchain: `min_height`, `max_height` * `rev_hash_bytes`: If true, the written block hash list will be byte-reversed. (In other words, the hash returned by getblockhash will have its @@ -17,6 +19,9 @@ bytes reversed.) False by default. Intended for generation of standalone hash lists but safe to use with linearize-data.py, which will output the same data no matter which byte format is chosen. +The `linearize-hashes` script requires a connection, local or remote, to a +JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient. + ## Step 2: Copy local block data $ ./linearize-data.py linearize.cfg @@ -24,13 +29,13 @@ the same data no matter which byte format is chosen. Required configuration file settings: * `output_file`: The file that will contain the final blockchain. or -* `output`: Output directory for linearized blocks/blkNNNNN.dat output. +* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output. Optional config file setting for linearize-data: * `file_timestamp`: Set each file's last-modified time to that of the most recent block in that file. * `genesis`: The hash of the genesis block in the blockchain. -* `input: bitcoind blocks/ directory containing blkNNNNN.dat +* `input`: bitcoind blocks/ directory containing blkNNNNN.dat * `hashlist`: text file containing list of block hashes created by linearize-hashes.py. * `max_out_sz`: Maximum size for files created by the `output_file` option. diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index adea35dbd5588..043bf2e814fc1 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # linearize-data.py: Construct a linear, no-fork version of the chain. # @@ -8,29 +8,33 @@ # from __future__ import print_function, division +try: # Python 3 + import http.client as httplib +except ImportError: # Python 2 + import httplib import json import struct import re import os import os.path import base64 -import httplib import sys import hashlib import datetime import time from collections import namedtuple +from binascii import hexlify, unhexlify settings = {} ##### Switch endian-ness ##### def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i]+s[i+1] for i in range(0,len(s),2)] - return ''.join(pairList[::-1]) + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() def uint32(x): - return x & 0xffffffffL + return x & 0xffffffff def bytereverse(x): return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | @@ -41,14 +45,14 @@ def bufreverse(in_buf): for i in range(0, len(in_buf), 4): word = struct.unpack('@I', in_buf[i:i+4])[0] out_words.append(struct.pack('@I', bytereverse(word))) - return ''.join(out_words) + return b''.join(out_words) def wordreverse(in_buf): out_words = [] for i in range(0, len(in_buf), 4): out_words.append(in_buf[i:i+4]) out_words.reverse() - return ''.join(out_words) + return b''.join(out_words) def calc_hdr_hash(blk_hdr): hash1 = hashlib.sha256() @@ -65,7 +69,7 @@ def calc_hash_str(blk_hdr): hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) - hash_str = hash.encode('hex') + hash_str = hexlify(hash).decode('utf-8') return hash_str def get_blk_dt(blk_hdr): @@ -217,7 +221,7 @@ def run(self): inMagic = inhdr[:4] if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + inMagic.encode('hex')) + print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) return inLenLE = inhdr[4:] su = struct.unpack("