From a00f042acae30069491359e23de46fb86f932511 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 21 Aug 2018 22:26:02 +0200 Subject: [PATCH] pytest: Add an RPC proxy inbetween bitcoind and bitcoin-cli This is a simple reverse proxy that `bitcoin-cli` can talk to when invoked by `lightningd`. It allows us to trace `bitcoin-cli` calls, and intercept calls to mock the replies, better than the current bash-script based method. --- tests/btcproxy.py | 81 ++++++++++++++++++++++++++++++++++++++++++ tests/fixtures.py | 4 +-- tests/requirements.txt | 2 ++ tests/test_misc.py | 2 -- tests/utils.py | 9 +++-- 5 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 tests/btcproxy.py diff --git a/tests/btcproxy.py b/tests/btcproxy.py new file mode 100644 index 000000000000..80d54d7a3a18 --- /dev/null +++ b/tests/btcproxy.py @@ -0,0 +1,81 @@ +""" A bitcoind proxy that allows instrumentation and canned responses +""" +from flask import Flask, request +from bitcoin.rpc import JSONRPCError +from bitcoin.rpc import RawProxy as BitcoinProxy +from utils import BitcoinD +from cheroot.wsgi import Server +from cheroot.wsgi import PathInfoDispatcher + +import decimal +import json +import logging +import os +import threading + + +class DecimalEncoder(json.JSONEncoder): + """By default json.dumps does not handle Decimals correctly, so we override it's handling + """ + def default(self, o): + if isinstance(o, decimal.Decimal): + return str(o) + return super(DecimalEncoder, self).default(o) + + +class ProxiedBitcoinD(BitcoinD): + def __init__(self, bitcoin_dir, proxyport=0): + BitcoinD.__init__(self, bitcoin_dir, rpcport=None) + self.app = Flask("BitcoindProxy") + self.app.add_url_rule("/", "API entrypoint", self.proxy, methods=['POST']) + self.proxyport = proxyport + + def proxy(self): + r = json.loads(request.data.decode('ASCII')) + conf_file = os.path.join(self.bitcoin_dir, 'bitcoin.conf') + brpc = BitcoinProxy(btc_conf_file=conf_file) + + try: + reply = { + "result": brpc._call(r['method'], *r['params']), + "error": None, + "id": r['id'] + } + except JSONRPCError as e: + reply = { + "error": e.error, + "id": r['id'] + } + return json.dumps(reply, cls=DecimalEncoder) + + def start(self): + d = PathInfoDispatcher({'/': self.app}) + self.server = Server(('0.0.0.0', self.proxyport), d) + self.proxy_thread = threading.Thread(target=self.server.start) + self.proxy_thread.daemon = True + self.proxy_thread.start() + BitcoinD.start(self) + + # Now that bitcoind is running on the real rpcport, let's tell all + # future callers to talk to the proxyport. We use the bind_addr as a + # signal that the port is bound and accepting connections. + while self.server.bind_addr[1] == 0: + pass + self.proxiedport = self.rpcport + self.rpcport = self.server.bind_addr[1] + logging.debug("bitcoind reverse proxy listening on {}, forwarding to {}".format( + self.rpcport, self.proxiedport + )) + + def stop(self): + BitcoinD.stop(self) + self.server.stop() + self.proxy_thread.join() + + +# The main entrypoint is mainly used to test the proxy. It is not used during +# lightningd testing. +if __name__ == "__main__": + p = ProxiedBitcoinD(bitcoin_dir='/tmp/bitcoind-test/', proxyport=5000) + p.start() + p.proxy_thread.join() diff --git a/tests/fixtures.py b/tests/fixtures.py index 85cfcb1d24dc..072a4f94a61a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,4 +1,5 @@ from concurrent import futures +from btcproxy import ProxiedBitcoinD from utils import NodeFactory import logging @@ -8,7 +9,6 @@ import shutil import sys import tempfile -import utils with open('config.vars') as configfile: @@ -68,7 +68,7 @@ def test_name(request): @pytest.fixture def bitcoind(directory): - bitcoind = utils.BitcoinD(bitcoin_dir=directory, rpcport=None) + bitcoind = ProxiedBitcoinD(bitcoin_dir=directory) try: bitcoind.start() except Exception: diff --git a/tests/requirements.txt b/tests/requirements.txt index b0193f03a282..818937667dfe 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,3 +3,5 @@ ephemeral-port-reserve==1.1.0 pytest-forked==0.2 pytest-xdist==1.22.2 flaky==3.4.0 +CherryPy==17.3.0 +Flask==1.0.2 diff --git a/tests/test_misc.py b/tests/test_misc.py index ad9def5fc92b..a58d1b47f382 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -570,8 +570,6 @@ def test_listconfigs(node_factory, bitcoind): configs = l1.rpc.listconfigs() # See utils.py - assert configs['bitcoin-datadir'] == bitcoind.bitcoin_dir - assert configs['lightning-dir'] == l1.daemon.lightning_dir assert configs['allow-deprecated-apis'] is False assert configs['network'] == 'regtest' assert configs['ignore-fee-limits'] is False diff --git a/tests/utils.py b/tests/utils.py index e998591054b3..a4e0ea55b9de 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -287,7 +287,7 @@ def generate_block(self, numblocks=1): class LightningD(TailableProc): - def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node_id=0): + def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node_id=0, bitcoin_rpcport=18332): TailableProc.__init__(self, lightning_dir) self.lightning_dir = lightning_dir self.port = port @@ -296,12 +296,14 @@ def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node self.opts = LIGHTNINGD_CONFIG.copy() opts = { - 'bitcoin-datadir': bitcoin_dir, 'lightning-dir': lightning_dir, 'addr': '127.0.0.1:{}'.format(port), 'allow-deprecated-apis': 'false', 'network': 'regtest', 'ignore-fee-limits': 'false', + 'bitcoin-rpcport': bitcoin_rpcport, + 'bitcoin-rpcuser': BITCOIND_CONFIG['rpcuser'], + 'bitcoin-rpcpassword': BITCOIND_CONFIG['rpcpassword'], } for k, v in opts.items(): @@ -689,7 +691,8 @@ def get_node(self, disconnect=None, options=None, may_fail=False, may_reconnect= socket_path = os.path.join(lightning_dir, "lightning-rpc").format(node_id) daemon = LightningD( lightning_dir, self.bitcoind.bitcoin_dir, - port=port, random_hsm=random_hsm, node_id=node_id + port=port, random_hsm=random_hsm, node_id=node_id, + bitcoin_rpcport=self.bitcoind.rpcport ) # If we have a disconnect string, dump it to a file for daemon. if disconnect: