From 367e239c71a0bb89155d05dd47d574e39f6752f7 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Thu, 13 Oct 2016 00:20:01 -0700 Subject: [PATCH 1/4] add Dockerfile and entrypoint.py --- .dockerignore | 2 + Dockerfile | 52 ++++++++ broadcast-tx.py | 0 check-config.py | 25 ++++ create-unsigned-tx.py | 0 docker/README | 6 + docker/entrypoint.py | 208 ++++++++++++++++++++++++++++++ joinmarket/__init__.py | 2 +- joinmarket/blockchaininterface.py | 2 +- joinmarket/jsonrpc.py | 14 +- joinmarket/wallet.py | 39 +++++- joinmarket/yieldgenerator.py | 13 +- ob-watcher.py | 0 patientsendpayment.py | 0 sendpayment.py | 0 tumbler.py | 0 wallet-tool.py | 24 ++-- yg-pe.py | 0 yield-generator-basic.py | 0 19 files changed, 356 insertions(+), 31 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile mode change 100644 => 100755 broadcast-tx.py create mode 100755 check-config.py mode change 100644 => 100755 create-unsigned-tx.py create mode 100644 docker/README create mode 100755 docker/entrypoint.py mode change 100644 => 100755 ob-watcher.py mode change 100644 => 100755 patientsendpayment.py mode change 100644 => 100755 sendpayment.py mode change 100644 => 100755 tumbler.py mode change 100644 => 100755 wallet-tool.py mode change 100644 => 100755 yg-pe.py mode change 100644 => 100755 yield-generator-basic.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..32c9f031 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git/ +env/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7db9900f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +# +# Quick and easy joinmarket +# +# docker build -t "bwstitt/joinmarket:latest" . +# +# Copying of the code is delayed as long as possible so that rebuilding the container while developing is faster +# This also means some of the install steps aren't in the most obvious order and the virtualenv is outside the code +# + +FROM bwstitt/python-jessie:python2 + +# Install packages for joinmarket +RUN docker-apt-install \ + gcc \ + libsodium13 \ + python-dev + +# i needed these when compiling myself, but new versions of pip with wheels save us + #libatlas-dev \ + #libblas-dev \ + #libfreetype6-dev \ + #libpng12-dev \ + #libsodium-dev \ + #pkg-config \ + #python-dev \ + +# install deps that don't depend on the code as the user and fix /pyenv/local/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment. +# TODO: double check that this actually builds the font caches +RUN chroot --userspec=abc / pip install matplotlib==2.0.0 \ + && chroot --userspec=abc / python -c \ + "from matplotlib.font_manager import FontManager; print(FontManager())" + +# copy requirements before code. this will make the image builds faster when code changes but requirements don't +COPY requirements.txt /src/ +RUN chroot --userspec=abc / pip install -r /src/requirements.txt + +# install the code +# todo: i wish copy would keep the user... +COPY . /src/ +WORKDIR /src +RUN chown -R abc:abc . + +# setup data volumes for logs and wallets +# todo: handle the blacklist and commitments +VOLUME /src/logs /src/wallets + +USER abc +ENV MPLBACKEND=agg +ENTRYPOINT ["python", "/src/docker/entrypoint.py"] +CMD ["ob_watcher", "-H", "0.0.0.0"] + +EXPOSE 62601 diff --git a/broadcast-tx.py b/broadcast-tx.py old mode 100644 new mode 100755 diff --git a/check-config.py b/check-config.py new file mode 100755 index 00000000..d31f874a --- /dev/null +++ b/check-config.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import + +import sys + +from joinmarket import get_log, load_program_config + + +log = get_log() + + +def main(): + """Simple command to make sure the config loads. + + This will exit 1 if the config cannot be loaded or the blockchaininterface + doesn't respond. + """ + try: + load_program_config() + except Exception: + log.exception("Error while loading config") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/create-unsigned-tx.py b/create-unsigned-tx.py old mode 100644 new mode 100755 diff --git a/docker/README b/docker/README new file mode 100644 index 00000000..b1ac0163 --- /dev/null +++ b/docker/README @@ -0,0 +1,6 @@ +todo: + - explain how to setup Tor + - explain how to setup bitcoind + - explain how to setup the config (and figure out how to keep sensitive things out of git) + - maybe use click instead of argparse + - use --entry-help instead of --help so we can still see joinmarket's help diff --git a/docker/entrypoint.py b/docker/entrypoint.py new file mode 100755 index 00000000..dc6409eb --- /dev/null +++ b/docker/entrypoint.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python2 +"""Entrypoints for Docker containers to run joinmarket. + +todo: wait_for_file should probably be a decorator +todo: either always force wallet.json or make waiting for the wallet smarter +""" +import argparse +import logging +import os +import os.path +import subprocess +import sys +import time + + +# +# Helpers +# + +DEBUG_LOG_FORMAT = ( + '-' * 80 + '\n' + + '%(asctime)s %(levelname)s in %(name)s @ %(threadName)s:\n' + + '%(pathname)s:%(lineno)d\n' + + '%(message)s\n' + + '-' * 80 +) +DEFAULT_LOG_FORMAT = '%(asctime)s %(levelname)s: %(threadName)s: %(name)s: %(message)s' # noqa + +log = logging.getLogger(__name__) + + +def run(*args): + """Run a python command inside the joinmarket virtualenv. + + Raises subprocess.CalledProcessError if the command fails. + """ + if not args: + raise ValueError("run needs at least one arg") + + command_list = [sys.executable] + map(str, args) + + log.info("Running %s...", command_list[1]) + log.debug("Full command: %s", command_list) + + return subprocess.check_call(command_list, env=os.environ) + + +def run_or_exit(*args): + """Run a python command inside the joinmarket virtualenv. + + Logs and exits if the command fails. + """ + try: + return run(*args) + except subprocess.CalledProcessError as e: + log.error("%s", e) + sys.exit(e.returncode) + + +def wait_for_config(*args): + """Sleep until config loads. + + Config loading includes bitcoind responding to getblockchaininfo + + Todo: exponential backoff of the sleep. maybe log less, too + Todo: args here are hacky. make this function and the command seperate + """ + while True: + try: + run('check-config.py') + except subprocess.CalledProcessError as e: + # TODO: this is too verbose + log.error("Unable to load config: %s. Sleeping..." % e) + time.sleep(60) + else: + break + + return True + + +def wait_for_file(filename, sleep=10): + """Sleep until a given file exists.""" + if os.path.exists(filename): + return + + log.info("'%s' does not exist. Check the README", filename) + + log.info("Sleeping until '%s' exists...", filename) + while not os.path.exists(filename): + time.sleep(sleep) + + log.info("Found '%s'", filename) + return + + +# +# Commands +# + +def get_parser(): + """Create an argument parser that routes to the command functions.""" + # create the top-level parser + parser = argparse.ArgumentParser() + # todo: configurable log level + subparsers = parser.add_subparsers() + + # create the parser for the "maker" command + parser_maker = subparsers.add_parser('maker') + parser_maker.set_defaults(func=maker) + + # create the parser for the "ob_watcher" command + parser_ob_watcher = subparsers.add_parser('ob_watcher') + parser_ob_watcher.set_defaults(func=ob_watcher) + + # create the parser for the "sendpayment" command + parser_sendpayment = subparsers.add_parser('sendpayment') + parser_sendpayment.set_defaults(func=sendpayment) + + # create the parser for the "taker" command + parser_tumbler = subparsers.add_parser('tumbler') + parser_tumbler.set_defaults(func=tumbler) + + # create the parser for the "wallet_tool" command + parser_wallet_tool = subparsers.add_parser('wallet_tool') + parser_wallet_tool.set_defaults(func=wallet_tool) + + # other scripts might find waiting for the config helpful, too + parser_wait_for_config = subparsers.add_parser('wait_for_config') + parser_wait_for_config.set_defaults(func=wait_for_config) + + return parser + + +def maker(args): + """Earn Bitcoins and privacy.""" + wallet_filename = 'wallet.json' + wait_for_file("wallets/%s" % wallet_filename) + + # wait for bitcoind to respond + wait_for_config() + + run_or_exit('yg-pe.py', wallet_filename, *args) + + +def ob_watcher(args): + """Watch the orderbook.""" + # wait for bitcoind to respond + # todo: although, why does the orderbook need bitcoind? + wait_for_config() + + run_or_exit('ob-watcher.py', *args) + + +def sendpayment(args): + """"Send Bitcoins with privacy. + + todo: make sure we only sendpayment with coins that have already been + joined at least once. + """ + wallet_filename = 'wallet.json' + wait_for_file("wallets/%s" % wallet_filename) + + # wait for bitcoind to respond + wait_for_config() + + run_or_exit('sendpayment.py', *args) + + +def tumbler(args): + """"Send Bitcoins with layers of privacy.""" + wallet_filename = 'wallet.json' + wait_for_file("wallets/%s" % wallet_filename) + + # wait for bitcoind to respond + wait_for_config() + + run_or_exit('tumbler.py', *args) + + +def wallet_tool(args): + """Inspect and manage your Bitcoin wallet.""" + run_or_exit('wallet-tool.py', *args) + + +def main(): + """Manage joinmarket.""" + parser = get_parser() + args, passthrough_args = parser.parse_known_args() + + log_level = logging.DEBUG # todo: get log_level from args + + if log_level == logging.DEBUG: + log_format = DEBUG_LOG_FORMAT + else: + log_format = DEFAULT_LOG_FORMAT + + logging.basicConfig( + format=log_format, + level=log_level, + stream=sys.stdout, + ) + log.debug("Hello!") + + args.func(passthrough_args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/joinmarket/__init__.py b/joinmarket/__init__.py index 70c791cc..1f659a1c 100644 --- a/joinmarket/__init__.py +++ b/joinmarket/__init__.py @@ -16,7 +16,7 @@ from .slowaes import decryptData, encryptData from .taker import Taker, OrderbookWatch, CoinJoinTX from .wallet import AbstractWallet, BitcoinCoreInterface, Wallet, \ - BitcoinCoreWallet + BitcoinCoreWallet, get_wallet_pass from .configure import load_program_config, jm_single, get_p2pk_vbyte, \ get_network, jm_single, get_network, validate_address, get_irc_mchannels, \ check_utxo_blacklist diff --git a/joinmarket/blockchaininterface.py b/joinmarket/blockchaininterface.py index d0241c11..584f89c2 100644 --- a/joinmarket/blockchaininterface.py +++ b/joinmarket/blockchaininterface.py @@ -947,7 +947,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print('restart Bitcoin Core with -rescan if you\'re ' 'recovering an existing wallet from backup seed') print(' otherwise just restart this joinmarket script') - sys.exit(0) + sys.exit(2) def sync_wallet(self, wallet, fast=False): #trigger fast sync if the index_cache is available diff --git a/joinmarket/jsonrpc.py b/joinmarket/jsonrpc.py index d12243f6..2ffa07d9 100644 --- a/joinmarket/jsonrpc.py +++ b/joinmarket/jsonrpc.py @@ -34,6 +34,9 @@ def __init__(self, obj): self.code = obj["code"] self.message = obj["message"] + def __str__(self): + return "%s %s: %s" % (self.__class__.__name__, self.code, self.message) + class JsonRpcConnectionError(Exception): """ @@ -78,19 +81,18 @@ def queryHTTP(self, obj): conn.request("POST", "", body, headers) response = conn.getresponse() - if response.status == 401: + if response.status in [401, 403]: conn.close() raise JsonRpcConnectionError( "authentication for JSON-RPC failed") - # All of the codes below are 'fine' from a JSON-RPC point of view. - if response.status not in [200, 404, 500]: - conn.close() - raise JsonRpcConnectionError("unknown error in JSON-RPC") - data = response.read() conn.close() + # All of the codes below are 'fine' from a JSON-RPC point of view. + if response.status not in [200, 404, 500]: + raise JsonRpcConnectionError("unknown error %s in JSON-RPC: %s" % (response.status, data)) + return json.loads(data) except JsonRpcConnectionError as exc: diff --git a/joinmarket/wallet.py b/joinmarket/wallet.py index 3ff97f6c..ee8aff0a 100644 --- a/joinmarket/wallet.py +++ b/joinmarket/wallet.py @@ -18,6 +18,36 @@ log = get_log() + +def get_wallet_pass(wallet_filepath=None, password=None, confirm=False): + if password is not None: + return password + + if wallet_filepath is not None: + pass_filename = wallet_filepath[:-4] + 'pass' + if os.path.exists(pass_filename): + with open(pass_filename, 'r') as f: + return f.read().strip() + else: + log.info( + "No pass (%s) found for wallet (%s)", + pass_filename, + wallet_filepath, + ) + + # TODO: error out if not interactive + password = getpass('Enter wallet encryption passphrase: ') + + if not confirm: + return password + + password2 = getpass('Reenter wallet encryption passphrase: ') + if password2 != password: + raise ValueError('Passwords did not match') + + return password + + def estimate_tx_fee(ins, outs, txtype='p2pkh'): '''Returns an estimate of the number of satoshis required for a transaction with the given number of inputs and outputs, @@ -118,7 +148,7 @@ def __init__(self, seedarg, max_mix_depth=2, gaplimit=6, - extend_mixdepth=False, + extend_mixdepth=True, storepassword=False): super(Wallet, self).__init__() self.max_mix_depth = max_mix_depth @@ -131,6 +161,7 @@ def __init__(self, self.imported_privkeys = {} self.seed = self.read_wallet_file_data(seedarg) if extend_mixdepth and len(self.index_cache) > max_mix_depth: + # keep us from dropping parts of the index_cache self.max_mix_depth = len(self.index_cache) self.gaplimit = gaplimit master = btc.bip32_master_key(self.seed, (btc.MAINNET_PRIVATE if @@ -178,10 +209,8 @@ def read_wallet_file_data(self, filename, pwd=None): self.max_mix_depth - len(self.index_cache)) decrypted = False while not decrypted: - if pwd: - password = pwd - else: - password = getpass('Enter wallet decryption passphrase: ') + password = get_wallet_pass(wallet_filepath=path, password=pwd) + password_key = btc.bin_dbl_sha256(password) encrypted_seed = walletdata['encrypted_seed'] try: diff --git a/joinmarket/yieldgenerator.py b/joinmarket/yieldgenerator.py index 8f95b6c8..81402fc8 100644 --- a/joinmarket/yieldgenerator.py +++ b/joinmarket/yieldgenerator.py @@ -1,8 +1,9 @@ -#! /usr/bin/env python +#!/usr/bin/env python from __future__ import absolute_import, print_function import datetime import os +import sys import time import abc from optparse import OptionParser @@ -16,6 +17,7 @@ log = get_log() +# TODO: move all definitions of this into one place MAX_MIX_DEPTH = 5 # is a maker for the purposes of generating a yield from held @@ -141,11 +143,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer' 'https://github.com/chris-belcher/joinmarket/wiki/Running' '-JoinMarket-with-Bitcoin-Core-full-node') print(c) - ret = raw_input('\nContinue? (y/n):') - if ret[0] != 'y': - return + if sys.stdout.isatty(): + ret = raw_input('\nContinue? (y/n):') + if ret[0] != 'y': + return - wallet = Wallet(seed, max_mix_depth=MAX_MIX_DEPTH, gaplimit=gaplimit) + wallet = Wallet(seed, max_mix_depth=MAX_MIX_DEPTH, gaplimit=gaplimit, extend_mixdepth=True) sync_wallet(wallet, fast=options.fastsync) mcs = [IRCMessageChannel(c, realname='btcint=' + jm_single().config.get( diff --git a/ob-watcher.py b/ob-watcher.py old mode 100644 new mode 100755 diff --git a/patientsendpayment.py b/patientsendpayment.py old mode 100644 new mode 100755 diff --git a/sendpayment.py b/sendpayment.py old mode 100644 new mode 100755 diff --git a/tumbler.py b/tumbler.py old mode 100644 new mode 100755 diff --git a/wallet-tool.py b/wallet-tool.py old mode 100644 new mode 100755 index a136f4a6..74120f47 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -10,7 +10,7 @@ from joinmarket import load_program_config, get_network, Wallet, encryptData, \ get_p2pk_vbyte, jm_single, mn_decode, mn_encode, BitcoinCoreInterface, \ - JsonRpcError, sync_wallet + JsonRpcError, sync_wallet, get_wallet_pass import bitcoin as btc @@ -219,18 +219,6 @@ def cus_print(s): sys.exit(0) seed = mn_decode(words) print(seed) - password = getpass.getpass('Enter wallet encryption passphrase: ') - password2 = getpass.getpass('Reenter wallet encryption passphrase: ') - if password != password2: - print('ERROR. Passwords did not match') - sys.exit(0) - password_key = btc.bin_dbl_sha256(password) - encrypted_seed = encryptData(password_key, seed.decode('hex')) - timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") - walletfile = json.dumps({'creator': 'joinmarket project', - 'creation_time': timestamp, - 'encrypted_seed': encrypted_seed.encode('hex'), - 'network': get_network()}) default_walletname = 'wallet.json' walletpath = os.path.join('wallets', default_walletname) @@ -252,6 +240,16 @@ def cus_print(s): print('ERROR: ' + walletpath + ' already exists. Aborting.') sys.exit(0) else: + password = get_wallet_pass(confirm=True, wallet_filepath=walletpath) + password_key = btc.bin_dbl_sha256(password) + encrypted_seed = encryptData(password_key, seed.decode('hex')) + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + walletfile = json.dumps({'creator': 'joinmarket project', + 'creation_time': timestamp, + 'encrypted_seed': encrypted_seed.encode('hex'), + 'network': get_network()}) + + fd = open(walletpath, 'w') fd.write(walletfile) fd.close() diff --git a/yg-pe.py b/yg-pe.py old mode 100644 new mode 100755 diff --git a/yield-generator-basic.py b/yield-generator-basic.py old mode 100644 new mode 100755 From 010e9ccd71f1434162df7c959ba9c1c82bb529ae Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Fri, 24 Mar 2017 00:33:56 -0700 Subject: [PATCH 2/4] partial revert of 72537e24464e76aac55f3d690a49797139c49d52 --- orderbook.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orderbook.html b/orderbook.html index c762bd5a..3d324fb8 100644 --- a/orderbook.html +++ b/orderbook.html @@ -4,6 +4,8 @@ PAGETITLE + + +