diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..32c9f031 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git/ +env/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6e63e747 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +# +# Quick and easy joinmarket +# +# docker build -t "bwstitt/joinmarket:latest" . +# +# Copying of the code is delayed as long as possible so that rebuilding the container while developing is faster +# This also means some of the install steps aren't in the most obvious order and the virtualenv is outside the code +# + +FROM bwstitt/python-jessie:python2 + +# Install packages for joinmarket +RUN docker-apt-install \ + gcc \ + libsodium13 \ + python-dev + +# i needed these when compiling myself, but new versions of pip with wheels save us + #libatlas-dev \ + #libblas-dev \ + #libfreetype6-dev \ + #libpng12-dev \ + #libsodium-dev \ + #pkg-config \ + #python-dev \ + +# install deps that don't depend on the code as the user and fix /pyenv/local/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment. +# TODO: double check that this actually builds the font caches +RUN chroot --userspec=abc / pip install \ + "matplotlib==2.0.0" \ + "scipy==0.19.0" \ + && chroot --userspec=abc / python -c \ + "from matplotlib.font_manager import FontManager; print(FontManager())" + +# copy requirements before code. this will make the image builds faster when code changes but requirements don't +COPY requirements.txt /src/ +RUN chroot --userspec=abc / pip install -r /src/requirements.txt + +# install the code +# todo: i wish copy would keep the user... +COPY . /src/ +WORKDIR /src +RUN chown -R abc:abc . + +# setup data volumes for logs and wallets +# todo: handle the blacklist and commitments +VOLUME /src/logs /src/wallets + +USER abc +ENV MPLBACKEND=agg +ENTRYPOINT ["python", "/src/docker/entrypoint.py"] +CMD ["ob_watcher", "-H", "0.0.0.0"] + +EXPOSE 62601 diff --git a/broadcast-tx.py b/broadcast-tx.py old mode 100644 new mode 100755 diff --git a/check-config.py b/check-config.py new file mode 100755 index 00000000..d31f874a --- /dev/null +++ b/check-config.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import + +import sys + +from joinmarket import get_log, load_program_config + + +log = get_log() + + +def main(): + """Simple command to make sure the config loads. + + This will exit 1 if the config cannot be loaded or the blockchaininterface + doesn't respond. + """ + try: + load_program_config() + except Exception: + log.exception("Error while loading config") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/create-unsigned-tx.py b/create-unsigned-tx.py old mode 100644 new mode 100755 diff --git a/docker/README b/docker/README new file mode 100644 index 00000000..b1ac0163 --- /dev/null +++ b/docker/README @@ -0,0 +1,6 @@ +todo: + - explain how to setup Tor + - explain how to setup bitcoind + - explain how to setup the config (and figure out how to keep sensitive things out of git) + - maybe use click instead of argparse + - use --entry-help instead of --help so we can still see joinmarket's help diff --git a/docker/entrypoint.py b/docker/entrypoint.py new file mode 100755 index 00000000..dc6409eb --- /dev/null +++ b/docker/entrypoint.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python2 +"""Entrypoints for Docker containers to run joinmarket. + +todo: wait_for_file should probably be a decorator +todo: either always force wallet.json or make waiting for the wallet smarter +""" +import argparse +import logging +import os +import os.path +import subprocess +import sys +import time + + +# +# Helpers +# + +DEBUG_LOG_FORMAT = ( + '-' * 80 + '\n' + + '%(asctime)s %(levelname)s in %(name)s @ %(threadName)s:\n' + + '%(pathname)s:%(lineno)d\n' + + '%(message)s\n' + + '-' * 80 +) +DEFAULT_LOG_FORMAT = '%(asctime)s %(levelname)s: %(threadName)s: %(name)s: %(message)s' # noqa + +log = logging.getLogger(__name__) + + +def run(*args): + """Run a python command inside the joinmarket virtualenv. + + Raises subprocess.CalledProcessError if the command fails. + """ + if not args: + raise ValueError("run needs at least one arg") + + command_list = [sys.executable] + map(str, args) + + log.info("Running %s...", command_list[1]) + log.debug("Full command: %s", command_list) + + return subprocess.check_call(command_list, env=os.environ) + + +def run_or_exit(*args): + """Run a python command inside the joinmarket virtualenv. + + Logs and exits if the command fails. + """ + try: + return run(*args) + except subprocess.CalledProcessError as e: + log.error("%s", e) + sys.exit(e.returncode) + + +def wait_for_config(*args): + """Sleep until config loads. + + Config loading includes bitcoind responding to getblockchaininfo + + Todo: exponential backoff of the sleep. maybe log less, too + Todo: args here are hacky. make this function and the command seperate + """ + while True: + try: + run('check-config.py') + except subprocess.CalledProcessError as e: + # TODO: this is too verbose + log.error("Unable to load config: %s. Sleeping..." % e) + time.sleep(60) + else: + break + + return True + + +def wait_for_file(filename, sleep=10): + """Sleep until a given file exists.""" + if os.path.exists(filename): + return + + log.info("'%s' does not exist. Check the README", filename) + + log.info("Sleeping until '%s' exists...", filename) + while not os.path.exists(filename): + time.sleep(sleep) + + log.info("Found '%s'", filename) + return + + +# +# Commands +# + +def get_parser(): + """Create an argument parser that routes to the command functions.""" + # create the top-level parser + parser = argparse.ArgumentParser() + # todo: configurable log level + subparsers = parser.add_subparsers() + + # create the parser for the "maker" command + parser_maker = subparsers.add_parser('maker') + parser_maker.set_defaults(func=maker) + + # create the parser for the "ob_watcher" command + parser_ob_watcher = subparsers.add_parser('ob_watcher') + parser_ob_watcher.set_defaults(func=ob_watcher) + + # create the parser for the "sendpayment" command + parser_sendpayment = subparsers.add_parser('sendpayment') + parser_sendpayment.set_defaults(func=sendpayment) + + # create the parser for the "taker" command + parser_tumbler = subparsers.add_parser('tumbler') + parser_tumbler.set_defaults(func=tumbler) + + # create the parser for the "wallet_tool" command + parser_wallet_tool = subparsers.add_parser('wallet_tool') + parser_wallet_tool.set_defaults(func=wallet_tool) + + # other scripts might find waiting for the config helpful, too + parser_wait_for_config = subparsers.add_parser('wait_for_config') + parser_wait_for_config.set_defaults(func=wait_for_config) + + return parser + + +def maker(args): + """Earn Bitcoins and privacy.""" + wallet_filename = 'wallet.json' + wait_for_file("wallets/%s" % wallet_filename) + + # wait for bitcoind to respond + wait_for_config() + + run_or_exit('yg-pe.py', wallet_filename, *args) + + +def ob_watcher(args): + """Watch the orderbook.""" + # wait for bitcoind to respond + # todo: although, why does the orderbook need bitcoind? + wait_for_config() + + run_or_exit('ob-watcher.py', *args) + + +def sendpayment(args): + """"Send Bitcoins with privacy. + + todo: make sure we only sendpayment with coins that have already been + joined at least once. + """ + wallet_filename = 'wallet.json' + wait_for_file("wallets/%s" % wallet_filename) + + # wait for bitcoind to respond + wait_for_config() + + run_or_exit('sendpayment.py', *args) + + +def tumbler(args): + """"Send Bitcoins with layers of privacy.""" + wallet_filename = 'wallet.json' + wait_for_file("wallets/%s" % wallet_filename) + + # wait for bitcoind to respond + wait_for_config() + + run_or_exit('tumbler.py', *args) + + +def wallet_tool(args): + """Inspect and manage your Bitcoin wallet.""" + run_or_exit('wallet-tool.py', *args) + + +def main(): + """Manage joinmarket.""" + parser = get_parser() + args, passthrough_args = parser.parse_known_args() + + log_level = logging.DEBUG # todo: get log_level from args + + if log_level == logging.DEBUG: + log_format = DEBUG_LOG_FORMAT + else: + log_format = DEFAULT_LOG_FORMAT + + logging.basicConfig( + format=log_format, + level=log_level, + stream=sys.stdout, + ) + log.debug("Hello!") + + args.func(passthrough_args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/joinmarket/__init__.py b/joinmarket/__init__.py index 70c791cc..1f659a1c 100644 --- a/joinmarket/__init__.py +++ b/joinmarket/__init__.py @@ -16,7 +16,7 @@ from .slowaes import decryptData, encryptData from .taker import Taker, OrderbookWatch, CoinJoinTX from .wallet import AbstractWallet, BitcoinCoreInterface, Wallet, \ - BitcoinCoreWallet + BitcoinCoreWallet, get_wallet_pass from .configure import load_program_config, jm_single, get_p2pk_vbyte, \ get_network, jm_single, get_network, validate_address, get_irc_mchannels, \ check_utxo_blacklist diff --git a/joinmarket/blockchaininterface.py b/joinmarket/blockchaininterface.py index d0241c11..584f89c2 100644 --- a/joinmarket/blockchaininterface.py +++ b/joinmarket/blockchaininterface.py @@ -947,7 +947,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print('restart Bitcoin Core with -rescan if you\'re ' 'recovering an existing wallet from backup seed') print(' otherwise just restart this joinmarket script') - sys.exit(0) + sys.exit(2) def sync_wallet(self, wallet, fast=False): #trigger fast sync if the index_cache is available diff --git a/joinmarket/jsonrpc.py b/joinmarket/jsonrpc.py index d12243f6..2ffa07d9 100644 --- a/joinmarket/jsonrpc.py +++ b/joinmarket/jsonrpc.py @@ -34,6 +34,9 @@ def __init__(self, obj): self.code = obj["code"] self.message = obj["message"] + def __str__(self): + return "%s %s: %s" % (self.__class__.__name__, self.code, self.message) + class JsonRpcConnectionError(Exception): """ @@ -78,19 +81,18 @@ def queryHTTP(self, obj): conn.request("POST", "", body, headers) response = conn.getresponse() - if response.status == 401: + if response.status in [401, 403]: conn.close() raise JsonRpcConnectionError( "authentication for JSON-RPC failed") - # All of the codes below are 'fine' from a JSON-RPC point of view. - if response.status not in [200, 404, 500]: - conn.close() - raise JsonRpcConnectionError("unknown error in JSON-RPC") - data = response.read() conn.close() + # All of the codes below are 'fine' from a JSON-RPC point of view. + if response.status not in [200, 404, 500]: + raise JsonRpcConnectionError("unknown error %s in JSON-RPC: %s" % (response.status, data)) + return json.loads(data) except JsonRpcConnectionError as exc: diff --git a/joinmarket/wallet.py b/joinmarket/wallet.py index 3ff97f6c..ee8aff0a 100644 --- a/joinmarket/wallet.py +++ b/joinmarket/wallet.py @@ -18,6 +18,36 @@ log = get_log() + +def get_wallet_pass(wallet_filepath=None, password=None, confirm=False): + if password is not None: + return password + + if wallet_filepath is not None: + pass_filename = wallet_filepath[:-4] + 'pass' + if os.path.exists(pass_filename): + with open(pass_filename, 'r') as f: + return f.read().strip() + else: + log.info( + "No pass (%s) found for wallet (%s)", + pass_filename, + wallet_filepath, + ) + + # TODO: error out if not interactive + password = getpass('Enter wallet encryption passphrase: ') + + if not confirm: + return password + + password2 = getpass('Reenter wallet encryption passphrase: ') + if password2 != password: + raise ValueError('Passwords did not match') + + return password + + def estimate_tx_fee(ins, outs, txtype='p2pkh'): '''Returns an estimate of the number of satoshis required for a transaction with the given number of inputs and outputs, @@ -118,7 +148,7 @@ def __init__(self, seedarg, max_mix_depth=2, gaplimit=6, - extend_mixdepth=False, + extend_mixdepth=True, storepassword=False): super(Wallet, self).__init__() self.max_mix_depth = max_mix_depth @@ -131,6 +161,7 @@ def __init__(self, self.imported_privkeys = {} self.seed = self.read_wallet_file_data(seedarg) if extend_mixdepth and len(self.index_cache) > max_mix_depth: + # keep us from dropping parts of the index_cache self.max_mix_depth = len(self.index_cache) self.gaplimit = gaplimit master = btc.bip32_master_key(self.seed, (btc.MAINNET_PRIVATE if @@ -178,10 +209,8 @@ def read_wallet_file_data(self, filename, pwd=None): self.max_mix_depth - len(self.index_cache)) decrypted = False while not decrypted: - if pwd: - password = pwd - else: - password = getpass('Enter wallet decryption passphrase: ') + password = get_wallet_pass(wallet_filepath=path, password=pwd) + password_key = btc.bin_dbl_sha256(password) encrypted_seed = walletdata['encrypted_seed'] try: diff --git a/joinmarket/yieldgenerator.py b/joinmarket/yieldgenerator.py index 8f95b6c8..81402fc8 100644 --- a/joinmarket/yieldgenerator.py +++ b/joinmarket/yieldgenerator.py @@ -1,8 +1,9 @@ -#! /usr/bin/env python +#!/usr/bin/env python from __future__ import absolute_import, print_function import datetime import os +import sys import time import abc from optparse import OptionParser @@ -16,6 +17,7 @@ log = get_log() +# TODO: move all definitions of this into one place MAX_MIX_DEPTH = 5 # is a maker for the purposes of generating a yield from held @@ -141,11 +143,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer' 'https://github.com/chris-belcher/joinmarket/wiki/Running' '-JoinMarket-with-Bitcoin-Core-full-node') print(c) - ret = raw_input('\nContinue? (y/n):') - if ret[0] != 'y': - return + if sys.stdout.isatty(): + ret = raw_input('\nContinue? (y/n):') + if ret[0] != 'y': + return - wallet = Wallet(seed, max_mix_depth=MAX_MIX_DEPTH, gaplimit=gaplimit) + wallet = Wallet(seed, max_mix_depth=MAX_MIX_DEPTH, gaplimit=gaplimit, extend_mixdepth=True) sync_wallet(wallet, fast=options.fastsync) mcs = [IRCMessageChannel(c, realname='btcint=' + jm_single().config.get( diff --git a/ob-watcher.py b/ob-watcher.py old mode 100644 new mode 100755 index d5960ffb..e1fa6b80 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -27,9 +27,6 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt -shutdownform = '
' -shutdownpage = '

Successfully Shut down

' - refresh_orderbook_form = '
' sorted_units = ('BTC', 'mBTC', 'μBTC', 'satoshi') unit_to_power = {'BTC': 8, 'mBTC': 5, 'μBTC': 2, 'satoshi': 0} @@ -260,7 +257,7 @@ def do_GET(self): (str(ordercount) + ' orders found by ' + self.get_counterparty_count() + ' counterparties' + alert_msg), 'MAINBODY': ( - shutdownform + refresh_orderbook_form + choose_units_form + + refresh_orderbook_form + choose_units_form + table_heading + ordertable + '\n') } elif self.path == '/ordersize': @@ -300,18 +297,7 @@ def do_GET(self): self.wfile.write(orderbook_page) def do_POST(self): - pages = ['/shutdown', '/refreshorderbook'] - if self.path not in pages: - return - if self.path == '/shutdown': - self.taker.msgchan.shutdown() - self.send_response(200) - self.send_header('Content-Type', 'text/html') - self.send_header('Content-Length', len(shutdownpage)) - self.end_headers() - self.wfile.write(shutdownpage) - self.base_server.__shutdown_request = True - elif self.path == '/refreshorderbook': + if self.path == '/refreshorderbook': self.taker.msgchan.request_orderbook() time.sleep(5) self.path = '/' diff --git a/orderbook.html b/orderbook.html index c762bd5a..3d324fb8 100644 --- a/orderbook.html +++ b/orderbook.html @@ -4,6 +4,8 @@ PAGETITLE + + +