diff --git a/lendingbot.py b/lendingbot.py index 19c875e8..0dccabb3 100755 --- a/lendingbot.py +++ b/lendingbot.py @@ -5,8 +5,13 @@ import time import traceback from decimal import Decimal -from httplib import BadStatusLine -from urllib2 import URLError +try: + from httplib import BadStatusLine + from urllib2 import URLError +except ImportError: + # Python 3 + from http.client import BadStatusLine + from urllib.error import URLError import modules.Configuration as Config import modules.Data as Data @@ -82,7 +87,7 @@ # load plugins PluginsManager.init(Config, api, log, notify_conf) -print 'Welcome to ' + Config.get("BOT", "label", "Lending Bot") + ' on ' + exchange +print("Welcome to {0} on {1}".format(Config.get("BOT", "label", "Lending Bot"), exchange)) try: while True: @@ -102,37 +107,39 @@ # allow existing the main bot loop raise except Exception as ex: + if not hasattr(ex, 'message'): + ex.message = str(ex) log.log_error(ex.message) log.persistStatus() if 'Invalid API key' in ex.message: - print "!!! Troubleshooting !!!" - print "Are your API keys correct? No quotation. Just plain keys." + print("!!! Troubleshooting !!!") + print("Are your API keys correct? No quotation. Just plain keys.") exit(1) elif 'Nonce must be greater' in ex.message: - print "!!! Troubleshooting !!!" - print "Are you reusing the API key in multiple applications? Use a unique key for every application." + print("!!! Troubleshooting !!!") + print("Are you reusing the API key in multiple applications? Use a unique key for every application.") exit(1) elif 'Permission denied' in ex.message: - print "!!! Troubleshooting !!!" - print "Are you using IP filter on the key? Maybe your IP changed?" + print("!!! Troubleshooting !!!") + print("Are you using IP filter on the key? Maybe your IP changed?") exit(1) elif 'timed out' in ex.message: - print "Timed out, will retry in " + str(Lending.get_sleep_time()) + "sec" + print("Timed out, will retry in {0} sec".format(Lending.get_sleep_time())) elif isinstance(ex, BadStatusLine): - print "Caught BadStatusLine exception from Poloniex, ignoring." + print("Caught BadStatusLine exception from Poloniex, ignoring.") elif 'Error 429' in ex.message: - additional_sleep = max(130.0-Lending.get_sleep_time(), 0) + additional_sleep = max(130.0 - Lending.get_sleep_time(), 0) sum_sleep = additional_sleep + Lending.get_sleep_time() log.log_error('IP has been banned due to many requests. Sleeping for {} seconds'.format(sum_sleep)) time.sleep(additional_sleep) # Ignore all 5xx errors (server error) as we can't do anything about it (https://httpstatuses.com/) elif isinstance(ex, URLError): - print "Caught {0} from exchange, ignoring.".format(ex.message) + print("Caught {0} from exchange, ignoring.".format(ex.message)) elif isinstance(ex, ApiError): - print "Caught {0} reading from exchange API, ignoring.".format(ex.message) + print("Caught {0} reading from exchange API, ignoring.".format(ex.message)) else: - print traceback.format_exc() - print "Unhandled error, please open a Github issue so we can fix it!" + print(traceback.format_exc()) + print("Unhandled error, please open a Github issue so we can fix it!") if notify_conf['notify_caught_exception']: log.notify("{0}\n-------\n{1}".format(ex, traceback.format_exc()), notify_conf) sys.stdout.flush() @@ -144,5 +151,5 @@ WebServer.stop_web_server() PluginsManager.on_bot_exit() log.log('bye') - print 'bye' + print('bye') os._exit(0) # Ad-hoc solution in place of 'exit(0)' TODO: Find out why non-daemon thread(s) are hanging on exit diff --git a/modules/Bitfinex.py b/modules/Bitfinex.py index 49278156..b4cb0fc1 100644 --- a/modules/Bitfinex.py +++ b/modules/Bitfinex.py @@ -20,8 +20,8 @@ def __init__(self, cfg, log): self.log = log self.lock = threading.RLock() self.req_per_min = 60 - self.req_period = 15 # seconds - self.req_per_period = int(self.req_per_min / ( 60.0 / self.req_period)) + self.req_period = 15 # seconds + self.req_per_period = int(self.req_per_min / (60.0 / self.req_period)) self.req_time_log = RingBuffer(self.req_per_period) self.url = 'https://api.bitfinex.com' self.key = self.cfg.get("API", "apikey", None) @@ -51,7 +51,7 @@ def limit_request_rate(self): time_since_oldest_req = now - self.req_time_log[0] # check if oldest request is more than self.req_period ago if time_since_oldest_req < self.req_period: - # print self.req_time_log.get() + # print(self.req_time_log.get()) # uncomment to debug # print("Waiting {0} sec, {1} to keep api request rate".format(self.req_period - time_since_oldest_req, # threading.current_thread())) @@ -61,7 +61,7 @@ def limit_request_rate(self): return # uncomment to debug # else: - # print self.req_time_log.get() + # print(self.req_time_log.get()) # print("Not Waiting {0}".format(threading.current_thread())) # print("Req:{0} Oldest req:{1} Diff:{2} sec".format(now, self.req_time_log[0], time_since_oldest_req)) # append current request time to the log, pushing out the 60th request time before it @@ -103,6 +103,9 @@ def _request(self, method, request, payload=None, verify=True): return r.json() + except ApiError as ex: + ex.message = "{0} Requesting {1}".format(str(ex), self.url + request) + raise ex except Exception as ex: ex.message = ex.message if ex.message else str(ex) ex.message = "{0} Requesting {1}".format(ex.message, self.url + request) @@ -259,7 +262,7 @@ def create_loan_offer(self, currency, amount, duration, auto_renew, lending_rate payload = { "currency": currency, "amount": str(amount), - "rate": str(round(float(lending_rate),10) * 36500), + "rate": str(round(float(lending_rate),10) * 36500), "period": int(duration), "direction": "lend" } @@ -346,7 +349,7 @@ def return_lending_history(self, start, stop, limit=500): "amount": "0.0", "duration": "0.0", "interest": str(amount / 0.85), - "fee": str(amount-amount / 0.85), + "fee": str(amount - amount / 0.85), "earned": str(amount), "open": Bitfinex2Poloniex.convertTimestamp(entry['timestamp']), "close": Bitfinex2Poloniex.convertTimestamp(entry['timestamp']) diff --git a/modules/Bitfinex2Poloniex.py b/modules/Bitfinex2Poloniex.py index 90b44dbf..4af7dad5 100644 --- a/modules/Bitfinex2Poloniex.py +++ b/modules/Bitfinex2Poloniex.py @@ -28,7 +28,7 @@ def convertOpenLoanOffers(bfxOffers): if offer['direction'] == 'lend' and float(offer['remaining_amount']) > 0: plxOffers[offer['currency']].append({ "id": offer['id'], - "rate": str(float(offer['rate'])/36500), + "rate": str(float(offer['rate']) / 36500), "amount": offer['remaining_amount'], "duration": offer['period'], "autoRenew": 0, diff --git a/modules/Configuration.py b/modules/Configuration.py index e2404321..f97a4840 100755 --- a/modules/Configuration.py +++ b/modules/Configuration.py @@ -1,8 +1,13 @@ # coding=utf-8 -from ConfigParser import SafeConfigParser +try: + from ConfigParser import SafeConfigParser +except ImportError: + # Python 3 + from configparser import SafeConfigParser import json import os from decimal import Decimal +from builtins import input config = SafeConfigParser() Data = None @@ -19,9 +24,9 @@ def init(file_location, data=None): # Copy default config file if not found try: shutil.copy('default.cfg.example', file_location) - print '\ndefault.cfg.example has been copied to ' + file_location + '\n' \ - 'Edit it with your API key and custom settings.\n' - raw_input("Press Enter to acknowledge and exit...") + print("\ndefault.cfg.example has been copied to ".format(file_location)) + print("Edit it with your API key and custom settings.\n") + input("Press Enter to acknowledge and exit...") exit(1) except Exception as ex: ex.message = ex.message if ex.message else str(ex) @@ -33,14 +38,16 @@ def init(file_location, data=None): def has_option(category, option): try: return True if os.environ["{0}_{1}".format(category, option)] else _ - except (KeyError, NameError): # KeyError for no env var, NameError for _ (empty var) and then to continue + except (KeyError, NameError): # KeyError for no env var, NameError for _ (empty var) and then to continue return config.has_option(category, option) def getboolean(category, option, default_value=False): if has_option(category, option): try: - return bool(os.environ["{0}_{1}".format(category, option)]) + v = os.environ["{0}_{1}".format(category, option)] + return v.lower() in ['true', '1', 't', 'y', 'yes'] + except KeyError: return config.getboolean(category, option) else: @@ -55,22 +62,24 @@ def get(category, option, default_value=False, lower_limit=False, upper_limit=Fa value = config.get(category, option) try: if lower_limit and float(value) < float(lower_limit): - print "WARN: [%s]-%s's value: '%s' is below the minimum limit: %s, which will be used instead." % \ - (category, option, value, lower_limit) + print("WARN: [{0}]-{1}'s value: '{2}' is below the minimum limit: {3}, which will be used instead." + .format(category, option, value, lower_limit)) value = lower_limit if upper_limit and float(value) > float(upper_limit): - print "WARN: [%s]-%s's value: '%s' is above the maximum limit: %s, which will be used instead." % \ - (category, option, value, upper_limit) + print("WARN: [{0}]-{1}'s value: '{2}' is above the maximum limit: {3}, which will be used instead." + .format(category, option, value, upper_limit)) value = upper_limit return value except ValueError: if default_value is None: - print "ERROR: [%s]-%s is not allowed to be left empty. Please check your config." % (category, option) + print("ERROR: [{0}]-{1} is not allowed to be left empty. Please check your config." + .format(category, option)) exit(1) return default_value else: if default_value is None: - print "ERROR: [%s]-%s is not allowed to be left unset. Please check your config." % (category, option) + print("ERROR: [{0}]-{1} is not allowed to be left unset. Please check your config." + .format(category, option)) exit(1) return default_value @@ -162,8 +171,8 @@ def get_gap_mode(category, option): full_list = ['raw', 'rawbtc', 'relative'] value = get(category, 'gapmode', False).lower().strip(" ") if value not in full_list: - print "ERROR: Invalid entry '%s' for [%s]-gapMode. Please check your config. Allowed values are: %s" % \ - (value, category, ", ".join(full_list)) + print("ERROR: Invalid entry '{0}' for [{1}]-gapMode. Please check your config. Allowed values are: {2}" + .format(value, category, ", ".join(full_list))) exit(1) return value.lower() else: @@ -193,8 +202,8 @@ def get_notification_config(): notify_conf = {'enable_notifications': config.has_section('notifications')} # For boolean parameters - for conf in ['notify_tx_coins', 'notify_xday_threshold', 'notify_new_loans', 'notify_caught_exception', 'email', 'slack', 'telegram', - 'pushbullet', 'irc']: + for conf in ['notify_tx_coins', 'notify_xday_threshold', 'notify_new_loans', 'notify_caught_exception', 'email', + 'slack', 'telegram', 'pushbullet', 'irc']: notify_conf[conf] = getboolean('notifications', conf) # For string-based parameters @@ -239,5 +248,5 @@ def get_notification_config(): def get_plugins_config(): active_plugins = [] if config.has_option("BOT", "plugins"): - active_plugins = map(str.strip, config.get("BOT", "plugins").split(',')) + active_plugins = list(map(str.strip, config.get("BOT", "plugins").split(','))) return active_plugins diff --git a/modules/ConsoleUtils.py b/modules/ConsoleUtils.py index c579a722..c3596df6 100644 --- a/modules/ConsoleUtils.py +++ b/modules/ConsoleUtils.py @@ -5,6 +5,7 @@ import platform import subprocess + def get_terminal_size(): """ getTerminalSize() - get width and height of console @@ -45,6 +46,7 @@ def _get_terminal_size_windows(): except: pass + def _get_terminal_size_tput(): # get terminal width # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window diff --git a/modules/Data.py b/modules/Data.py index f4f46e29..fad74bbd 100644 --- a/modules/Data.py +++ b/modules/Data.py @@ -1,6 +1,11 @@ import datetime from decimal import Decimal -from urllib import urlopen +try: + from urllib import urlopen +except ImportError: + # Python 3 + from urllib.request import urlopen + import json api = None @@ -27,7 +32,7 @@ def get_max_duration(end_date, context): return "" try: now_time = datetime.date.today() - config_date = map(int, end_date.split(',')) + config_date = list(map(int, end_date.split(','))) end_time = datetime.date(*config_date) # format YEAR,MONTH,DAY all ints, also used splat operator diff_days = (end_time - now_time).days if context == "order": @@ -35,7 +40,7 @@ def get_max_duration(end_date, context): if context == "status": return " - Days Remaining: " + str(diff_days) # Status needs string except Exception as ex: - ex.message = ex.message if ex.message else str(ex) + ex.message = ex.message if hasattr(ex, 'message') and ex.message else str(ex) print("ERROR: There is something wrong with your endDate option. Error: {0}".format(ex.message)) exit(1) @@ -45,10 +50,8 @@ def get_total_lent(): total_lent = {} rate_lent = {} for item in crypto_lent["provided"]: - item_str = item["amount"].encode("utf-8") - item_float = Decimal(item_str) - item_rate_str = item["rate"].encode("utf-8") - item_rate_float = Decimal(item_rate_str) + item_float = Decimal(item["amount"]) + item_rate_float = Decimal(item["rate"]) if item["currency"] in total_lent: crypto_lent_sum = total_lent[item["currency"]] + item_float crypto_lent_rate = rate_lent[item["currency"]] + (item_rate_float * item_float) diff --git a/modules/ExchangeApi.py b/modules/ExchangeApi.py index 6ef3f10e..e3b158e1 100644 --- a/modules/ExchangeApi.py +++ b/modules/ExchangeApi.py @@ -3,13 +3,13 @@ """ import abc +import six import calendar import time +@six.add_metaclass(abc.ABCMeta) class ExchangeApi(object): - __metaclass__ = abc.ABCMeta - def __str__(self): return self.__class__.__name__.upper() diff --git a/modules/Lending.py b/modules/Lending.py index 4f234ab9..8f0ea709 100644 --- a/modules/Lending.py +++ b/modules/Lending.py @@ -1,5 +1,6 @@ # coding=utf-8 from decimal import Decimal +from six import iteritems import sched import time import threading @@ -58,7 +59,7 @@ def init(cfg, api1, log1, data, maxtolend, dry_run1, analysis, notify_conf1): global sleep_time, sleep_time_active, sleep_time_inactive, min_daily_rate, max_daily_rate, spread_lend, \ gap_bottom_default, gap_top_default, xday_threshold, xday_spread, xdays, min_loan_size, end_date, coin_cfg, \ min_loan_sizes, dry_run, transferable_currencies, keep_stuck_orders, hide_coins, scheduler, gap_mode_default, \ - exchange, analysis_method, currencies_to_analyse + exchange, analysis_method, currencies_to_analyse exchange = Config.get_exchange() @@ -131,17 +132,18 @@ def notify_new_loans(sleep_time): if loans_provided: # function to return a set of ids from the api result # get_id_set = lambda loans: set([x['id'] for x in loans]) - def get_id_set(loans): return set([x['id'] for x in loans]) + def get_id_set(loans): + return set([x['id'] for x in loans]) loans_amount = {} loans_info = {} for loan_id in get_id_set(new_provided) - get_id_set(loans_provided): loan = [x for x in new_provided if x['id'] == loan_id][0] # combine loans with the same rate - k = 'c'+loan['currency']+'r'+loan['rate']+'d'+str(loan['duration']) + k = 'c' + loan['currency'] + 'r' + loan['rate'] + 'd' + str(loan['duration']) loans_amount[k] = float(loan['amount']) + (loans_amount[k] if k in loans_amount else 0) loans_info[k] = loan # send notifications with the grouped info - for k, amount in loans_amount.iteritems(): + for k, amount in iteritems(loans_amount): loan = loans_info[k] t = "{0} {1} loan filled for {2} days at a rate of {3:.4f}%" text = t.format(amount, loan['currency'], loan['duration'], float(loan['rate']) * 100) @@ -175,7 +177,7 @@ def create_lend_offer(currency, amt, rate): if Config.has_option('BOT', 'endDate'): days_remaining = int(Data.get_max_duration(end_date, "order")) if int(days_remaining) <= 2: - print "endDate reached. Bot can no longer lend.\nExiting..." + print("endDate reached. Bot can no longer lend.\nExiting...") log.log("The end date has almost been reached and the bot can no longer lend. Exiting.") log.refreshStatus(Data.stringify_total_lent(*Data.get_total_lent()), Data.get_max_duration( end_date, "status")) @@ -218,7 +220,7 @@ def cancel_all(): ex.message = ex.message if ex.message else str(ex) log.log("Error canceling loan offer: {0}".format(ex.message)) else: - print "Not enough " + CUR + " to lend if bot canceled open orders. Not cancelling." + print("Not enough {0} to lend if bot canceled open orders. Not cancelling.".format(CUR)) def lend_all(): @@ -378,12 +380,12 @@ def get_gap_mode_rates(cur, cur_active_bal, cur_total_balance, ticker): top_rate = get_gap_rate(cur, gap_top, order_book, cur_total_balance) else: if use_gap_cfg: - print "WARN: Invalid setting for gapMode for [%s], using defaults..." % cur + print("WARN: Invalid setting for gapMode for [{0}], using defaults...".format(cur)) coin_cfg[cur]['gapmode'] = "rawbtc" coin_cfg[cur]['gapbottom'] = 10 coin_cfg[cur]['gaptop'] = 100 else: - print "WARN: Invalid setting for gapMode, using defaults..." + print("WARN: Invalid setting for gapMode, using defaults...") gap_mode_default = "relative" gap_bottom_default = 10 gap_top_default = 200 @@ -457,5 +459,5 @@ def transfer_balances(): log.log(log.digestApiMsg(msg)) log.notify(log.digestApiMsg(msg), notify_conf) if coin not in exchange_balances: - print "WARN: Incorrect coin entered for transferCurrencies: " + coin + print("WARN: Incorrect coin entered for transferCurrencies: ".format(coin)) transferable_currencies.remove(coin) diff --git a/modules/Logger.py b/modules/Logger.py index 0298a1f9..a0faf19a 100644 --- a/modules/Logger.py +++ b/modules/Logger.py @@ -6,10 +6,16 @@ import sys import time -import ConsoleUtils import modules.Configuration as Config -from RingBuffer import RingBuffer -from Notify import send_notification +from builtins import str +try: + import ConsoleUtils + from Notify import send_notification + from RingBuffer import RingBuffer +except ModuleNotFoundError: + from . import ConsoleUtils + from .Notify import send_notification + from .RingBuffer import RingBuffer class ConsoleOutput(object): @@ -61,7 +67,7 @@ def printline(self, line): def writeJsonFile(self): with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f: self.jsonOutput["log"] = self.jsonOutputLog.get() - f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) + f.write(str(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True))) f.close() def addSectionLog(self, section, key, value): @@ -110,7 +116,7 @@ def log_error(self, msg): log_message = "{0} Error {1}".format(self.timestamp(), msg) self.output.printline(log_message) if isinstance(self.output, JsonOutput): - print log_message + print(log_message) self.refreshStatus() def offer(self, amt, cur, rate, days, msg): diff --git a/modules/MarketAnalysis.py b/modules/MarketAnalysis.py index c61d2ce7..bd9d125d 100644 --- a/modules/MarketAnalysis.py +++ b/modules/MarketAnalysis.py @@ -7,6 +7,7 @@ import pandas as pd import sqlite3 as sqlite from sqlite3 import Error +from builtins import range # Bot libs import modules.Configuration as Config @@ -161,7 +162,7 @@ def update_market_thread(self, cur, levels=None): except Exception as ex: self.print_traceback(ex, "Error in returning data from exchange") market_data = [] - for i in xrange(levels): + for i in range(levels): market_data.append(str(raw_data[i]['rate'])) market_data.append(str(raw_data[i]['amount'])) market_data.append('0') # Percentile field not being filled yet. @@ -171,7 +172,7 @@ def insert_into_db(self, db_con, market_data, levels=None): if levels is None: levels = self.recorded_levels insert_sql = "INSERT INTO loans (" - for level in xrange(levels): + for level in range(levels): insert_sql += "rate{0}, amnt{0}, ".format(level) insert_sql += "percentile) VALUES ({0});".format(','.join(market_data)) # percentile = 0 with db_con: @@ -405,7 +406,7 @@ def create_rate_table(self, db_con, levels): cursor = db_con.cursor() create_table_sql = "CREATE TABLE IF NOT EXISTS loans (id INTEGER PRIMARY KEY AUTOINCREMENT," + \ "unixtime integer(4) not null default (strftime('%s','now'))," - for level in xrange(levels): + for level in range(levels): create_table_sql += "rate{0} FLOAT, ".format(level) create_table_sql += "amnt{0} FLOAT, ".format(level) create_table_sql += "percentile FLOAT);" diff --git a/modules/Notify.py b/modules/Notify.py index f2edeb7d..eefb499a 100644 --- a/modules/Notify.py +++ b/modules/Notify.py @@ -1,8 +1,17 @@ # coding=utf-8 -import urllib -import urllib2 import json import smtplib +from six import iteritems +try: + # Python 3 + from urllib.parse import urlencode + from urllib.request import urlopen, Request + from urllib.error import HTTPError + unicode = str +except ImportError: + # Python 2 + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError try: from irc import client IRC_LOADED = True @@ -16,7 +25,7 @@ # Slack post data needs to be encoded in UTF-8 def encoded_dict(in_dict): out_dict = {} - for k, v in in_dict.iteritems(): + for k, v in iteritems(in_dict): if isinstance(v, unicode): v = v.encode('utf8') elif isinstance(v, str): @@ -41,9 +50,9 @@ def check_urlib_response(response, platform): def post_to_slack(msg, channels, token): for channel in channels: post_data = {'text': msg, 'channel': channel, 'token': token} - enc_post_data = urllib.urlencode(encoded_dict(post_data)) + enc_post_data = urlencode(encoded_dict(post_data)) url = 'https://{}/api/{}'.format('slack.com', 'chat.postMessage') - response = urllib2.urlopen(url, enc_post_data) + response = urlopen(url, enc_post_data) check_urlib_response(response, 'slack') @@ -52,9 +61,9 @@ def post_to_telegram(msg, chat_ids, bot_id): post_data = {"chat_id": chat_id, "text": msg} url = "https://api.telegram.org/bot" + bot_id + "/sendMessage" try: - response = urllib2.urlopen(url, urllib.urlencode(post_data)) + response = urlopen(url, urlencode(post_data).encode('utf8')) check_urlib_response(response, 'telegram') - except urllib2.HTTPError as e: + except HTTPError as e: msg = "Your bot id is probably configured incorrectly" raise NotificationException("{0}\n{1}".format(e, msg)) @@ -89,8 +98,8 @@ def send_email(msg, email_login_address, email_login_password, email_smtp_server def post_to_pushbullet(msg, token, deviceid): post_data = {'body': msg, 'device_iden': deviceid, 'title': 'Poloniex Bot', 'type': 'note'} opener = urllib2.build_opener() - req = urllib2.Request('https://api.pushbullet.com/v2/pushes', data=json.dumps(post_data), - headers={'Content-Type': 'application/json', 'Access-Token': token}) + req = Request('https://api.pushbullet.com/v2/pushes', data=json.dumps(post_data), + headers={'Content-Type': 'application/json', 'Access-Token': token}) try: opener.open(req) except Exception as e: diff --git a/modules/Poloniex.py b/modules/Poloniex.py index 1eebf0b8..4b13c444 100644 --- a/modules/Poloniex.py +++ b/modules/Poloniex.py @@ -4,14 +4,25 @@ import json import socket import time -import urllib -import urllib2 import threading import modules.Configuration as Config from modules.RingBuffer import RingBuffer from modules.ExchangeApi import ExchangeApi from modules.ExchangeApi import ApiError +from builtins import range + +try: + # Python 3 + from urllib.parse import urlencode + from urllib.request import urlopen, Request + from urllib.error import HTTPError + PYVER = 3 +except ImportError: + # Python 2 + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError + PYVER = 2 def post_process(before): @@ -20,10 +31,11 @@ def post_process(before): # Add timestamps if there isnt one but is a datetime if 'return' in after: if isinstance(after['return'], list): - for x in xrange(0, len(after['return'])): + for x in range(0, len(after['return'])): if isinstance(after['return'][x], dict): if 'datetime' in after['return'][x] and 'timestamp' not in after['return'][x]: - after['return'][x]['timestamp'] = float(ExchangeApi.create_time_stamp(after['return'][x]['datetime'])) + after['return'][x]['timestamp'] \ + = float(ExchangeApi.create_time_stamp(after['return'][x]['datetime'])) return after @@ -35,6 +47,8 @@ def __init__(self, cfg, log): self.log = log self.APIKey = self.cfg.get("API", "apikey", None) self.Secret = self.cfg.get("API", "secret", None) + if PYVER == 3: + self.Secret = bytes(self.Secret, 'latin-1') self.req_per_sec = 6 self.req_time_log = RingBuffer(self.req_per_sec) self.lock = threading.RLock() @@ -48,17 +62,17 @@ def limit_request_rate(self): time_since_oldest_req = now - self.req_time_log[0] # check if oldest request is more than 1sec ago if time_since_oldest_req < 1: - # print self.req_time_log.get() + # print(self.req_time_log.get()) # uncomment to debug - # print "Waiting %s sec to keep api request rate" % str(1 - time_since_oldest_req) - # print "Req: %d 6th Req: %d Diff: %f sec" %(now, self.req_time_log[0], time_since_oldest_req) + # print("Waiting %s sec to keep api request rate" % str(1 - time_since_oldest_req)) + # print("Req: %d 6th Req: %d Diff: %f sec" %(now, self.req_time_log[0], time_since_oldest_req)) self.req_time_log.append(now + 1 - time_since_oldest_req) time.sleep(1 - time_since_oldest_req) return # uncomment to debug # else: - # print self.req_time_log.get() - # print "Req: %d 6th Req: %d Diff: %f sec" % (now, self.req_time_log[0], time_since_oldest_req) + # print(self.req_time_log.get()) + # print("Req: %d 6th Req: %d Diff: %f sec" % (now, self.req_time_log[0], time_since_oldest_req)) # append current request time to the log, pushing out the 6th request time before it self.req_time_log.append(now) @@ -78,14 +92,14 @@ def _read_response(resp): try: if command == "returnTicker" or command == "return24hVolume": - ret = urllib2.urlopen(urllib2.Request('https://poloniex.com/public?command=' + command)) + ret = urlopen(Request('https://poloniex.com/public?command=' + command)) return _read_response(ret) elif command == "returnOrderBook": - ret = urllib2.urlopen(urllib2.Request( + ret = urlopen(Request( 'https://poloniex.com/public?command=' + command + '¤cyPair=' + str(req['currencyPair']))) return _read_response(ret) elif command == "returnMarketTradeHistory": - ret = urllib2.urlopen(urllib2.Request( + ret = urlopen(Request( 'https://poloniex.com/public?command=' + "returnTradeHistory" + '¤cyPair=' + str( req['currencyPair']))) return _read_response(ret) @@ -94,12 +108,14 @@ def _read_response(resp): + '¤cy=' + str(req['currency'])) if req['limit'] > 0: req_url += ('&limit=' + str(req['limit'])) - ret = urllib2.urlopen(urllib2.Request(req_url)) + ret = urlopen(Request(req_url)) return _read_response(ret) else: req['command'] = command req['nonce'] = int(time.time() * 1000) - post_data = urllib.urlencode(req) + post_data = urlencode(req) + if PYVER == 3: + post_data = bytes(post_data, 'latin-1') sign = hmac.new(self.Secret, post_data, hashlib.sha512).hexdigest() headers = { @@ -107,10 +123,10 @@ def _read_response(resp): 'Key': self.APIKey } - ret = urllib2.urlopen(urllib2.Request('https://poloniex.com/tradingApi', post_data, headers)) + ret = urlopen(Request('https://poloniex.com/tradingApi', post_data, headers)) json_ret = _read_response(ret) return post_process(json_ret) - except urllib2.HTTPError as ex: + except HTTPError as ex: raw_polo_response = ex.read() try: data = json.loads(raw_polo_response) @@ -122,11 +138,11 @@ def _read_response(resp): ': The web server reported a bad gateway or gateway timeout error.' else: polo_error_msg = raw_polo_response - ex.message = ex.message if ex.message else str(ex) + ex.message = ex.message if hasattr(ex, 'message') and ex.message else str(ex) ex.message = "{0} Requesting {1}. Poloniex reports: '{2}'".format(ex.message, command, polo_error_msg) raise ex except Exception as ex: - ex.message = ex.message if ex.message else str(ex) + ex.message = ex.message if hasattr(ex, 'message') and ex.message else str(ex) ex.message = "{0} Requesting {1}".format(ex.message, command) raise diff --git a/modules/RingBuffer.py b/modules/RingBuffer.py index 07abe6f0..2fd7d123 100644 --- a/modules/RingBuffer.py +++ b/modules/RingBuffer.py @@ -2,41 +2,43 @@ # also known as ring buffer, pops the oldest data item # to make room for newest data item when max size is reached # uses the double ended queue available in Python24 - + from collections import deque - + + class RingBuffer(deque): - """ - inherits deque, pops the oldest data to make room - for the newest data when size is reached - """ - def __init__(self, size): - deque.__init__(self) - self.size = size - - def full_append(self, item): - deque.append(self, item) - # full, pop the oldest item, left most item - self.popleft() - - def append(self, item): - deque.append(self, item) - # max size reached, append becomes full_append - if len(self) == self.size: - self.append = self.full_append - - def get(self): - """returns a list of size items (newest items)""" - return list(self) - + """ + inherits deque, pops the oldest data to make room + for the newest data when size is reached + """ + def __init__(self, size): + deque.__init__(self) + self.size = size + + def full_append(self, item): + deque.append(self, item) + # full, pop the oldest item, left most item + self.popleft() + + def append(self, item): + deque.append(self, item) + # max size reached, append becomes full_append + if len(self) == self.size: + self.append = self.full_append + + def get(self): + """returns a list of size items (newest items)""" + return list(self) + + # testing if __name__ == '__main__': - size = 5 - ring = RingBuffer(size) - for x in range(9): - ring.append(x) - print ring.get() # test - + size = 5 + ring = RingBuffer(size) + for x in range(9): + ring.append(x) + print(ring.get()) # test + """ notice that the left most item is popped to make room result = @@ -49,4 +51,4 @@ def get(self): [2, 3, 4, 5, 6] [3, 4, 5, 6, 7] [4, 5, 6, 7, 8] -""" \ No newline at end of file +""" diff --git a/modules/WebServer.py b/modules/WebServer.py index 6ffc9022..891df9ba 100644 --- a/modules/WebServer.py +++ b/modules/WebServer.py @@ -43,9 +43,14 @@ def start_web_server(): ''' Start the web server ''' - import SimpleHTTPServer - import SocketServer import socket + try: + import SimpleHTTPServer + import SocketServer + except ImportError: + # Python 3 (this isn't a nice way to do it, but the nicer way involves installing future from pip for py2) + import http.server as SimpleHTTPServer + import socketserver as SocketServer try: port = int(web_server_port) diff --git a/requirements.txt b/requirements.txt index 6555d924..7a846210 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pandas hypothesis requests pytz +future diff --git a/tests/test_Configuration.py b/tests/test_Configuration.py new file mode 100644 index 00000000..448aa3a4 --- /dev/null +++ b/tests/test_Configuration.py @@ -0,0 +1,123 @@ +import pytest +import tempfile +from six import iteritems +from decimal import Decimal + +# Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for +# pytest to work +import os, sys, inspect +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0, parentdir) + + +def add_to_env(category, option, value): + os.environ['{0}_{1}'.format(category, option)] = value + + +def rm_from_env(category, option): + os.environ.pop('{0}_{1}'.format(category, option)) + + +def write_to_cfg(filename, category, val_dict): + with open(filename, 'a') as out_file: + out_file.write('\n') + out_file.write('[{0}]\n'.format(category)) + for option, value in iteritems(val_dict): + out_file.write('{0} = {1}\n'.format(option, value)) + + +def write_skeleton_exchange(filename, exchange): + write_to_cfg(filename, 'API', {'exchange': exchange}) + write_to_cfg(filename, exchange.upper(), {'all_currencies': 'XMR'}) + + +@pytest.fixture(autouse=True) +def env_vars(): + var_set_1 = "ENVVAR,BOOL_T,true" + var_set_2 = "ENVVAR,BOOL_F,false" + var_set_3 = "ENVVAR,NUM,60" + var_list = [var_set_1, var_set_2, var_set_3] + for var_set in var_list: + c, o, v = var_set.split(',') + add_to_env(c, o, v) + yield var_list # Teardown after yield + for var_set in var_list: + c, o, v = var_set.split(',') + rm_from_env(c, o) + + +@pytest.fixture() +def config(): + import modules.Configuration as Config + cfg = {"BOOL_T": "true", + "BOOL_F": "false", + "NUM": "60"} + f = tempfile.NamedTemporaryFile(delete=False) + write_to_cfg(f.name, 'CFG', cfg) + Config.filename = f.name + Config.init(Config.filename) + yield Config # Teardown after yield + del Config + os.remove(f.name) + + +class TestClass(object): + def test_has_option(self, config): + assert(not config.has_option('fail', 'fail')) + assert(config.has_option('ENVVAR', 'BOOL_T')) + assert(config.has_option('CFG', 'BOOL_T')) + + def test_getboolean(self, config): + assert(not config.getboolean('fail', 'fail')) + assert(config.getboolean('ENVVAR', 'BOOL_T')) + assert(config.getboolean('ENVVAR', 'BOOL_F') is False) + assert(config.getboolean('CFG', 'BOOL_T')) + assert(config.getboolean('CFG', 'BOOL_F') is False) + with pytest.raises(ValueError): + config.getboolean('ENVVAR', 'NUM') + config.getboolean('CFG', 'NUM') + assert(config.getboolean('some', 'default', True)) + assert(config.getboolean('some', 'default') is False) + + def test_get(self, config): + assert(config.get('ENVVAR', 'NUM') == '60') + assert(config.get('ENVVAR', 'NUM', False, 61) == 61) + assert(config.get('ENVVAR', 'NUM', False, 1, 59) == 59) + assert(config.get('ENVVAR', 'NO_NUM', 100) == 100) + with pytest.raises(SystemExit): + assert(config.get('ENVVAR', 'NO_NUM', None)) + + def test_get_exchange_poloniex(self, config): + write_skeleton_exchange(config.filename, 'Poloniex') + config.init(config.filename) + assert(config.get_exchange() == 'POLONIEX') + + def test_get_exchange_bitfinex(self, config): + write_skeleton_exchange(config.filename, 'Bitfinex') + config.init(config.filename) + assert(config.get_exchange() == 'BITFINEX') + + def test_get_coin_cfg_new(self, config): + write_skeleton_exchange(config.filename, 'Bitfinex') + cfg = {'minloansize': 0.01, + 'mindailyrate': 0.18, + 'maxactiveamount': 1, + 'maxtolend': 0, + 'maxpercenttolend': 0, + 'maxtolendrate': 0} + write_to_cfg(config.filename, 'XMR', cfg) + config.init(config.filename) + result = {'XMR': {'minrate': Decimal('0.0018'), 'maxactive': Decimal('1'), 'maxtolend': Decimal('0'), + 'maxpercenttolend': Decimal('0'), 'maxtolendrate': Decimal('0'), 'gapmode': False, + 'gapbottom': Decimal('0'), 'gaptop': Decimal('0')}} + assert(config.get_coin_cfg() == result) + + def test_get_coin_cfg_old(self, config): + write_to_cfg(config.filename, 'BOT', {'coinconfig': '["BTC:0.18:1:0:0:0","DASH:0.6:1:0:0:0"]'}) + config.init(config.filename) + result = {'BTC': {'minrate': Decimal('0.0018'), 'maxactive': Decimal('1'), 'maxtolend': Decimal('0'), + 'maxpercenttolend': Decimal('0'), 'maxtolendrate': Decimal('0')}, + 'DASH': {'minrate': Decimal('0.006'), 'maxactive': Decimal('1'), 'maxtolend': Decimal('0'), + 'maxpercenttolend': Decimal('0'), 'maxtolendrate': Decimal('0')}} + assert(config.get_coin_cfg() == result) diff --git a/tests/test_PoloniexAPI.py b/tests/test_PoloniexAPI.py index 7091ea7b..3bb6aba0 100644 --- a/tests/test_PoloniexAPI.py +++ b/tests/test_PoloniexAPI.py @@ -1,4 +1,5 @@ import time +from builtins import range # Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for # pytest to work @@ -25,7 +26,7 @@ # thread1.start() # except Exception as e: # assert False, 'api_query ' + str(i + 1) + ':' + e.message -# +# # # # Test fast api calls # def test_multiple_calls(): @@ -36,13 +37,14 @@ def api_rate_limit(n, start): api.limit_request_rate() # verify that the (N % 6) th request is delayed by (N / 6) sec from the start time if n != 0 and n % 6 == 0: - print 'limit request ' + str(n) + ' ' + str(start) + ' ' + str(time.time()) + '\n' + print('limit request ' + str(n) + ' ' + str(start) + ' ' + str(time.time()) + '\n') assert time.time() - start >= int(n / 6), "rate limit failed" # Test rate limiter def test_rate_limiter(): start = time.time() - for i in xrange(20): - thread1 = threading.Thread(target=api_rate_limit, args=(i, start)) - thread1.start() + for i in range(20): + thread = threading.Thread(target=api_rate_limit, args=(i, start)) + thread.start() + thread.join()