diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..15de3902b1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +node_modules +data diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4f8c6f1821 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,23 @@ +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +[*.{js,json}] +charset = utf-8 +indent_style = space + +[*.{js,yml}] +indent_size = 2 + +[*.{json,sh}] +indent_size = 4 + +[Makefile] +indent_style = tab + +[{package.json,.travis.yml}] +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore index 35c7504d2f..ba7a05aa7a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,18 +18,3 @@ conf-* sim_result* *_test backtesting_* - -.ipynb_checkpoints -data.csv -value_net.json - -extensions/strategies/bumblebee -zen/logs/history/*.png -zen/logs/hof/*.txt -zen/evolution/__pycache__/ - -zen/__pycache__/ - -zen/temp\.html - -*.pyc diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f0253eb587..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "extensions/strategies/bumblebee"] - path = extensions/strategies/bumblebee - url = https://github.com/mkarpis/bumblebee.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..8df3afc385 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,97 @@ +FROM node:latest + + + +# NOTE: THIS DOCKERFILE IS GENERATED VIA "update.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + tcl \ + tk \ + && rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D +ENV PYTHON_VERSION 3.6.0 + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 9.0.1 + +RUN set -ex \ + && buildDeps=' \ + tcl-dev \ + tk-dev \ + ' \ + && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ + \ + && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \ + && gpg --batch --verify python.tar.xz.asc python.tar.xz \ + && rm -r "$GNUPGHOME" python.tar.xz.asc \ + && mkdir -p /usr/src/python \ + && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ + && rm python.tar.xz \ + \ + && cd /usr/src/python \ + && ./configure \ + --enable-loadable-sqlite-extensions \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + \ +# explicit path to "pip3" to ensure distribution-provided "pip3" cannot interfere + && if [ ! -e /usr/local/bin/pip3 ]; then : \ + && wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \ + && rm /tmp/get-pip.py \ + ; fi \ +# we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python +# ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages") +# https://github.com/docker-library/python/pull/143#issuecomment-241032683 + && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \ +# then we use "pip list" to ensure we don't have more than one pip version installed +# https://github.com/docker-library/python/pull/100 + && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] \ + \ + && find /usr/local -depth \ + \( \ + \( -type d -a -name test -o -name tests \) \ + -o \ + \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ + \) -exec rm -rf '{}' + \ + && apt-get purge -y --auto-remove $buildDeps \ + && rm -rf /usr/src/python ~/.cache + +# make some useful symlinks that are expected to exist +RUN cd /usr/local/bin \ + && { [ -e easy_install ] || ln -s easy_install-* easy_install; } \ + && ln -s idle3 idle \ + && ln -s pydoc3 pydoc \ + && ln -s python3 python \ + && ln -s python3-config python-config + + + + +RUN apt-get update +RUN apt install -y graphviz libgraphviz-dev pkg-config +RUN PKG_CONFIG_ALLOW_SYSTEM_LIBS=OHYESPLEASE pip install pygraphviz +ADD zen/requirements.txt / +RUN pip3 install -r /requirements.txt +WORKDIR /app +RUN /usr/local/bin/npm install +RUN npm update + + diff --git a/README.md b/README.md index 0af7a14dc7..6f0d09c158 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ rsi --overbought_rsi= sell when RSI reaches or goes above this value (default: 82) --rsi_recover= allow RSI to recover this many points before buying (default: 3) --rsi_drop= allow RSI to fall this many points before selling (default: 0) - --rsi_divisor= sell when RSI reaches high-water reading divided by this value (default: 2) + --rsi_dividend= sell when RSI reaches high-water reading divided by this value (default: 2) sar description: @@ -452,8 +452,15 @@ zenbot sell gdax.BTC-USD --pct=10 ## TODO - cancel pending orders on SIGINT +- determine and fix what is causing live trading to underperform vs. paper trading/simulations +- improve order execution speed, possibly by using market-type orders (incurring taker fees) +- support for limiting the amount of balance Zenbot can use for trading +- fix partial filled orders sometimes not getting recognized, due to race conditions - tool to generate graph and stats from live or paper trading sessions -- review PRs +- save sim data to db, for front-end UI +- make error output compact, no stack trace +- review PR for Bitfinex +- more exchange support - web UI with graphs and logs - "reaper" to automatically prune trades collection to a certain day length - "lite mode" for trader, an option to run without MongoDB diff --git a/commands/sim.js b/commands/sim.js index 7004cb80d4..42a2f8c9ea 100644 --- a/commands/sim.js +++ b/commands/sim.js @@ -6,7 +6,6 @@ var tb = require('timebucket') , moment = require('moment') , colors = require('colors') - module.exports = function container (get, set, clear) { var c = get('conf') return function (program) { @@ -78,7 +77,6 @@ module.exports = function container (get, set, clear) { var cursor, reversing, reverse_point var query_start = so.start ? tb(so.start).resize(so.period).subtract(so.min_periods + 2).toMilliseconds() : null - function exitSim () { console.log() if (!s.period) { @@ -165,7 +163,6 @@ module.exports = function container (get, set, clear) { process.exit(0) } - function getNext () { var opts = { query: { diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..ca9286e9ee --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +server: + build: . + volumes: + - .:/app + - /app/node_modules + links: + - mongodb + command: ./zenbot.sh trade --paper + restart: always + +mongodb: + image: mongo:latest + volumes_from: + - mongodb-data + command: mongod --smallfiles + +mongodb-data: + image: mongo:latest + volumes: + - ./data/db:/data/db + command: "true" diff --git a/extensions/exchanges/gdax/exchange.js b/extensions/exchanges/gdax/exchange.js index 8311281ea9..ba95e6be32 100644 --- a/extensions/exchanges/gdax/exchange.js +++ b/extensions/exchanges/gdax/exchange.js @@ -81,7 +81,6 @@ module.exports = function container (get, set, clear) { side: trade.side } }) - trades.reverse() cb(null, trades) }) }, diff --git a/extensions/exchanges/kraken/products.json b/extensions/exchanges/kraken/products.json index 98ae7da2df..b64c0eee7a 100644 --- a/extensions/exchanges/kraken/products.json +++ b/extensions/exchanges/kraken/products.json @@ -90,6 +90,20 @@ "increment": "0.00000001", "label": "ETH/XBT" }, + { + "asset": "XETH", + "currency": "XXBT", + "min_size": "0.01", + "increment": "0.00000001", + "label": "ETH/XBT" + }, + { + "asset": "XETH", + "currency": "ZCAD", + "min_size": "0.01", + "increment": "0.00000001", + "label": "ETH/CAD" + }, { "asset": "XETH", "currency": "ZCAD", @@ -104,6 +118,20 @@ "increment": "0.00000001", "label": "ETH/EUR" }, + { + "asset": "XETH", + "currency": "ZEUR", + "min_size": "0.01", + "increment": "0.00000001", + "label": "ETH/EUR" + }, + { + "asset": "XETH", + "currency": "ZGBP", + "min_size": "0.01", + "increment": "0.00000001", + "label": "ETH/GBP" + }, { "asset": "XETH", "currency": "ZGBP", @@ -118,6 +146,20 @@ "increment": "0.00000001", "label": "ETH/JPY" }, + { + "asset": "XETH", + "currency": "ZJPY", + "min_size": "0.01", + "increment": "0.00000001", + "label": "ETH/JPY" + }, + { + "asset": "XETH", + "currency": "ZUSD", + "min_size": "0.01", + "increment": "0.00000001", + "label": "ETH/USD" + }, { "asset": "XETH", "currency": "ZUSD", @@ -209,6 +251,13 @@ "increment": "0.00000001", "label": "XBT/CAD" }, + { + "asset": "XXBT", + "currency": "ZCAD", + "min_size": "0.01", + "increment": "0.00000001", + "label": "XBT/CAD" + }, { "asset": "XXBT", "currency": "ZEUR", @@ -216,6 +265,20 @@ "increment": "0.00000001", "label": "XBT/EUR" }, + { + "asset": "XXBT", + "currency": "ZEUR", + "min_size": "0.01", + "increment": "0.00000001", + "label": "XBT/EUR" + }, + { + "asset": "XXBT", + "currency": "ZGBP", + "min_size": "0.01", + "increment": "0.00000001", + "label": "XBT/GBP" + }, { "asset": "XXBT", "currency": "ZGBP", @@ -230,6 +293,20 @@ "increment": "0.00000001", "label": "XBT/JPY" }, + { + "asset": "XXBT", + "currency": "ZJPY", + "min_size": "0.01", + "increment": "0.00000001", + "label": "XBT/JPY" + }, + { + "asset": "XXBT", + "currency": "ZUSD", + "min_size": "0.01", + "increment": "0.00000001", + "label": "XBT/USD" + }, { "asset": "XXBT", "currency": "ZUSD", diff --git a/extensions/exchanges/kraken/update-products.sh b/extensions/exchanges/kraken/update-products.sh index c77afc0abd..4748354805 100755 --- a/extensions/exchanges/kraken/update-products.sh +++ b/extensions/exchanges/kraken/update-products.sh @@ -34,9 +34,7 @@ kraken.api('Assets', null, function (error, data) { process.exit(1) } else { Object.keys(data.result).forEach(function (result) { - if(!result.match('\.d')) { - addProduct(data.result[result].base, data.result[result].quote, data.result[result].altname) - } + addProduct(data.result[result].base, data.result[result].quote, data.result[result].altname) }) var target = require('path').resolve(__dirname, 'products.json') require('fs').writeFileSync(target, JSON.stringify(products, null, 2)) diff --git a/extensions/strategies/deep_net0/_codemap.js b/extensions/strategies/deep_net0/_codemap.js deleted file mode 100644 index b36b345029..0000000000 --- a/extensions/strategies/deep_net0/_codemap.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - _ns: 'zenbot', - - 'strategies.deep_net0': require('./strategy'), - 'strategies.list[]': '#strategies.deep_net0' -} diff --git a/extensions/strategies/deep_net0/strategy.js b/extensions/strategies/deep_net0/strategy.js deleted file mode 100644 index 167f539bc1..0000000000 --- a/extensions/strategies/deep_net0/strategy.js +++ /dev/null @@ -1,268 +0,0 @@ -var z = require('zero-fill') -, n = require('numbro') -, deepqlearn = require('../../../node_modules/convnetjs/build/deepqlearn.js') -, convnetjs = require('convnetjs') -, fs = require('fs') - -module.exports = function container (get, set, clear) { - return { - name: 'deep_net0', - description: 'Deep net - v0', - - - getOptions: function () { - this.option('period', 'period length in minutes', String, '1h') - //this.option('min_periods', 'min. number of history periods', Number, 200) - - - this.option('brain', 'Brain object container', Object, null) - this.option('states', 'States container array', Object, null) - this.option('input_buffer_size', 'History buffer size, that should be used while learning', Number, 200) - this.option('min_lookback', 'Minimum lookback buffer needed before simulation start', Number, 1) - this.option('model', 'Path to pre-trained model', String, null) - this.option('saved_since_ticker', 'Incremental ticker since last save', Number, 0) - this.option('save_every_x', 'Save every x tick', Number, 1000) - - // working variables - this.option('last_close_price', 'Last closing price (working variable)', Number, 0) - - - /* - // History containers - this.option('ma10_prev', '', Number, 0) - this.option('ma30_prev', '', Number, 0) - this.option('ma60_prev', '', Number, 0) - this.option('ma90_prev', '', Number, 0) - this.option('ma150_prev', '', Number, 0) - this.option('ma360_prev', '', Number, 0) - - this.option('hold', 'Hold', Number, 0) - this.option('model', 'Path to pre-trained model', String, null) - this.option('order_finished', 'Indicator whether the order was finished', Boolean, true) - this.option('last_action', 'Indicator whether the order was finished', String, null) - - this.option('ticker_timeout_limit', 'How long the bot can go without checking new actions', Number, 10) - this.option('ticker', 'Value of previous profit', Number, 0) - this.option('ticker_expired', 'Value of previous profit', Boolean, false) - - this.option('previous_profit', 'Value of previous profit', Number, 0) - this.option('prev_mode', 'Prev mode', Number, 0) - this.option('data_ready', 'Data ready', Boolean, false) - this.option('holding_currency', 'Holding currency indicator', Boolean, false) - this.option('holding_assets', 'Holding assets indicator', Boolean, false) - - this.option('saved_since_ticker', 'Incremental ticker since last save', Number, 0) - this.option('save_every_x', 'Save every x tick', Number, 1000) - - //this.option('floor_coef', 'floor coefficient', Number, 0.75) - //this.option('moon_coef', 'moon coefficient', Number, 1.05) - //this.option('resistance_coef', 'resistance coefficient', Number, 2.8) - */ - - /* - this.option('ema_short_period', 'number of periods for the shorter EMA', Number, 12) - this.option('ema_long_period', 'number of periods for the longer EMA', Number, 26) - this.option('signal_period', 'number of periods for the signal EMA', Number, 9) - this.option('up_trend_threshold', 'threshold to trigger a buy signal', Number, 0) - this.option('down_trend_threshold', 'threshold to trigger a sold signal', Number, 0) - this.option('overbought_rsi_periods', 'number of periods for overbought RSI', Number, 25) - this.option('overbought_rsi', 'sold when RSI exceeds this value', Number, 70) - */ - }, - - - initialize: function(s){ - - function loadnet(brain, file) { - var dataNet = fs.readFileSync('value_net.json', 'utf8') - var j = JSON.parse(dataNet); - s.options.brain.value_net.fromJSON(j); - brain.learning = false; // stop learning - } - - // General settins - s.options.states = [] - s.options.min_lookback = s.options.input_buffer_size - - - // Initialize deep-net - var layer_defs = []; - var num_inputs = s.options.input_buffer_size - var num_actions = 3 // 3 possible actions (buy, sell, hold) - var temporal_window = 1 // amount of temporal memory. 0 = agent lives in-the-moment :) - var network_size = num_inputs*temporal_window + num_actions*temporal_window + num_inputs - layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:network_size}); - //layer_defs.push({type:'fc', num_neurons: 200, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 100, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'regression', num_neurons: num_actions}); - - - /* - layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:network_size}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'regression', num_neurons: num_actions}); - */ - // options for the Temporal Difference learner that trains the above net - // by backpropping the temporal difference learning rule. - var tdtrainer_options = {learning_rate:0.0003, momentum:0.0, batch_size:64, l2_decay:0.01}; - var opt = {}; - opt.temporal_window = temporal_window; - // size of experience replay memory - opt.experience_size = 30000; - // number of examples in experience replay memory before we begin learning - opt.start_learn_threshold = 1000; - // gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1] - opt.gamma = 0.8; - opt.learning_steps_total = 200000; - // how many steps of the above to perform only random actions (in the beginning)? - opt.learning_steps_burnin = 3000; - opt.epsilon_min = 0.05; - opt.epsilon_test_time = 0.05; - opt.layer_defs = layer_defs; - opt.tdtrainer_options = tdtrainer_options; - s.options.brain = new deepqlearn.Brain(num_inputs, num_actions, opt) - - if(s.options.model){ - loadnet(s.options.brain, s.options.model) - } - - }, - - - /* Called every time when a trade was done - (for example if in 5min period was done 20 trades, the method will be called 20 times) - */ - calculate: function (s) { - if(!s.options.brain) - this.initialize(s) - - return; - }, - - onPeriod: function (s, cb) { - - // Wait until we get all the samples that we need - if(s.lookback.length <= s.options.min_lookback){ - if(s.lookback.length > 0){ - s.options.states.push(s.period.close - s.options.last_close_price) - //s.options.last_close_price = s.period.close - } - s.options.last_close_price = s.period.close - cb() - return - } - - - //console.log('calculating...', s.period.period_id, '--', s.period.volume) - // get current ticker data - //get('lib.ema')(s, 'ma2', 2/periodInMinutes) - - s.options.states.pop() - s.options.states.push(s.period.close - s.options.last_close_price) - s.options.last_close_price = s.period.close - - - function calcProfit(s) { - //if we didn't make any trades yet just return 0 - if(s.my_trades.length === 0) - return 0 - - var total_currency = parseInt(s.balance.asset)*s.period.close + parseInt(s.balance.currency) - var start_capital = s.start_capital - //var profit = total_currency - start_capital - var profit = total_currency - s.options.previous_profit - // TODO we can add also change from previous run - s.options.previous_profit = total_currency - var binary_profit = profit > 0 ? 1 : 0 - console.log('\nprevious_profit: ', s.options.previous_profit, ', current_profit: ', profit, ', action:', s.action, ' last_signal: ', s.last_signal) - - return binary_profit - - // calculate profit - /*var profit = s.start_capital ? n(s.balance.currency).subtract(s.start_capital).divide(s.start_capital) : n(0) - var current_profit = profit.value() - var profit_slope = current_profit - s.options.previous_profit - //minor punishment - if(profit_slope === 0.0) - profit_slope = -0.01 - - console.log('\nprevious_profit: ', s.options.previous_profit, ', current_profit: ', current_profit, ', slope:', profit_slope, ', action:', s.action, ' last_signal: ', s.last_signal) - return profit_slope - */ - } - - - function savenet(brain) { - if(brain){ - var filename = 'value_net.json'; - var j = brain.value_net.toJSON(); - var t = JSON.stringify(j) - - fs.writeFile(filename, t, function(err) { - if(err) { - return console.log(err); - } - }); - } - } - - - // training happens here - - - // ***** Reward the last action - reward = calcProfit(s) - s.options.brain.backward(reward) - - // ***** Predict next state - var action = s.options.brain.forward(s.options.states); - - if(action === 0) - s.signal = null - else if (action === 1) - s.signal = 'buy' - else if (action === 2) - s.signal = 'sell' - - console.log('\nsetting new action:', s.signal) - console.log('brain-age:', s.options.brain.age) - console.log('average Q-learning loss:', s.options.brain.average_loss_window.get_average()) - console.log('smooth-ish reward: ', s.options.brain.average_reward_window.get_average()) - console.log() - - - if(s.options.saved_since_ticker > s.options.save_every_x) { - //save trained brain to file - savenet(s.options.brain) - s.options.saved_since_ticker = 0 - } - - s.options.saved_since_ticker += 1 - - cb() -}, - - - -onReport: function (s) { - var cols = [] - if (typeof s.period.macd_histogram === 'number') { - var color = 'grey' - if (s.period.macd_histogram > 0) { - color = 'green' - } - else if (s.period.macd_histogram < 0) { - color = 'red' - } - cols.push(z(8, n(s.period.macd_histogram).format('+00.0000'), ' ')[color]) - cols.push(z(8, n(s.period.overbought_rsi).format('00'), ' ').cyan) - } - else { - cols.push(' ') - } - return cols -} -} -} diff --git a/extensions/strategies/deep_net1/_codemap.js b/extensions/strategies/deep_net1/_codemap.js deleted file mode 100644 index 7ad1bdfd4f..0000000000 --- a/extensions/strategies/deep_net1/_codemap.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - _ns: 'zenbot', - - 'strategies.deep_net1': require('./strategy'), - 'strategies.list[]': '#strategies.deep_net1' -} diff --git a/extensions/strategies/deep_net1/strategy.js b/extensions/strategies/deep_net1/strategy.js deleted file mode 100644 index 5f9355f0b3..0000000000 --- a/extensions/strategies/deep_net1/strategy.js +++ /dev/null @@ -1,357 +0,0 @@ -var z = require('zero-fill') -, n = require('numbro') -, deepqlearn = require('../../../node_modules/convnetjs/build/deepqlearn.js') -, convnetjs = require('convnetjs') -, fs = require('fs') - -module.exports = function container (get, set, clear) { - return { - name: 'deep_net1', - description: 'Deep net 1 (reinforcement learning) ', - - - getOptions: function () { - // build-in - this.option('period', 'period length in minutes', String, '2m') - this.option('period_interval', 'period length in minutes', Number, 0) - this.option('brain', 'Brain object container', Object, null) - this.option('intervals', 'Intervals array', Object, [10, 30, 60, 120, 240]) - this.option('model', 'Path to pre-trained model', String, null) - this.option('saved_since_ticker', 'Incremental ticker since last save', Number, 0) - this.option('save_every_x', 'Save every x tick', Number, 1000) - this.option('save_dataset', 'Saves dateset to data.csv file', Boolean, false) - this.option('states', 'states', Object, null) - - - // Working variables - this.option('model_initialized', 'initialize var.', Boolean, false) - this.option('history_ema', 'Ema data from previous run', Object, null) - this.option('previous_closing_price', 'previous closing price', Number, 0) - this.option('previous_predicted_state', 'Placeholder for previous predicted state', String, null) - - - - - /* - - this.option('hold', 'Hold', Number, 0) - this.option('order_finished', 'Indicator whether the order was finished', Boolean, true) - this.option('last_action', 'Indicator whether the order was finished', String, null) - - this.option('ticker_timeout_limit', 'How long the bot can go without checking new actions', Number, 10) - this.option('ticker', 'Value of previous profit', Number, 0) - this.option('ticker_expired', 'Value of previous profit', Boolean, false) - - this.option('previous_profit', 'Value of previous profit', Number, 0) - this.option('prev_mode', 'Prev mode', Number, 0) - this.option('data_ready', 'Data ready', Boolean, false) - this.option('holding_currency', 'Holding currency indicator', Boolean, false) - this.option('holding_assets', 'Holding assets indicator', Boolean, false) - - - - this.option('ema_short_period', 'number of periods for the shorter EMA', Number, 12) - this.option('ema_long_period', 'number of periods for the longer EMA', Number, 26) - this.option('signal_period', 'number of periods for the signal EMA', Number, 9) - this.option('up_trend_threshold', 'threshold to trigger a buy signal', Number, 0) - this.option('down_trend_threshold', 'threshold to trigger a sold signal', Number, 0) - this.option('overbought_rsi_periods', 'number of periods for overbought RSI', Number, 25) - this.option('overbought_rsi', 'sold when RSI exceeds this value', Number, 70) - */ - }, - - - initialize: function(s){ - - s.options.history_ema = [] - s.options.period_interval = getPeriodInMinutes(s.options.period) - - function loadnet(brain, file) { - var dataNet = fs.readFileSync('value_net.json', 'utf8') - var j = JSON.parse(dataNet); - s.options.brain.value_net.fromJSON(j); - brain.learning = false; // stop learning - } - - // Your initialization goes here - periodInMinutes = getPeriodInMinutes(s.options.period) - - //initialize q-grid - var layer_defs = []; - var num_inputs = 30 - var num_actions = 2 // 3 possible actions (buy, sell, hold) - var temporal_window = 1 // amount of temporal memory. 0 = agent lives in-the-moment :) - var network_size = num_inputs*temporal_window + num_actions*temporal_window + num_inputs - layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:network_size}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'regression', num_neurons: num_actions}); - var tdtrainer_options = {learning_rate:0.01, momentum:0.0, batch_size:64, l2_decay:0.01}; - var opt = {}; - opt.temporal_window = temporal_window; - // size of experience replay memory - opt.experience_size = 30000; - // number of examples in experience replay memory before we begin learning - opt.start_learn_threshold = 1000; - // gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1] - opt.gamma = 0.7; - //opt.learning_steps_total = 200000; - // how many steps of the above to perform only random actions (in the beginning)? - opt.learning_steps_burnin = 3000; - opt.epsilon_min = 0.05; - opt.epsilon_test_time = 0.01; - opt.layer_defs = layer_defs; - opt.tdtrainer_options = tdtrainer_options; - s.options.brain = new deepqlearn.Brain(num_inputs, num_actions, opt) - - if(s.options.model){ - loadnet(s.options.brain, s.options.model) - } - - s.options.model_initialized = true - - //function getPeriodInMinutes(p) - function getPeriodInMinutes (p) { - if(p.indexOf('m') !== -1) - return Number(p.replace('m','')) - - if(p.indexOf('h') !== -1) - return Number(p.replace('h',''))*60 - } - }, - - - /* Called every time when a trade was done - (for example if in 5min period was done 20 trades, the method will be called 20 times) - */ - calculate: function (s) { - - if(!s.options.model_initialized) - this.initialize(s) - - //console.log('in calculate:', s.period.close, ', ', s.period.close_time, ', ', s.lookback.length) - }, - - //----------------------------------------------------------- - //----------------------------------------------------------- - - - onPeriod: function (s, cb) { - - //console.log('on perdio:', s.period.close, ', ', s.period.close_time, ', ', s.lookback.length) - // Wait until we get all the samples that we need - var interval = s.options.period_interval - if(s.lookback.length <= s.options.intervals[s.options.intervals.length-1]/interval){ - cb() - return - } - - - - function storeEmaValues(s){ - // Store previous ema - s.options.history_ema = [] - var interval = s.options.period_interval - for(i = 0; i < s.options.intervals.length; i++) { - var periodName = 'ema' + s.options.intervals[i] - get('lib.ema')(s, periodName, s.options.intervals[i]/interval) - s.options.history_ema.push(s.period[periodName]) - } - } - - - function getNewEmaValues(s) { - - // Get ema - var interval = s.options.period_interval - for(i = 0; i < s.options.intervals.length; i++) { - var periodName = 'ema' + s.options.intervals[i] - get('lib.ema')(s, periodName, s.options.intervals[i]/interval) - } - } - - - function calcSlopes(s, states) { - for(i = 0; i < s.options.intervals.length; i++) { - var periodName = 'ema' + s.options.intervals[i] - - var currentValue = s.period[periodName] - var previousValue = s.options.history_ema[i] - - var slope_value = currentValue - previousValue - var slope_binary = slope_value >= 0 ? 1 : 0 - states.push(slope_binary) - } - } - - - function calcNeighbourEmaStates(s, states) { - for(i = 0; i < s.options.intervals.length; i++) { - var iName = 'ema' + s.options.intervals[i] - for(j = 0; j < s.options.intervals.length; j++) { - var jName = 'ema' + s.options.intervals[j] - states.push(s.period[iName] > s.period[jName]) - } - } - } - - - function calcProfit(s) { - //if we didn't make any trades yet just return 0 - if(s.my_trades.length === 0) - return 0 - - var total_currency = parseInt(s.balance.asset)*s.period.close + parseInt(s.balance.currency) - var start_capital = s.start_capital - //var profit = total_currency - start_capital - var profit = total_currency - s.options.previous_profit - // TODO we can add also change from previous run - s.options.previous_profit = total_currency - - var binary_profit = 0 - if(profit === 0 && s.options.previous_predicted_state === null) - binary_profit = 1 - - if(profit > 0 && s.options.previous_predicted_state === 'buy') - binary_profit = 1 - - if(profit < 0 && s.options.previous_predicted_state === 'sell') - binary_profit = 1 - - //var binary_profit = profit > 0 ? 1 : 0 - console.log('\nprevious_profit: ', s.options.previous_profit, ', current_profit: ', profit, ', action:', s.action, ' last_signal: ', s.last_signal) - - return binary_profit - - // calculate profit - /*var profit = s.start_capital ? n(s.balance.currency).subtract(s.start_capital).divide(s.start_capital) : n(0) - var current_profit = profit.value() - var profit_slope = current_profit - s.options.previous_profit - //minor punishment - if(profit_slope === 0.0) - profit_slope = -0.01 - - console.log('\nprevious_profit: ', s.options.previous_profit, ', current_profit: ', current_profit, ', slope:', profit_slope, ', action:', s.action, ' last_signal: ', s.last_signal) - return profit_slope - */ - } - - - function savenet(brain) { - if(brain){ - var filename = 'value_net.json'; - var j = brain.value_net.toJSON(); - var t = JSON.stringify(j) - - fs.writeFile(filename, t, function(err) { - if(err) { - return console.log(err); - } - }); - } - } - - - function convertStatesToInt(states){ - for (i = 0; i < states.length; i++) { - if(typeof(states[i]) === "boolean") - states[i] = states[i] ? 1 : 0 - } - } - - - //************************* END OF FUNCTIONS DEFINITION SECTION - - - if(s.options.save_dataset === true && s.options.states){ - var line = s.options.states.toString() - var priceDiff = s.period.close - s.options.previous_closing_price - var price_slope = 0 - if(priceDiff === 0) - price_slope = 0 - else - price_slope = priceDiff > 0 ? 1 : -1 - - line += ',' + price_slope + '\n' - fs.appendFile('data.csv', line) - } - - - // ***** Reward the last action - reward = calcProfit(s) - //s.options.agent.learn(reward) - s.options.brain.backward(reward) - //s.options.previous_profit = reward - p = s.period - - // ***** Save current states - var states = [] - //addPriceStates(s, states) - getNewEmaValues(s) - calcSlopes(s, states) - calcNeighbourEmaStates(s, states) - convertStatesToInt(states) - storeEmaValues(s) - - s.options.states = states - s.options.previous_closing_price = s.period.close - - // ***** Predict next state - // var action = s.options.agent.act(states) - var action = s.options.brain.forward(states); - //console.log('signal:', action) - - -/* - if(action === 0) - s.signal = null - else if (action === 1) - s.signal = 'buy' - else if (action === 2) - s.signal = 'sell' -*/ - if(action === 0) - s.signal = 'buy' - else if (action === 1) - s.signal = 'sell' - - s.options.previous_predicted_state = s.signal - console.log('\nsetting new action:', s.signal) - console.log('brain-age:', s.options.brain.age) - console.log('average Q-learning loss:', s.options.brain.average_loss_window.get_average()) - console.log('smooth-ish reward: ', s.options.brain.average_reward_window.get_average()) - console.log() - - if(!s.options.model && s.options.saved_since_ticker > s.options.save_every_x) { - //save trained brain to file if we are in training mode - savenet(s.options.brain) - s.options.saved_since_ticker = 0 - } - - s.options.saved_since_ticker += 1 - - cb() - }, - - - - onReport: function (s) { - var cols = [] - if (typeof s.period.macd_histogram === 'number') { - var color = 'grey' - if (s.period.macd_histogram > 0) { - color = 'green' - } - else if (s.period.macd_histogram < 0) { - color = 'red' - } - cols.push(z(8, n(s.period.macd_histogram).format('+00.0000'), ' ')[color]) - cols.push(z(8, n(s.period.overbought_rsi).format('00'), ' ').cyan) - } - else { - cols.push(' ') - } - return cols - } - } -} diff --git a/extensions/strategies/ql0/_codemap.js b/extensions/strategies/ql0/_codemap.js deleted file mode 100644 index 2ada9469db..0000000000 --- a/extensions/strategies/ql0/_codemap.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - _ns: 'zenbot', - - 'strategies.ql0': require('./strategy'), - 'strategies.list[]': '#strategies.ql0' -} diff --git a/extensions/strategies/ql0/strategy.js b/extensions/strategies/ql0/strategy.js deleted file mode 100644 index d26e3ac911..0000000000 --- a/extensions/strategies/ql0/strategy.js +++ /dev/null @@ -1,441 +0,0 @@ -var z = require('zero-fill') - , n = require('numbro') - , deepqlearn = require('../../../node_modules/convnetjs/build/deepqlearn.js') - , convnetjs = require('convnetjs') - , fs = require('fs') - -module.exports = function container (get, set, clear) { - return { - name: 'ql0', - description: 'Q-learning 0 (reinforcement learning) ', - initialized: false, - - //function getPeriodInMinutes(p) - getPeriodInMinutes: function(p) - { - if(p.indexOf('m') !== -1) - return Number(p.replace('m','')) - - if(p.indexOf('h') !== -1) - return Number(p.replace('h',''))*60 - }, - - - getOptions: function () { - // build-in - this.option('period', 'period length in minutes', String, '30m') - - //periodInMinutes = this.getPeriodInMinutes(s.options.period) - //this.option('min_periods', 'min. number of history periods', Number, 10) //43200 - this.option('brain', 'Brain object container', Object, null) - this.option('agent', 'Agent object container', Object, null) - - - // History containers - this.option('ma10_prev', '', Number, 0) - this.option('ma30_prev', '', Number, 0) - this.option('ma60_prev', '', Number, 0) - this.option('ma90_prev', '', Number, 0) - this.option('ma150_prev', '', Number, 0) - this.option('ma360_prev', '', Number, 0) - - - this.option('hold', 'Hold', Number, 0) - this.option('model', 'Path to pre-trained model', String, null) - this.option('order_finished', 'Indicator whether the order was finished', Boolean, true) - this.option('last_action', 'Indicator whether the order was finished', String, null) - - this.option('ticker_timeout_limit', 'How long the bot can go without checking new actions', Number, 10) - this.option('ticker', 'Value of previous profit', Number, 0) - this.option('ticker_expired', 'Value of previous profit', Boolean, false) - - this.option('previous_profit', 'Value of previous profit', Number, 0) - this.option('prev_mode', 'Prev mode', Number, 0) - this.option('data_ready', 'Data ready', Boolean, false) - this.option('holding_currency', 'Holding currency indicator', Boolean, false) - this.option('holding_assets', 'Holding assets indicator', Boolean, false) - this.option('saved_since_ticker', 'Incremental ticker since last save', Number, 0) - this.option('save_every_x', 'Save every x tick', Number, 1000) - - //this.option('floor_coef', 'floor coefficient', Number, 0.75) - //this.option('moon_coef', 'moon coefficient', Number, 1.05) - //this.option('resistance_coef', 'resistance coefficient', Number, 2.8) - - /* - this.option('ema_short_period', 'number of periods for the shorter EMA', Number, 12) - this.option('ema_long_period', 'number of periods for the longer EMA', Number, 26) - this.option('signal_period', 'number of periods for the signal EMA', Number, 9) - this.option('up_trend_threshold', 'threshold to trigger a buy signal', Number, 0) - this.option('down_trend_threshold', 'threshold to trigger a sold signal', Number, 0) - this.option('overbought_rsi_periods', 'number of periods for overbought RSI', Number, 25) - this.option('overbought_rsi', 'sold when RSI exceeds this value', Number, 70) - */ - }, - - - initialize: function(s){ - - function loadnet(brain, file) { - var dataNet = fs.readFileSync('value_net.json', 'utf8') - var j = JSON.parse(dataNet); - s.options.brain.value_net.fromJSON(j); - brain.learning = false; // stop learning - } - - - // Your initialization goes here - periodInMinutes = this.getPeriodInMinutes(s.options.period) - - //initialize q-grid - var layer_defs = []; - - //q-learning settings - var num_inputs = 26 - var num_actions = 3 // 3 possible actions (buy, sell, hold) - var temporal_window = 1 // amount of temporal memory. 0 = agent lives in-the-moment :) - var network_size = num_inputs*temporal_window + num_actions*temporal_window + num_inputs - - layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:network_size}); - layer_defs.push({type:'fc', num_neurons: 200, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 100, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'regression', num_neurons: num_actions}); - - -/* - layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:network_size}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'fc', num_neurons: 50, activation:'relu'}); - layer_defs.push({type:'regression', num_neurons: num_actions}); -*/ - // options for the Temporal Difference learner that trains the above net - // by backpropping the temporal difference learning rule. - var tdtrainer_options = {learning_rate:0.001, momentum:0.0, batch_size:64, l2_decay:0.01}; - - var opt = {}; - opt.temporal_window = temporal_window; - // size of experience replay memory - opt.experience_size = 30000; - // number of examples in experience replay memory before we begin learning - opt.start_learn_threshold = 1000; - // gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1] - opt.gamma = 0.8; - opt.learning_steps_total = 200000; - // how many steps of the above to perform only random actions (in the beginning)? - opt.learning_steps_burnin = 3000; - opt.epsilon_min = 0.05; - opt.epsilon_test_time = 0.01; - opt.layer_defs = layer_defs; - opt.tdtrainer_options = tdtrainer_options; - - //s.brain = new deepqlearn.Brain(num_inputs, num_actions, opt) - s.options.brain = new deepqlearn.Brain(num_inputs, num_actions, opt) - - if(s.options.model){ - loadnet(s.options.brain, s.options.model) - } - /* - let env = {}; - env.getNumStates = function() { return num_inputs; } - env.getMaxNumActions = function() { return num_actions } - let spec = { alpha: 0.01 } // see full options on DQN page - let Agent = reinforce.DQNAgent - s.options.agent = new Agent(env, spec) - */ - this.initialized = true - }, - - - /* Called every time when a trade was done - (for example if in 5min period was done 20 trades, the method will be called 20 times) - */ - calculate: function (s) { - - if(!this.initialized) - this.initialize(s) - //console.log('calculating...', s.period.period_id, '--', s.period.volume) - // get current ticker data - //get('lib.ema')(s, 'ma2', 2/periodInMinutes) - - var periodInMinutes = 30 - //get('lib.ema')(s, 'ma10', 10/periodInMinutes) - get('lib.ema')(s, 'ma30', 30/periodInMinutes) - get('lib.ema')(s, 'ma60', 60/periodInMinutes) - get('lib.ema')(s, 'ma90', 90/periodInMinutes) - get('lib.ema')(s, 'ma150', 150/periodInMinutes) - get('lib.ema')(s, 'ma360', 360/periodInMinutes) // 6 hours - - //get('lib.sma')(s, 'avg_vol10', 10/periodInMinutes, 'volume') - get('lib.sma')(s, 'avg_vol30', 30/periodInMinutes, 'volume') - get('lib.sma')(s, 'avg_vol60', 60/periodInMinutes, 'volume') - get('lib.sma')(s, 'avg_vol90', 90/periodInMinutes, 'volume') - get('lib.sma')(s, 'avg_vol150', 150/periodInMinutes, 'volume') - get('lib.sma')(s, 'avg_vol360', 360/periodInMinutes, 'volume') // 6 hours - - }, - - onPeriod: function (s, cb) { - - function addVolumeStates(s, states) { - p = s.period - // 2ma states - //states.push(p.avg_vol2 > p.avg_vol10) - //states.push(p.avg_vol2 > p.avg_vol30) - //states.push(p.avg_vol2 > p.avg_vol60) - //states.push(p.avg_vol2 > p.avg_vol90) - //states.push(p.avg_vol2 > p.avg_vol150) - //states.push(p.avg_vol2 > p.avg_vol360) - // 10avg_vol states - //states.push(p.avg_vol10 > p.avg_vol30) - //states.push(p.avg_vol10 > p.avg_vol60) - //states.push(p.avg_vol10 > p.avg_vol90) - //states.push(p.avg_vol10 > p.avg_vol150) - //states.push(p.avg_vol10 > p.avg_vol360) - // 30avg_vol states - states.push(p.avg_vol30 > p.avg_vol60) - states.push(p.avg_vol30 > p.avg_vol90) - states.push(p.avg_vol30 > p.avg_vol150) - states.push(p.avg_vol30 > p.avg_vol360) - // 60avg_vol states - states.push(p.avg_vol60 > p.avg_vol90) - states.push(p.avg_vol60 > p.avg_vol150) - states.push(p.avg_vol60 > p.avg_vol360) - // 90avg_vol states - states.push(p.avg_vol90 > p.avg_vol150) - states.push(p.avg_vol90 > p.avg_vol360) - // 150avg_vol states - states.push(p.avg_vol150 > p.avg_vol360) - } - - - function addPriceStates(s, states) { - p = s.period - // 2ma states - //states.push(p.ma2 > p.ma10) - //states.push(p.ma2 > p.ma30) - //states.push(p.ma2 > p.ma60) - //states.push(p.ma2 > p.ma90) - //states.push(p.ma2 > p.ma150) - //states.push(p.ma2 > p.ma360) - // 10ma states - //states.push(p.ma10 > p.ma30) - //states.push(p.ma10 > p.ma60) - //states.push(p.ma10 > p.ma90) - //states.push(p.ma10 > p.ma150) - //states.push(p.ma10 > p.ma360) - // 30ma states - states.push(p.ma30 > p.ma60) - states.push(p.ma30 > p.ma90) - states.push(p.ma30 > p.ma150) - states.push(p.ma30 > p.ma360) - // 60ma states - states.push(p.ma60 > p.ma90) - states.push(p.ma60 > p.ma150) - states.push(p.ma60 > p.ma360) - // 90ma states - states.push(p.ma90 > p.ma150) - states.push(p.ma90 > p.ma360) - // 150ma states - states.push(p.ma150 > p.ma360) - } - - - function addHistoryStates(s, states){ - p = s.period - o = s.options - - //states.push(p.ma10 > o.ma10_prev) - states.push(p.ma30 > o.ma30_prev) - states.push(p.ma60 > o.ma60_prev) - states.push(p.ma90 > o.ma90_prev) - states.push(p.ma150 > o.ma150_prev) - states.push(p.ma360 > o.ma360_prev) - - } - - - - function calcProfit(s) { - //if we didn't make any trades yet just return 0 - if(s.my_trades.length === 0) - return 0 - - var total_currency = parseInt(s.balance.asset)*s.period.close + parseInt(s.balance.currency) - var start_capital = s.start_capital - //var profit = total_currency - start_capital - var profit = total_currency - s.options.previous_profit - // TODO we can add also change from previous run - s.options.previous_profit = total_currency - var binary_profit = profit > 0 ? 1 : 0 - - console.log('\nprevious_profit: ', s.options.previous_profit, ', current_profit: ', profit, ', action:', s.action, ' last_signal: ', s.last_signal) - - return binary_profit - - // calculate profit - /*var profit = s.start_capital ? n(s.balance.currency).subtract(s.start_capital).divide(s.start_capital) : n(0) - var current_profit = profit.value() - var profit_slope = current_profit - s.options.previous_profit - //minor punishment - if(profit_slope === 0.0) - profit_slope = -0.01 - - console.log('\nprevious_profit: ', s.options.previous_profit, ', current_profit: ', current_profit, ', slope:', profit_slope, ', action:', s.action, ' last_signal: ', s.last_signal) - return profit_slope - */ - } - - - function savenet(brain) { - if(brain){ - var filename = 'value_net.json'; - var j = brain.value_net.toJSON(); - var t = JSON.stringify(j) - - fs.writeFile(filename, t, function(err) { - if(err) { - return console.log(err); - } - }); - } - } - - - function addCustomStates(s, states){ - var currentState = 0 - if(!s.action) - currentState = 0 - else if (s.action === 'bought') - currentState = 1 - else if (s.action === 'sold') - currentState = 2 - - //states.push(currentState) - - states.push(s.in_preroll) - } - - - function convertStatesToInt(states){ - for (i = 0; i < states.length; i++) { - if(typeof(states[i]) === "boolean") - states[i] = states[i] ? 1 : 0 - } - } - - - function savePeriodValues(s){ - //s.options.ma10_prev = s.period.ma10 - s.options.ma30_prev = s.period.ma30 - s.options.ma60_prev = s.period.ma60 - s.options.ma90_prev = s.period.ma90 - s.options.ma150_prev = s.period.ma150 - s.options.ma360_prev = s.period.ma360 - - } - - - - //************************* END OF FUNCTIONS DEFINITION SECTION - - - if(!s.options.brain || !s.period.ma360){ - cb() - return - } - - if (s.options.last_action === null || s.action === 'bought' || s.action === 'sold' || s.options.ticker > s.options.ticker_timeout_limit) - s.options.order_finished = true - - - // training happens here - if(s.options.order_finished){ - - // ***** Reward the last action - reward = calcProfit(s) - //s.options.agent.learn(reward) - s.options.brain.backward(reward) - //s.options.previous_profit = reward - p = s.period - - // ***** Save current states - var states = [] - addPriceStates(s, states) - addVolumeStates(s, states) - addCustomStates(s, states) - addHistoryStates(s, states) - convertStatesToInt(states) - - savePeriodValues(s) - - // ***** Predict next state - // var action = s.options.agent.act(states) - var action = s.options.brain.forward(states); - //console.log('signal:', action) -/* - if(action === 0) - s.signal = null - else if (action === 1) - s.signal = 'buy' - else if (action === 2) - s.signal = 'sell' -*/ - - if(action === 0) - s.signal = 'buy' - else if (action === 1) - s.signal = 'sell' - - - console.log('\nsetting new action:', s.signal) - - if(s.options.last_action != s.signal) - s.options.order_finished = false - - s.options.last_action = s.signal - s.options.ticker = 0 - - - console.log('brain-age:', s.options.brain.age) - console.log('average Q-learning loss:', s.options.brain.average_loss_window.get_average()) - console.log('smooth-ish reward: ', s.options.brain.average_reward_window.get_average()) - console.log() - } - - s.options.ticker += 1 - - - if(s.options.saved_since_ticker > s.options.save_every_x) { - //save trained brain to file - savenet(s.options.brain) - s.options.saved_since_ticker = 0 - } - - s.options.saved_since_ticker += 1 - - - cb() - }, - - - - onReport: function (s) { - var cols = [] - if (typeof s.period.macd_histogram === 'number') { - var color = 'grey' - if (s.period.macd_histogram > 0) { - color = 'green' - } - else if (s.period.macd_histogram < 0) { - color = 'red' - } - cols.push(z(8, n(s.period.macd_histogram).format('+00.0000'), ' ')[color]) - cols.push(z(8, n(s.period.overbought_rsi).format('00'), ' ').cyan) - } - else { - cols.push(' ') - } - return cols - } - } -} diff --git a/extensions/strategies/rsi/strategy.js b/extensions/strategies/rsi/strategy.js index d4a30d81bd..f841611fbe 100644 --- a/extensions/strategies/rsi/strategy.js +++ b/extensions/strategies/rsi/strategy.js @@ -14,7 +14,7 @@ module.exports = function container (get, set, clear) { this.option('overbought_rsi', 'sell when RSI reaches or goes above this value', Number, 82) this.option('rsi_recover', 'allow RSI to recover this many points before buying', Number, 3) this.option('rsi_drop', 'allow RSI to fall this many points before selling', Number, 0) - this.option('rsi_divisor', 'sell when RSI reaches high-water reading divided by this value', Number, 2) + this.option('rsi_dividend', 'sell when RSI reaches high-water reading divided by this value', Number, 2) }, calculate: function (s) { @@ -38,7 +38,7 @@ module.exports = function container (get, set, clear) { } if (s.trend === 'long') { s.rsi_high = Math.max(s.rsi_high, s.period.rsi) - if (s.period.rsi <= s.rsi_high / s.options.rsi_divisor) { + if (s.period.rsi <= s.rsi_high / s.options.rsi_dividend) { s.trend = 'short' s.signal = 'sell' } diff --git a/extensions/strategies/template/_codemap.js b/extensions/strategies/template/_codemap.js deleted file mode 100644 index ae2180b097..0000000000 --- a/extensions/strategies/template/_codemap.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - _ns: 'zenbot', - - 'strategies.honey_badger': require('./strategy'), - 'strategies.list[]': '#strategies.honey_badger' -} diff --git a/extensions/strategies/template/strategy.js b/extensions/strategies/template/strategy.js deleted file mode 100644 index 2f0e1688a8..0000000000 --- a/extensions/strategies/template/strategy.js +++ /dev/null @@ -1,70 +0,0 @@ -var z = require('zero-fill') - , n = require('numbro') - -module.exports = function container (get, set, clear) { - return { - name: 'template-strategy', - description: 'Your awsome strategy description', - - - getOptions: function () { - // build-in - this.option('period', 'period length in minutes', String, '10m') - - // Working globals - this.option('initialized', 'period length in minutes', Boolean, false) - }, - - - // --------------------INITIALIZATION--------------------- - // ------------------------------------------------------- - initialize: function(s){ - // Your initialization goes here.. - - s.options.initialized = true - }, - - - // --------------------CALCULATE-------------------------- - // ------------------------------------------------------- - // Called every time when a trade was done (for example if in 5min period was done 20 trades, the method will be called 20 times) - calculate: function (s) { - - if(!s.options.initialized) - s.options.initialize(s) - //get('lib.ema')(s, 'ma30', 30/periodInMinutes) - }, - - - // --------------------ON PERIOD-------------------------- - // ------------------------------------------------------- - // Called on every period specified in s.options.period - onPeriod: function (s, cb) { - - cb() - }, - - - // --------------------ON REPORT-------------------------- - // ------------------------------------------------------- - // Called on every report interval s.options.balance_snapshot_period - onReport: function (s) { - var cols = [] - if (typeof s.period.macd_histogram === 'number') { - var color = 'grey' - if (s.period.macd_histogram > 0) { - color = 'green' - } - else if (s.period.macd_histogram < 0) { - color = 'red' - } - cols.push(z(8, n(s.period.macd_histogram).format('+00.0000'), ' ')[color]) - cols.push(z(8, n(s.period.overbought_rsi).format('00'), ' ').cyan) - } - else { - cols.push(' ') - } - return cols - } - } -} diff --git a/extensions/strategies/trend_ema/strategy.js b/extensions/strategies/trend_ema/strategy.js index e77ab0eaab..1b4e939e01 100644 --- a/extensions/strategies/trend_ema/strategy.js +++ b/extensions/strategies/trend_ema/strategy.js @@ -93,4 +93,4 @@ module.exports = function container (get, set, clear) { return cols } } -} +} \ No newline at end of file diff --git a/fabfile.py b/fabfile.py index 6333940dfd..6fc15c652e 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1,32 +1,37 @@ import datetime import shlex + import subprocess from fabric.api import run, cd,local -from zen import path, pyexc, selectors + def sim(instrument, days, popsize, strategy): + with cd('zenbot'): params = dict(instrument=instrument, days=days, strategy=strategy, popsize=popsize, timestamp=datetime.datetime.now().strftime('%Y-%m-%d-%H-%M')) - cmd = "cd {path}/zen && {pyexc} -m scoop main.py {instrument} {days} {popsize} {strategy}".format(path=path, pyexc=pyexc, **params) - total = '("{cmd}" > {instrument}_{strategy}_{days}_{popsize}_{timestamp}.out 2>&1 &) && sleep 1'.format( + cmd = "cd zen && python -m scoop main.py {instrument} {days} {popsize} {strategy}".format(**params) + total = '(nohup docker-compose exec -T server bash -c "{cmd}" > {instrument}_{strategy}_{days}_{popsize}_{timestamp}.out 2>&1 &) && sleep 1'.format( cmd=cmd, **params) print(total) run(total) + def remote(cmd,logfile): - total = '("{cmd}" > {logfile} 2>&1 &) && sleep 1'.format(cmd=cmd, logfile=logfile) + with cd('zenbot'): + total = '(nohup docker-compose exec -T server bash -c "{cmd}" > {logfile} 2>&1 &) && sleep 1'.format(cmd=cmd, + logfile=logfile) print(total) run(total) -def backfill_remote(selector, TOTAL_DAYS): - products = selectors[selector] +def backfill_remote(TOTAL_DAYS): + products = ['gdax.BTC-EUR','gdax.BTC-USD','gdax.BTC-GBP']+['gdax.ETH-BTC','poloniex.ETH-BTC'] for instrument in products: - cmd = '{path}/zenbot.sh backfill {instrument} --days {days}'.format(path=path, days=TOTAL_DAYS, instrument=instrument) + cmd = '/app/zenbot.sh backfill {instrument} --days {days}'.format(days=TOTAL_DAYS, instrument=instrument) remote(cmd,'backfill_'+instrument) - -def backfill_local(selector, TOTAL_DAYS): - products = selectors[selector] +def backfill_local(TOTAL_DAYS): + products = ['gdax.BTC-EUR','gdax.BTC-USD','gdax.BTC-GBP']+['gdax.ETH-BTC','poloniex.ETH-BTC'] for instrument in products: - cmd = '{path}/zenbot.sh backfill {instrument} --days {days}'.format(path=path, days=TOTAL_DAYS, instrument=instrument) + cmd = '/app/zenbot.sh backfill {instrument} --days {days}'.format(days=TOTAL_DAYS, instrument=instrument) local(cmd) + diff --git a/lib/_codemap.js b/lib/_codemap.js index 7b82eacc2a..41c441f659 100644 --- a/lib/_codemap.js +++ b/lib/_codemap.js @@ -1,14 +1,10 @@ module.exports = { _ns: 'zenbot', _folder: 'lib', - _maps: [ - require('./talib/_codemap') - ], + 'ema': require('./ema'), 'engine': require('./engine'), 'normalize-selector': require('./normalize-selector'), 'rsi': require('./rsi'), - 'ema': require('./ema'), - 'sma': require('./sma'), 'stddev': require('./stddev') -} +} \ No newline at end of file diff --git a/lib/ema.js b/lib/ema.js index 429f7fb25c..b5815da429 100644 --- a/lib/ema.js +++ b/lib/ema.js @@ -1,7 +1,5 @@ module.exports = function container (get, set, clear) { - /* - Exponential Moving Average - */ return function ema (s, key, length, source_key) { + return function ema (s, key, length, source_key) { if (!source_key) source_key = 'close' if (s.lookback.length >= length) { var prev_ema = s.lookback[0][key] diff --git a/lib/sma.js b/lib/sma.js deleted file mode 100644 index 9bc1378b87..0000000000 --- a/lib/sma.js +++ /dev/null @@ -1,24 +0,0 @@ -module.exports = function container (get, set, clear) { - /* - Simple Moving Average - */ - return function sma (s, key, length, source_key) { - if (!source_key) source_key = 'close' - if (s.lookback.length >= length) { - var sum = 0 - /*console.log('s.lookback.length:', s.lookback.length ) - - var tx = s.lookback[0][source_key] - var time_key = 'time' - var t1 = s.lookback[0][time_key] - var t2 = s.lookback[length-1][time_key] - //console.log('t1-t2:', t1, '---', t2) - */ - s.lookback.slice(0, length).forEach(function (period) { - sum += period[source_key] - }) - s.period[key] = sum/length - //console.log('sma:', sum/length) - } - } -} diff --git a/lib/talib/_codemap.js b/lib/talib/_codemap.js deleted file mode 100644 index d021d3d07e..0000000000 --- a/lib/talib/_codemap.js +++ /dev/null @@ -1,19 +0,0 @@ -module.exports = { - _ns: 'zenbot' - - ,'talib.ema': require('./ema') - ,'talib.obv': require('./obv') - ,'talib.obvs': require('./obvs') - ,'talib.ad': require('./ad') - ,'talib.ads': require('./ads') - - - // To be reviewed - ,'talib.macd': require('./macd') - //,'talib.list[2]': '#talib.macd' - ,'talib.rsi': require('./rsi') - //,'talib.list[5]': '#talib.rsi' - ,'talib.adx': require('./adx') - //,'talib.list[7]': '#talib.adx' - //TODO: add OBV, ARONOSC -} diff --git a/lib/talib/ad.js b/lib/talib/ad.js deleted file mode 100644 index 9ec823b53a..0000000000 --- a/lib/talib/ad.js +++ /dev/null @@ -1,34 +0,0 @@ -var t = require('talib') - ,regression = require('regression') - -module.exports = function container (get, set, clear) { - /* - Average Directional Index: - An indicator that tracks the relationship between volume and price. - It is often considered a leading indicator because it shows when a - stock is being accumulated or distributed, foreshadowing major price moves. - */ - - return function ad (s, key, length) { - var close = s.lookback.map(function(a) { return a[this.key]}, {key: 'close'}); - close = close.slice(0, length) - var volume = s.lookback.map(function(a) { return a[this.key]}, {key: 'volume'}); - volume = volume.slice(0, length) - var high = s.lookback.map(function(a) { return a[this.key]}, {key: 'high'}); - high = high.slice(0, length) - var low = s.lookback.map(function(a) { return a[this.key]}, {key: 'low'}); - low = low.slice(0, length) - t.execute({ - name: "AD", - startIdx: 0, - endIdx: close.length - 1, - high: high, - low: low, - close: close, - volume: volume - - }, function (err, result) { - s.period[key] = result.result.outReal - }) - } -} diff --git a/lib/talib/ads.js b/lib/talib/ads.js deleted file mode 100644 index 6489a19fae..0000000000 --- a/lib/talib/ads.js +++ /dev/null @@ -1,41 +0,0 @@ -var t = require('talib') - ,regression = require('regression') - -module.exports = function container (get, set, clear) { - /* - Linear Slope of Average Directional Index: - An indicator that tracks the relationship between volume and price. - It is often considered a leading indicator because it shows when a - stock is being accumulated or distributed, foreshadowing major price moves. - */ - - return function ads (s, key, length) { - var close = s.lookback.map(function(a) { return a[this.key]}, {key: 'close'}); - close = close.slice(0, length) - var volume = s.lookback.map(function(a) { return a[this.key]}, {key: 'volume'}); - volume = volume.slice(0, length) - var high = s.lookback.map(function(a) { return a[this.key]}, {key: 'high'}); - high = high.slice(0, length) - var low = s.lookback.map(function(a) { return a[this.key]}, {key: 'low'}); - low = low.slice(0, length) - t.execute({ - name: "AD", - startIdx: 0, - endIdx: close.length - 1, - high: high, - low: low, - close: close, - volume: volume - - }, function (err, result) { - var obvs = result.result.outReal - var data = [], i = -1 - while ( obvs[++i] ) { - data.push( [ i, obvs[i] ] ); - } - var slope = regression('linear', data); - s.period[key] = slope.equation[0] - - }) - } -} diff --git a/lib/talib/adx.js b/lib/talib/adx.js deleted file mode 100644 index 9ad9525e22..0000000000 --- a/lib/talib/adx.js +++ /dev/null @@ -1,20 +0,0 @@ - -var t = require('talib') -module.exports = function container (get, set, clear) { - return function macd (s, options) { - if (!options) { - var options = 0 - } - t.execute({ - name: "ADX", - startIdx: 0, - endIdx: s.data.close.length - 1, - high: s.data.high, - low: s.data.low, - close: s.data.close, - optInTimePeriod: options.adx_period ? options.adx_period : 20, - }, function (err, result) { - s.period.adx = result.result.outReal - }); - } -} diff --git a/lib/talib/ema.js b/lib/talib/ema.js deleted file mode 100644 index 7f05acaa3f..0000000000 --- a/lib/talib/ema.js +++ /dev/null @@ -1,21 +0,0 @@ -var t = require('talib') -module.exports = function container (get, set, clear) { - /* - Exponential Moving Average - */ - return function ema (s, key, length, source_key) { - if (!source_key) source_key = 'close' - var data = s.lookback.map(function(a) { return a[this.key]}, {key: source_key}); - data = data.slice(0, length) - t.execute({ - name: "EMA", - startIdx: 0, - endIdx: data.length - 1, - inReal: data, - optInTimePeriod: length, - }, function (err, result) { - var indicator - s.period[key] = result.result.outReal.slice(-1)[0] - }) - } -} diff --git a/lib/talib/macd.js b/lib/talib/macd.js deleted file mode 100644 index 5677d7f797..0000000000 --- a/lib/talib/macd.js +++ /dev/null @@ -1,19 +0,0 @@ -var t = require('talib') -module.exports = function container (get, set, clear) { - return function macd (s, options) { - if (!options) { - var options = 0 - } - t.execute({ - name: "MACD", - startIdx: 0, - endIdx: s.data.close.length - 1, - inReal: s.data.close, - optInFastPeriod: options.macd_fast_period ? options.macd_fast_period: 12, - optInSlowPeriod: options.macd_slow_period ? options.macd_slow_period: 26, - optInSignalPeriod: options.macd_signal_period ? options.macd_signal_period: 9, - }, function (err, result) { - s.period.macd = result.result - }) - } -} diff --git a/lib/talib/obv.js b/lib/talib/obv.js deleted file mode 100644 index d713ff0f5b..0000000000 --- a/lib/talib/obv.js +++ /dev/null @@ -1,22 +0,0 @@ -var t = require('talib') -module.exports = function container (get, set, clear) { - /* - On-Balance Volume - */ - return function obv (s, key, length) { - var close = s.lookback.map(function(a) { return a[this.key]}, {key: 'close'}); - close = close.slice(0, length) - var volume = s.lookback.map(function(a) { return a[this.key]}, {key: 'volume'}); - volume = volume.slice(0, length) - t.execute({ - name: "OBV", - startIdx: 0, - endIdx: close.length - 1, - inReal: close, - volume: volume, - }, function (err, result) { - var indicator - s.period[key] = result.result.outReal.slice(-1)[0] - }) - } -} diff --git a/lib/talib/obvs.js b/lib/talib/obvs.js deleted file mode 100644 index 664d674dd8..0000000000 --- a/lib/talib/obvs.js +++ /dev/null @@ -1,29 +0,0 @@ -var t = require('talib') - ,regression = require('regression') - -module.exports = function container (get, set, clear) { - /* - Returns slope of On-Balance Volume Indicator - */ - return function obvs (s, key, length) { - var close = s.lookback.map(function(a) { return a[this.key]}, {key: 'close'}); - close = close.slice(0, length) - var volume = s.lookback.map(function(a) { return a[this.key]}, {key: 'volume'}); - volume = volume.slice(0, length) - t.execute({ - name: "OBV", - startIdx: 0, - endIdx: close.length - 1, - inReal: close, - volume: volume, - }, function (err, result) { - var obvs = result.result.outReal - var data = [], i = -1 - while ( obvs[++i] ) { - data.push( [ i, obvs[i] ] ); - } - var slope = regression('linear', data); - s.period[key] = slope.equation[0] - }) - } -} diff --git a/lib/talib/rsi.js b/lib/talib/rsi.js deleted file mode 100644 index a08d66ece1..0000000000 --- a/lib/talib/rsi.js +++ /dev/null @@ -1,24 +0,0 @@ -var t = require('talib') -module.exports = function container (get, set, clear) { - return function macd (s, options) { - if (!options) { - var options = 0 - } - t.execute({ - name: "RSI", - startIdx: 0, - endIdx: s.data.close.length - 1, - inReal: s.data.close, - optInTimePeriod: options.rsi_periods ? options.rsi_periods: 14, - }, function (err, result) { - s.period.rsi = result.result.outReal.slice(-1)[0] - }) - // console.log(s.period.rsi ? s.period.rsi : null) - if (s.options.auto_rsi && s.lookback[0]) { - get('lib.stddev')(s, 'rsi_stddev', Math.floor(s.options.rsi_periods * 7), 'rsi') - s.options.overbought_rsi = 50 + (1.5 * s.period.rsi_stddev) - s.options.oversold_rsi = 50 - (1.5 * s.period.rsi_stddev) - } - - } -} diff --git a/package.json b/package.json index 942a0810f3..28fed8df5a 100644 --- a/package.json +++ b/package.json @@ -32,11 +32,7 @@ "semver": "^5.3.0", "sosa_mongo": "^1.0.3", "timebucket": "^0.4.0", - "zero-fill": "^2.2.3", - "convnetjs": "^0.3.0", - "talib": "^1.0.1", - "regression": "^1.4.0", - "stats-lite": "^2.0.4" + "zero-fill": "^2.2.3" }, "devDependencies": { "eslint": "^4.0.0" diff --git a/zen/.idea/inspectionProfiles/profiles_settings.xml b/zen/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000000..c23ecacb3a --- /dev/null +++ b/zen/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/zen/.idea/misc.xml b/zen/.idea/misc.xml new file mode 100644 index 0000000000..12f37ed383 --- /dev/null +++ b/zen/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/zen/.idea/modules.xml b/zen/.idea/modules.xml new file mode 100644 index 0000000000..82bda8cb06 --- /dev/null +++ b/zen/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/zen/.idea/workspace.xml b/zen/.idea/workspace.xml new file mode 100644 index 0000000000..a57525783e --- /dev/null +++ b/zen/.idea/workspace.xml @@ -0,0 +1,736 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + DEFINITION_ORDER + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1496865691829 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/zen/.idea/zen.iml b/zen/.idea/zen.iml new file mode 100644 index 0000000000..fca0252aa7 --- /dev/null +++ b/zen/.idea/zen.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/zen/__init__.py b/zen/__init__.py index 020556d483..e69de29bb2 100644 --- a/zen/__init__.py +++ b/zen/__init__.py @@ -1 +0,0 @@ -from conf import path, pyexc, selectors diff --git a/zen/conf.py b/zen/conf.py index a06551754d..0fefe31d2d 100644 --- a/zen/conf.py +++ b/zen/conf.py @@ -1,18 +1,8 @@ -# Don't touch these lines!!! import random -runid=random.randint(1000,9999) -# User configurable parameters: -# ----------------------------------------------------------------------------------------- -# zenbot dir, ex. "app/zenbot" for docker "/full/path/of/zenbot/dir" for linux envinronment -path = '/home/gecko/zenbot' -# name of python executable -pyexc = 'python3.6' -# Autobackfill interval in seconds, min 600 secs -> 10 mins -bkfint = 1800 -# You can edit this, please respect the python dict syntax + selectors = { 'BTC-CUR': ['gdax.BTC-USD', 'gdax.BTC-EUR', 'gdax.BTC-GBP'], 'ETH-BTC': ['gdax.ETH-BTC'], @@ -20,22 +10,11 @@ 'ETH-USD': ['gdax.ETH-USD'], 'ETH-CUR': ['gdax.ETH-USD','gdax.ETH-EUR'], } - -# Concurrent threads partitions=2 - -# Swozny please explain these parameters selectivity = 0.3 -# Ditto +runid=random.randint(1000,9999) sigma = 20 - -# Ditto indpb = 0.3 - -# Ditto mutpb = 0.3 - -# Ditto cxpb = 0.3 - diff --git a/zen/evaluation.py b/zen/evaluation.py index 3eef62dc15..c37e6d4e15 100644 --- a/zen/evaluation.py +++ b/zen/evaluation.py @@ -6,14 +6,13 @@ import sys from typing import List -from blessings import Terminal +from termcolor import colored -from conf import partitions, path +from conf import partitions from evolution.individual_base import Individual from objective_function import soft_maximum_worst_case from parsing import parse_trades, args_for_strategy -term = Terminal() def pct(x): return x / 100.0 @@ -32,7 +31,7 @@ def runzen(cmdline): class Andividual(Individual): - BASE_COMMAND = '{path}/zenbot.sh sim {instrument} --strategy {strategy} --avg_slippage_pct 0.33 --filename temp.html' + BASE_COMMAND = '/app/zenbot.sh sim {instrument} --strategy {strategy} --avg_slippage_pct 0.33 --filename temp.html' def __init__(self, *args,**kwargs): super(Andividual, self).__init__(*args, **kwargs) self.args = args_for_strategy(self.strategy) @@ -40,7 +39,7 @@ def __init__(self, *args,**kwargs): self.append(50 + (random.random() - 0.5) * 100) def __repr__(self): - return f"{self.cmdline} {super(Andividual, self).__repr__()}" + return colored(f"{self.cmdline} {super(Andividual, self).__repr__()}", 'grey') @property def instrument(self): @@ -74,7 +73,7 @@ def format(key, value): @property def cmdline(self) -> str: - base = self.BASE_COMMAND.format(path=self.path, instrument=self.instrument, strategy=self.strategy) + base = self.BASE_COMMAND.format(instrument=self.instrument, strategy=self.strategy) result = ' '.join([base] + self.params) return result @@ -103,7 +102,7 @@ def convert(self, param, value): elif 'sar_max_af' == param: res = pct(value) else: - raise ValueError(term.red(f"I don't understand {param} please add it to evaluation.py")) + raise ValueError(colored(f"I don't understand {param} please add it to evaluation.py", 'red')) return param, res diff --git a/zen/evolution/core.py b/zen/evolution/core.py index 1e6577785a..a800f505cb 100644 --- a/zen/evolution/core.py +++ b/zen/evolution/core.py @@ -1,36 +1,36 @@ import random from deap.tools import History, Statistics -from blessings import Terminal +from termcolor import colored from conf import cxpb, mutpb from .utils import log_stuff -term = Terminal() def algorithm(individual,popsize,map,evaluate,select,breed,mutate,stats,history,hof): # Create initial Population and evaluate it population = set() - print(term.blue("Sampling an initial valid population, this may take a while...")) + print(colored(f"Sampling an initial valid population, this may take a while...", 'blue')) while len(population) < popsize: - print(term.blue(f"\nCurrently {len(population)} valid individuals")) + print(colored(f"Currently {len(population)} valid individuals", 'blue')) would_be = [individual() for _ in range(popsize)] evaluate_group(would_be,map,evaluate) population = population | set(would_be) # Commence evolution for g in range(0, 1000): log_stuff(g, history, hof, population, stats) - print(term.blue("It's breeding season, we're expecting new members of the tribe...")) + print(colored(f"It's breeding season, we're expecting new members of the tribe...", 'blue')) offspring = breed(population) - print(term.blue("Radiation and toxic waste are causing mutations in the population...")) + print(colored(f"Radiation and toxic waste are causing mutations in the population...", 'blue')) mutants = mutate(population) - print(term.blue("Summer is here, evaluating our new arrivals...")) + print(colored(f"Summer is here, evaluating our new arrivals...", 'blue')) evaluate_group(offspring + mutants, map,evaluate) survivors = select(set(offspring) | set(mutants) | population) population = survivors return hof + def evaluate_group(population, map, evaluate): invalid_ind = [ind for ind in population if not ind.fitness.valid] print(' ' * len(invalid_ind) + '|') @@ -46,16 +46,14 @@ def breed(population): child1, child2 = parent1 + parent2 offspring.append(child1) offspring.append(child2) - offspstr = str(len(offspring)) - print(term.green(offspstr) + term.blue(" children have been born.")) + print(colored(len(offspring), 'green') + colored(f" children have been born.", 'blue')) return offspring def mutate(population): mutants = [] for individual in population: - mutstr = str(len(mutants)) if random.random() < mutpb: mutants.append(~individual) - print(term.green(mutstr) + term.blue(" individuals have mutated.")) + print(colored(len(mutants), 'green') + colored(f" individuals have mutated.", 'blue')) return mutants diff --git a/zen/evolution/selection.py b/zen/evolution/selection.py index ba2d60d08f..e81a0134b7 100644 --- a/zen/evolution/selection.py +++ b/zen/evolution/selection.py @@ -1,15 +1,15 @@ import random from operator import attrgetter from typing import Iterable, Set -from blessings import Terminal + +from termcolor import colored + from conf import partitions from evolution.individual_base import Individual -term = Terminal() -term.grey = term.color(8) def harsh_winter(population: Set[Individual], count: int) -> Set[Individual]: - # Selects `popsize` many individuals from the current population. + """ Selects `popsize` many individuals from the current population.""" elitist_count = int(count * 0.3) specialist_count = int(count * 0.4 / partitions) elites = select_elites(population, elitist_count) @@ -39,14 +39,14 @@ def select_specialists(individuals: Iterable[Individual], count: int): def log_stuff(elites, rest: Set, specialists): - print(term.blue("\n\nWinter has come, weeding out the unworthy.")) + print(colored("\n\nWinter has come, weeding out the unworthy.", 'blue')) print(f"{len(elites)} Elites will survive, they're currently the strongest:") for elite in sorted(elites, key=attrgetter('objective'), reverse=True): - print(term.grey(str(elite))) + print(elite) print(f"{len(specialists)} Specialists will survive, they're the best in their domain:") for specialist in specialists: - print(term.grey(str(specialist))) + print(specialist) print(f"Some other have fought their way through:") - for r in random.sample(rest, 3): - print(term.grey(str(r))) - print(term.grey("...")) + for r in random.sample(rest, len(rest) // 5): + print(r) + print(colored('...', 'grey')) diff --git a/zen/evolution/utils.py b/zen/evolution/utils.py index 548decc166..eab2ccf20b 100644 --- a/zen/evolution/utils.py +++ b/zen/evolution/utils.py @@ -2,12 +2,11 @@ import numpy from deap.tools import Statistics from matplotlib import pyplot as plt -from blessings import Terminal +from termcolor import colored -from conf import runid, path +from conf import runid from objective_function import soft_maximum_worst_case -term = Terminal() def draw(history, toolbox): ax = plt.figure() @@ -21,7 +20,7 @@ def draw(history, toolbox): positions = networkx.drawing.nx_agraph.graphviz_layout(graph, prog="dot") networkx.draw(graph, positions, node_color=colors, ax=ax.add_subplot(111), figsize=(30, 30), node_size=150) - ax.savefig('{path}/zen/logs/history/{runid}.png'.format(path=path, runid=runid)) + ax.savefig('logs/history/{runid}.png'.format(runid=runid)) def log_stuff(g, history, hof, population, stats): @@ -29,7 +28,7 @@ def log_stuff(g, history, hof, population, stats): record = stats.compile(population) hof.update(population) hof.persist() - print(term.green(f'\nGeneration {g} {record}')) + print(colored(f'\nGeneration {g} {record}','green') ) # print(hof) diff --git a/zen/halloffame.py b/zen/halloffame.py index 2332e43778..dd3fa275a4 100644 --- a/zen/halloffame.py +++ b/zen/halloffame.py @@ -1,6 +1,6 @@ from operator import attrgetter -from conf import runid, path +from conf import runid class ObjectiveFunctionHallOfFame(object): @@ -21,8 +21,8 @@ def len(self): def __repr__(self): header = ["Current Hall of Fame:"] report = [f"{ind}" for ind in sorted(self.inner, key=attrgetter('objective'), reverse=True)] - return "\n\n".join(header + report) + return "\n".join(header + report) def persist(self): - with open('{path}/zen/logs/hof/{runid}.txt'.format(path=path, runid=runid), 'w') as f: + with open('logs/hof/{runid}.txt'.format(runid=runid), 'w') as f: f.write(str(self)) diff --git a/zen/logs/history/placeholder.md b/zen/logs/history/placeholder.md deleted file mode 100644 index 4063542c22..0000000000 --- a/zen/logs/history/placeholder.md +++ /dev/null @@ -1,2 +0,0 @@ -Graphs logs will go here - diff --git a/zen/logs/hof/placeholder.md b/zen/logs/hof/placeholder.md deleted file mode 100644 index 4282a47b54..0000000000 --- a/zen/logs/hof/placeholder.md +++ /dev/null @@ -1,2 +0,0 @@ -Hall of fame logs will go here - diff --git a/zen/main.py b/zen/main.py index 5e9ce0bd93..076d8e761b 100644 --- a/zen/main.py +++ b/zen/main.py @@ -1,34 +1,30 @@ import sys from functools import partial -from timer import ThreadTimer + from deap.tools import cxTwoPoint, mutGaussian from scoop import shared -from blessings import Terminal -from subprocess import run, DEVNULL -from conf import indpb, sigma, partitions, selectors, path, bkfint +from termcolor import colored + +from conf import indpb, sigma, partitions, selectors from evaluation import evaluate_zen, Andividual from evolution import evolve +blue = partial(lambda text, color: colored(str(text), color), color='blue') +green = partial(lambda text, color: colored(str(text), color), color='green') -term = Terminal() -def main(instrument, days, popsize, strategy='trend_ema', nobf=None): - print(term.blue("Starting evolution....")) +def main(instrument, days, popsize, strategy='trend_ema'): + print(colored("Starting evolution....", 'blue')) evaluate = partial(evaluate_zen, days=days) - print(term.blue("Evaluating ") + term.green(str(popsize)) + term.blue(" individuals over ") - + term.green(str(days)) + term.blue(" days in ") + term.green(str(partitions)) + term.blue(" partitions.")) - Andividual.path = path + print(blue("Evaluating ")+green(popsize)+blue(" individuals over ") + green(days) + blue(' days in ') + green(partitions) + blue(' partitions.')) Andividual.instruments = selectors[instrument] Andividual.mate = cxTwoPoint Andividual.mutate = partial(mutGaussian, mu=0, sigma=sigma, indpb=indpb) Andividual.strategy = strategy - print(term.blue("Mating function is ") + term.green(str(Andividual.mate))) - print(term.blue("Mutating function is ") + term.green(str(Andividual.mutate))) + print(colored(f"Mating function is ", 'blue') + colored(Andividual.mate, 'green')) + print(colored(f"Mutating function is ", 'blue') + colored(Andividual.mutate, 'green')) res = evolve(evaluate, Andividual, popsize) return res -def bkfcmd(): - run(f"cd {path} && fab backfill_local:\"{INSTRUMENT},{TOTAL_DAYS}\"", shell=True, stdout=DEVNULL) - return if __name__ == '__main__': INSTRUMENT = sys.argv[1] @@ -37,32 +33,11 @@ def bkfcmd(): popsize = int(sys.argv[3]) except: popsize = 10 - if popsize < 10: - popsize = 10 - print(term.red("Minimum population size is 10, overriding...")) try: strategy = sys.argv[4] except: strategy = 'trend_ema' - try: - nobf = sys.argv[5] - if nobf == "backfill": - bkf = True - except: - bkf = False - if bkf == True: - print(term.green("Auto backfilling is ON")) - print(term.green("Executing inital backfilling, please wait...")) - bkfcmd() - if bkfint < 600: - print(term.red("Minimum interval for auto backfilling is 600 secs, overriding...")) - bkfint = 600 - print(term.green("Done")) - print(term.green(f"Auto backfilling interval is {bkfint} secs")) - ThreadTimer(bkfint, bkfcmd) - else: - print(term.red("You don\'t have enabled autobackfill,")) - print(term.red("MAKE SURE YOU RUN fab backfill_local:\",\"")) - print(term.red("otherwise it's all crap")) + print(colored("MAKE SURE YOU RUN fab backfill_local:", 'red')) + print(colored("otherwise it's all crap", 'red')) res = main(INSTRUMENT, TOTAL_DAYS, popsize, strategy) print(res) diff --git a/zen/parsing.py b/zen/parsing.py index c199bcbce0..efd3eff8cb 100644 --- a/zen/parsing.py +++ b/zen/parsing.py @@ -1,6 +1,6 @@ import shlex import subprocess -from conf import path + def parse_trades(stuff): """ @@ -13,7 +13,7 @@ def parse_trades(stuff): def args_for_strategy(strat): - available = subprocess.check_output(shlex.split('{path}/zenbot.sh list-strategies'.format(path=path))) + available = subprocess.check_output(shlex.split('/app/zenbot.sh list-strategies')) strats = [strat.strip() for strat in available.split(b'\n\n')] groups = [group.splitlines() for group in strats] output = {split[0].split()[0]: split[1:] for split in groups if split} diff --git a/zen/requirements.txt b/zen/requirements.txt new file mode 100644 index 0000000000..53e7ab0378 --- /dev/null +++ b/zen/requirements.txt @@ -0,0 +1,8 @@ +deap +numpy +networkx +matplotlib +scoop +names +fabric3 +termcolor diff --git a/zen/tests/.cache/v/cache/lastfailed b/zen/tests/.cache/v/cache/lastfailed new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/zen/tests/.cache/v/cache/lastfailed @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/zen/tests/__init__.py b/zen/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/zen/timer.py b/zen/timer.py deleted file mode 100644 index 4a4e091892..0000000000 --- a/zen/timer.py +++ /dev/null @@ -1,25 +0,0 @@ -import time -from threading import Event, Thread -from blessings import Terminal - -term = Terminal() - -class ThreadTimer: - - def __init__(self, interval, function): - self.interval = interval - self.function = function - self.start = time.time() - self.event = Event() - self.thread = Thread(target=self._target) - self.thread.start() - - def _target(self): - while not self.event.wait(self._time): - with term.location(x=term.width-14): - print(term.green("Backfilling..."), end='') - self.function() - - @property - def _time(self): - return self.interval - ((time.time() - self.start) % self.interval) diff --git a/zen/utils.py b/zen/utils.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/zen/zendividual.py b/zen/zendividual.py new file mode 100644 index 0000000000..e69de29bb2