Merge pull request #332 from enarjord/v5.5.0_ohlcv

V5.5.0 ohlcv

README.md

@@ -4,7 +4,8 @@
 
 :warning: **Use at own risk** :warning:
 
-v5.4.3  
+v5.5.0  
+
 
 ## Overview
 

auto_profit_transfer.py

@@ -0,0 +1,83 @@
+import os
+
+if "NOJIT" not in os.environ:
+    os.environ["NOJIT"] = "true"
+
+import traceback
+import json
+import argparse
+import asyncio
+from procedures import create_binance_bot, make_get_filepath
+from pure_funcs import get_template_live_config, flatten
+from njit_funcs import round_dynamic
+from time import sleep
+import logging
+import logging.config
+
+
+async def main():
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        level=logging.INFO,
+        datefmt="%Y-%m-%dT%H:%M:%S",
+    )
+    parser = argparse.ArgumentParser(
+        prog="auto profit transfer",
+        description="automatically transfer percentage of profits from futures wallet to spot wallet",
+    )
+    parser.add_argument("user", type=str, help="user/account_name defined in api-keys.json")
+    parser.add_argument(
+        "-p",
+        "--percentage",
+        type=float,
+        required=False,
+        default=0.5,
+        dest="percentage",
+        help="per uno, i.e. 0.02==2%.  default=0.5",
+    )
+    args = parser.parse_args()
+    config = get_template_live_config()
+    config["user"] = args.user
+    config["symbol"] = "BTCUSDT"  # dummy symbol
+    config["market_type"] = "futures"
+    bot = await create_binance_bot(config)
+    transfer_log_fpath = make_get_filepath(
+        os.path.join("logs", f"automatic_profit_transfer_log_{config['user']}.json")
+    )
+    try:
+        already_transferred_ids = set(json.load(open(transfer_log_fpath)))
+        logging.info(f"loaded already transferred IDs: {transfer_log_fpath}")
+    except:
+        already_transferred_ids = set()
+        logging.info(f"no previous transfers to load")
+    while True:
+        now = (await bot.public_get(bot.endpoints["time"]))["serverTime"]
+        try:
+            income = await bot.get_all_income(start_time=now - 1000 * 60 * 60 * 24)
+        except Exception as e:
+            logging.error(f"failed fetching income {e}")
+            traceback.print_exc()
+            income = []
+        income = [e for e in income if e["transaction_id"] not in already_transferred_ids]
+        profit = sum([e["income"] for e in income])
+        to_transfer = round_dynamic(profit * args.percentage, 4)
+        if to_transfer > 0:
+            try:
+                transferred = await bot.private_post(
+                    bot.endpoints["futures_transfer"],
+                    {"asset": "USDT", "amount": to_transfer, "type": 2},
+                    base_endpoint=bot.spot_base_endpoint,
+                )
+                logging.info(f"income: {profit} transferred {to_transfer} USDT")
+                already_transferred_ids.update([e["transaction_id"] for e in income])
+                json.dump(list(already_transferred_ids), open(transfer_log_fpath, "w"))
+            except Exception as e:
+                logging.error(f"failed transferring {e}")
+                traceback.print_exc()
+        else:
+            logging.info("nothing to transfer")
+        sleep(60 * 60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backtest.py

@@ -10,7 +10,7 @@
 import numpy as np
 import pandas as pd
 
-from downloader import Downloader
+from downloader import Downloader, load_hlc_cache
 from njit_funcs import backtest_static_grid, round_
 from njit_funcs_recursive_grid import backtest_recursive_grid
 from plotting import dump_plots
@@ -127,6 +127,12 @@ async def main():
         default=None,
         help="set n backtest slices to plot",
     )
+    parser.add_argument(
+        "-oh",
+        "--ohlcv",
+        help="use 1m ohlcv instead of 1s ticks",
+        action="store_true",
+    )
 
     args = parser.parse_args()
     for symbol in args.symbol.split(","):
@@ -155,8 +161,8 @@ async def main():
             config["short"]["enabled"] = "y" in args.short_enabled.lower()
         if "spot" in config["market_type"]:
             live_config = spotify_config(live_config)
+        config["ohlcv"] = args.ohlcv
 
-        downloader = Downloader(config)
         print()
         for k in (
             keys := [
@@ -170,12 +176,23 @@ async def main():
                 "start_date",
                 "end_date",
                 "latency_simulation_ms",
+                "base_dir",
             ]
         ):
             if k in config:
                 print(f"{k: <{max(map(len, keys)) + 2}} {config[k]}")
         print()
-        data = await downloader.get_sampled_ticks()
+        if config["ohlcv"]:
+            data = load_hlc_cache(
+                symbol,
+                config["start_date"],
+                config["end_date"],
+                base_dir=config["base_dir"],
+                spot=config["spot"],
+            )
+        else:
+            downloader = Downloader(config)
+            data = await downloader.get_sampled_ticks()
         config["n_days"] = round_((data[-1][0] - data[0][0]) / (1000 * 60 * 60 * 24), 0.1)
         pprint.pprint(denumpyize(live_config))
         plot_wrap(config, data)

balance_overview.ipynb

@@ -36,7 +36,7 @@
    "outputs": [],
    "source": [
     "plt.rcParams[\"figure.figsize\"] = [21, 13]\n",
-    "pd.set_option(\"precision\", 10)"
+    "pd.set_option(\"display.precision\", 10)"
    ]
   },
   {
@@ -47,10 +47,10 @@
    "source": [
     "config = get_template_live_config()\n",
     "config[\"market_type\"] = \"futures\"\n",
-    "config[\"user\"] = \"user_name\"\n",
+    "config[\"user\"] = \"binance_01\"\n",
     "config[\"exchange\"], _, _ = load_exchange_key_secret(config[\"user\"])\n",
     "\n",
-    "n_days = 30\n",
+    "n_days = 21\n",
     "start_time = (time() - 60 * 60 * 24 * n_days) * 1000\n",
     "end_time = time() * 1000\n",
     "\n",
@@ -162,13 +162,6 @@
    "outputs": [],
    "source": []
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -179,7 +172,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -193,7 +186,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.3"
+   "version": "3.9.7"
   }
  },
  "nbformat": 4,

binance.py

@@ -104,6 +104,7 @@ async def init_market_type(self):
             self.inverse = self.config["inverse"] = False
             self.base_endpoint = fapi_endpoint
             self.endpoints = {
+                "time": "/fapi/v1/time",
                 "position": "/fapi/v2/positionRisk",
                 "balance": "/fapi/v2/balance",
                 "exchange_info": "/fapi/v1/exchangeInfo",
@@ -134,6 +135,7 @@ async def init_market_type(self):
                 self.market_type += "_inverse_coin_margined"
                 self.inverse = self.config["inverse"] = True
                 self.endpoints = {
+                    "time": "/dapi/v1/time",
                     "position": "/dapi/v1/positionRisk",
                     "balance": "/dapi/v1/balance",
                     "exchange_info": "/dapi/v1/exchangeInfo",
@@ -159,6 +161,7 @@ async def init_market_type(self):
 
         self.spot_base_endpoint = "https://api.binance.com"
         self.endpoints["transfer"] = "/sapi/v1/asset/transfer"
+        self.endpoints["futures_transfer"] = "/sapi/v1/futures/transfer"
         self.endpoints["account"] = "/api/v3/account"
 
     async def _init(self):
@@ -448,7 +451,7 @@ async def fetch_income(
                     "timestamp": float(e["time"]),
                     "info": e["info"],
                     "transaction_id": float(e["tranId"]),
-                    "trade_id": float(e["tradeId"]),
+                    "trade_id": float(e["tradeId"]) if e["tradeId"] != "" else 0,
                 }
                 for e in fetched
             ]

configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json

@@ -1,30 +1,30 @@
-{"config_name": "multi_80symbols_2021-05-01to2022-01-26_recursive_grid_auto_unstuck_enabled",
+{"config_name": "recursive_grid_126_symbols_285days",
  "logging_level": 0,
- "long": {"auto_unstuck_ema_dist": 0.04533399066282969,
-          "auto_unstuck_wallet_exposure_threshold": 0.5089280603807499,
-          "ddown_factor": 0.1945371972359446,
-          "ema_span_0": 2344.941992440916,
-          "ema_span_1": 3569.8635166063227,
+ "long": {"auto_unstuck_ema_dist": 0.0008943228930055836,
+          "auto_unstuck_wallet_exposure_threshold": 0.12672073730468353,
+          "ddown_factor": 0.09916040750089808,
+          "ema_span_0": 789.7249165793291,
+          "ema_span_1": 1440,
           "enabled": true,
-          "initial_eprice_ema_dist": 0.05,
-          "initial_qty_pct": 0.011188848380606475,
-          "markup_range": 0.0466190685279263,
-          "min_markup": 0.03771922872466367,
-          "n_close_orders": 11,
-          "rentry_pprice_dist": 0.012851520514242834,
-          "rentry_pprice_dist_wallet_exposure_weighting": 30.493849924344417,
-          "wallet_exposure_limit": 0.05},
- "short": {"auto_unstuck_ema_dist": 0.014817833487147757,
-           "auto_unstuck_wallet_exposure_threshold": 0.6984580115288681,
-           "ddown_factor": 2.346969025866142,
-           "ema_span_0": 4677.572747718445,
-           "ema_span_1": 4313.626811916282,
+          "initial_eprice_ema_dist": 0,
+          "initial_qty_pct": 0.01,
+          "markup_range": 0.04699693966985813,
+          "min_markup": 0.04298063900843883,
+          "n_close_orders": 10,
+          "rentry_pprice_dist": 0.02209348612738956,
+          "rentry_pprice_dist_wallet_exposure_weighting": 11.95394791078704,
+          "wallet_exposure_limit": 0.1},
+ "short": {"auto_unstuck_ema_dist": 0.02360107918060952,
+           "auto_unstuck_wallet_exposure_threshold": 0.8696017484288251,
+           "ddown_factor": 2.8578807917477422,
+           "ema_span_0": 1845.9622772519149,
+           "ema_span_1": 2220.149357438459,
            "enabled": false,
-           "initial_eprice_ema_dist": -0.04258632696780627,
-           "initial_qty_pct": 0.09718074802124348,
-           "markup_range": 0.027535790140754787,
-           "min_markup": 0.01832088726601801,
-           "n_close_orders": 8,
-           "rentry_pprice_dist": 0.039694739209284916,
-           "rentry_pprice_dist_wallet_exposure_weighting": 61.012162690905335,
-           "wallet_exposure_limit": 0.05}}
+           "initial_eprice_ema_dist": -0.08285630558057448,
+           "initial_qty_pct": 0.030171219081039008,
+           "markup_range": 0.023949596946105916,
+           "min_markup": 0.03925744129630296,
+           "n_close_orders": 5,
+           "rentry_pprice_dist": 0.024734943307523287,
+           "rentry_pprice_dist_wallet_exposure_weighting": 54.338880623212155,
+           "wallet_exposure_limit": 0.1}}