diff --git a/README.md b/README.md index 69946f406..33cd54019 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ :warning: **Used at one's own risk** :warning: -v7.2.7 +v7.2.8 ## Overview diff --git a/configs/template.json b/configs/template.json index 0616b5567..6c0e3b4a3 100644 --- a/configs/template.json +++ b/configs/template.json @@ -4,56 +4,56 @@ "exchange": "binance", "start_date": "2021-05-01", "starting_balance": 100000.0}, - "bot": {"long": {"close_grid_markup_range": 0.0016219, - "close_grid_min_markup": 0.012842, - "close_grid_qty_pct": 0.65242, - "close_trailing_grid_ratio": 0.021638, - "close_trailing_qty_pct": 0.88439, - "close_trailing_retracement_pct": 0.028672, - "close_trailing_threshold_pct": 0.065293, - "ema_span_0": 465.26, + "bot": {"long": {"close_grid_markup_range": 0.0013425, + "close_grid_min_markup": 0.0047292, + "close_grid_qty_pct": 0.85073, + "close_trailing_grid_ratio": 0.037504, + "close_trailing_qty_pct": 0.54254, + "close_trailing_retracement_pct": 0.021623, + "close_trailing_threshold_pct": 0.065009, + "ema_span_0": 469.33, "ema_span_1": 1120.5, - "entry_grid_double_down_factor": 2.3744, - "entry_grid_spacing_pct": 0.052341, - "entry_grid_spacing_weight": 0.070271, - "entry_initial_ema_dist": -0.0059754, - "entry_initial_qty_pct": 0.029454, - "entry_trailing_grid_ratio": -0.28169, - "entry_trailing_retracement_pct": 0.0024748, - "entry_trailing_threshold_pct": -0.051708, - "filter_relative_volume_clip_pct": 0.51416, - "filter_rolling_window": 60.0, - "n_positions": 10.675, - "total_wallet_exposure_limit": 0.95859, - "unstuck_close_pct": 0.071741, - "unstuck_ema_dist": -0.053527, - "unstuck_loss_allowance_pct": 0.033558, - "unstuck_threshold": 0.49002}, - "short": {"close_grid_markup_range": 0.0049057, - "close_grid_min_markup": 0.013579, - "close_grid_qty_pct": 0.6168, - "close_trailing_grid_ratio": 0.88873, - "close_trailing_qty_pct": 0.97705, - "close_trailing_retracement_pct": 0.095287, - "close_trailing_threshold_pct": -0.060579, - "ema_span_0": 819.23, - "ema_span_1": 246.39, - "entry_grid_double_down_factor": 2.3062, - "entry_grid_spacing_pct": 0.072015, - "entry_grid_spacing_weight": 1.4565, - "entry_initial_ema_dist": -0.072047, - "entry_initial_qty_pct": 0.072205, - "entry_trailing_grid_ratio": -0.02319, - "entry_trailing_retracement_pct": 0.017338, - "entry_trailing_threshold_pct": -0.084177, - "filter_relative_volume_clip_pct": 0.5183, - "filter_rolling_window": 68.072, - "n_positions": 1.1534, - "total_wallet_exposure_limit": 0.209, - "unstuck_close_pct": 0.052695, - "unstuck_ema_dist": -0.026947, - "unstuck_loss_allowance_pct": 0.046017, - "unstuck_threshold": 0.58422}}, + "entry_grid_double_down_factor": 2.2661, + "entry_grid_spacing_pct": 0.05224, + "entry_grid_spacing_weight": 0.070246, + "entry_initial_ema_dist": -0.015187, + "entry_initial_qty_pct": 0.032679, + "entry_trailing_grid_ratio": -0.29357, + "entry_trailing_retracement_pct": 0.002646, + "entry_trailing_threshold_pct": -0.043522, + "filter_relative_volume_clip_pct": 0.51429, + "filter_rolling_window": 330.17, + "n_positions": 5.2399, + "total_wallet_exposure_limit": 1.2788, + "unstuck_close_pct": 0.05968, + "unstuck_ema_dist": -0.027416, + "unstuck_loss_allowance_pct": 0.035915, + "unstuck_threshold": 0.45572}, + "short": {"close_grid_markup_range": 0.0020933, + "close_grid_min_markup": 0.016488, + "close_grid_qty_pct": 0.93256, + "close_trailing_grid_ratio": 0.035892, + "close_trailing_qty_pct": 0.98975, + "close_trailing_retracement_pct": 0.0042704, + "close_trailing_threshold_pct": -0.046918, + "ema_span_0": 1174.4, + "ema_span_1": 1217.3, + "entry_grid_double_down_factor": 2.0966, + "entry_grid_spacing_pct": 0.070355, + "entry_grid_spacing_weight": 1.5293, + "entry_initial_ema_dist": -0.090036, + "entry_initial_qty_pct": 0.07003, + "entry_trailing_grid_ratio": 0.075994, + "entry_trailing_retracement_pct": 0.023943, + "entry_trailing_threshold_pct": -0.079098, + "filter_relative_volume_clip_pct": 0.49361, + "filter_rolling_window": 57.016, + "n_positions": 1.1103, + "total_wallet_exposure_limit": 0.0, + "unstuck_close_pct": 0.063395, + "unstuck_ema_dist": -0.025704, + "unstuck_loss_allowance_pct": 0.04867, + "unstuck_threshold": 0.58437}}, "live": {"approved_coins": [], "auto_gs": true, "coin_flags": {}, @@ -128,9 +128,10 @@ "crossover_probability": 0.7, "iters": 30000, "limits": {"lower_bound_drawdown_worst": 0.25, - "lower_bound_equity_balance_diff_mean": 0.01, + "lower_bound_drawdown_worst_mean_1pct": 0.15, + "lower_bound_equity_balance_diff_mean": 0.02, "lower_bound_loss_profit_ratio": 0.6}, "mutation_probability": 0.2, "n_cpus": 5, "population_size": 500, - "scoring": ["mdg", "sharpe_ratio"]}} \ No newline at end of file + "scoring": ["mdg", "sortino_ratio"]}} \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md index ef2b59fc6..144a7a1b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -195,6 +195,8 @@ When optimizing, parameter values are within the lower and upper bounds. - Default values are median daily gain and Sharpe ratio. - The script uses the NSGA-II algorithm (Non-dominated Sorting Genetic Algorithm II) for multi-objective optimization. - The fitness function is set up to minimize both objectives (converted to negative values internally). + - Options: adg, mdg, sharpe_ratio, sortino_ratio, omega_ratio, calmar_ratio, sterling_ratio + - Examples: ["mdg", "sharpe_ratio"], ["adg", "sortino_ratio"], ["sortino_ratio", "omega_ratio"] ### Optimization Limits diff --git a/passivbot-rust/src/backtest.rs b/passivbot-rust/src/backtest.rs index 397fa24a3..a8ab1aa1d 100644 --- a/passivbot-rust/src/backtest.rs +++ b/passivbot-rust/src/backtest.rs @@ -1450,54 +1450,142 @@ pub fn analyze_backtest(fills: &[Fill], equities: &Vec) -> Analysis { // Calculate daily equities let mut daily_eqs = Vec::new(); let mut current_day = 0; - let mut sum = 0.0; - let mut count = 0; + let mut current_min = equities[0]; + for (i, &equity) in equities.iter().enumerate() { let day = i / 1440; if day > current_day { - daily_eqs.push(sum / count as f64); + daily_eqs.push(current_min); current_day = day; - sum = equity; - count = 1; + current_min = equity; } else { - sum += equity; - count += 1; + current_min = current_min.min(equity); } } - if count > 0 { - daily_eqs.push(sum / count as f64); + if current_min != f64::INFINITY { + daily_eqs.push(current_min); } // Calculate daily percentage changes let daily_eqs_pct_change: Vec = daily_eqs.windows(2).map(|w| (w[1] - w[0]) / w[0]).collect(); - // Calculate ADG and Sharpe ratio + // Calculate ADG and standard metrics let adg = daily_eqs_pct_change.iter().sum::() / daily_eqs_pct_change.len() as f64; - // Calculate MDG - let mut sorted_pct_change = daily_eqs_pct_change.clone(); - sorted_pct_change.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); - let mdg = if sorted_pct_change.len() % 2 == 0 { - (sorted_pct_change[sorted_pct_change.len() / 2 - 1] - + sorted_pct_change[sorted_pct_change.len() / 2]) - / 2.0 - } else { - sorted_pct_change[sorted_pct_change.len() / 2] + let mdg = { + let mut sorted_pct_change = daily_eqs_pct_change.clone(); + sorted_pct_change.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + if sorted_pct_change.len() % 2 == 0 { + (sorted_pct_change[sorted_pct_change.len() / 2 - 1] + + sorted_pct_change[sorted_pct_change.len() / 2]) + / 2.0 + } else { + sorted_pct_change[sorted_pct_change.len() / 2] + } }; - // Calculate Sharpe Ratio + + // Calculate variance and standard deviation let variance = daily_eqs_pct_change .iter() .map(|&x| (x - adg).powi(2)) .sum::() / daily_eqs_pct_change.len() as f64; - let sharpe_ratio = adg / variance.sqrt(); + let std_dev = variance.sqrt(); + + // Calculate Sharpe Ratio + let sharpe_ratio = if std_dev != 0.0 { adg / std_dev } else { 0.0 }; + + // Calculate Sortino Ratio (using downside deviation) + let downside_returns: Vec = daily_eqs_pct_change + .iter() + .filter(|&&x| x < 0.0) + .cloned() + .collect(); + let downside_deviation = if !downside_returns.is_empty() { + (downside_returns.iter().map(|x| x.powi(2)).sum::() / downside_returns.len() as f64) + .sqrt() + } else { + 0.0 + }; + let sortino_ratio = if downside_deviation != 0.0 { + adg / downside_deviation + } else { + 0.0 + }; + + // Calculate Omega Ratio (threshold = 0) + let (gains_sum, losses_sum) = + daily_eqs_pct_change + .iter() + .fold((0.0, 0.0), |(gains, losses), &ret| { + if ret >= 0.0 { + (gains + ret, losses) + } else { + (gains, losses + ret.abs()) + } + }); + let omega_ratio = if losses_sum != 0.0 { + gains_sum / losses_sum + } else { + f64::INFINITY + }; + + // Calculate Expected Shortfall (99%) + let expected_shortfall_1pct = { + let mut sorted_returns = daily_eqs_pct_change.clone(); + sorted_returns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + let cutoff_index = (daily_eqs_pct_change.len() as f64 * 0.01) as usize; + if cutoff_index > 0 { + sorted_returns[..cutoff_index] + .iter() + .map(|x| x.abs()) + .sum::() + / cutoff_index as f64 + } else { + sorted_returns[0].abs() + } + }; // Calculate drawdowns - let drawdowns = calc_drawdowns(&equities); + let drawdowns = calc_drawdowns(&daily_eqs); + let drawdown_worst_mean_1pct = { + let mut sorted_drawdowns = drawdowns.clone(); + sorted_drawdowns.sort_by(|a, b| b.abs().partial_cmp(&a.abs()).unwrap_or(Ordering::Equal)); + let cutoff_index = std::cmp::max(1, (sorted_drawdowns.len() as f64 * 0.01) as usize); + let worst_n = std::cmp::min(cutoff_index, sorted_drawdowns.len()); + sorted_drawdowns[..worst_n] + .iter() + .map(|x| x.abs()) + .sum::() + / worst_n as f64 + }; let drawdown_worst = drawdowns .iter() .fold(f64::NEG_INFINITY, |a, &b| f64::max(a, b.abs())); + let calmar_ratio = if drawdown_worst != 0.0 { + adg / drawdown_worst + } else { + 0.0 + }; + + // Calculate Sterling Ratio (using average of worst N drawdowns) + let sterling_ratio = { + let mut sorted_drawdowns = drawdowns.clone(); + sorted_drawdowns.sort_by(|a, b| b.abs().partial_cmp(&a.abs()).unwrap_or(Ordering::Equal)); + let worst_n = std::cmp::min(10, sorted_drawdowns.len()); + let avg_worst_drawdowns = sorted_drawdowns[..worst_n] + .iter() + .map(|x| x.abs()) + .sum::() + / worst_n as f64; + if avg_worst_drawdowns != 0.0 { + adg / avg_worst_drawdowns // Using raw daily gain instead of annualized + } else { + 0.0 + } + }; + // Calculate equity-balance differences let mut bal_eq = Vec::with_capacity(equities.len()); let mut fill_iter = fills.iter().peekable(); @@ -1542,7 +1630,13 @@ pub fn analyze_backtest(fills: &[Fill], equities: &Vec) -> Analysis { adg, mdg, sharpe_ratio, + sortino_ratio, + omega_ratio, + expected_shortfall_1pct, + calmar_ratio, + sterling_ratio, drawdown_worst, + drawdown_worst_mean_1pct, equity_balance_diff_mean, equity_balance_diff_max, loss_profit_ratio, diff --git a/passivbot-rust/src/python.rs b/passivbot-rust/src/python.rs index a86882d55..feebe0c27 100644 --- a/passivbot-rust/src/python.rs +++ b/passivbot-rust/src/python.rs @@ -87,7 +87,16 @@ pub fn run_backtest( py_analysis.set_item("adg", analysis.adg)?; py_analysis.set_item("mdg", analysis.mdg)?; py_analysis.set_item("sharpe_ratio", analysis.sharpe_ratio)?; + py_analysis.set_item("sortino_ratio", analysis.sortino_ratio)?; + py_analysis.set_item("omega_ratio", analysis.omega_ratio)?; + py_analysis.set_item("expected_shortfall_1pct", analysis.expected_shortfall_1pct)?; + py_analysis.set_item("calmar_ratio", analysis.calmar_ratio)?; + py_analysis.set_item("sterling_ratio", analysis.sterling_ratio)?; py_analysis.set_item("drawdown_worst", analysis.drawdown_worst)?; + py_analysis.set_item( + "drawdown_worst_mean_1pct", + analysis.drawdown_worst_mean_1pct, + )?; py_analysis.set_item( "equity_balance_diff_mean", analysis.equity_balance_diff_mean, diff --git a/passivbot-rust/src/types.rs b/passivbot-rust/src/types.rs index 98714ba4b..1d9e7dec3 100644 --- a/passivbot-rust/src/types.rs +++ b/passivbot-rust/src/types.rs @@ -217,7 +217,13 @@ pub struct Analysis { pub adg: f64, pub mdg: f64, pub sharpe_ratio: f64, + pub sortino_ratio: f64, + pub omega_ratio: f64, + pub expected_shortfall_1pct: f64, + pub calmar_ratio: f64, + pub sterling_ratio: f64, pub drawdown_worst: f64, + pub drawdown_worst_mean_1pct: f64, pub equity_balance_diff_mean: f64, pub equity_balance_diff_max: f64, pub loss_profit_ratio: f64, @@ -229,7 +235,13 @@ impl Default for Analysis { adg: 0.0, mdg: 0.0, sharpe_ratio: 0.0, + sortino_ratio: 0.0, + omega_ratio: 0.0, + expected_shortfall_1pct: 0.0, + calmar_ratio: 0.0, + sterling_ratio: 0.0, drawdown_worst: 1.0, + drawdown_worst_mean_1pct: 1.0, equity_balance_diff_mean: 1.0, equity_balance_diff_max: 1.0, loss_profit_ratio: 1.0, diff --git a/src/exchanges/binance.py b/src/exchanges/binance.py index 8e2bfecae..8077a42fa 100644 --- a/src/exchanges/binance.py +++ b/src/exchanges/binance.py @@ -26,6 +26,9 @@ class BinanceBot(Passivbot): def __init__(self, config: dict): super().__init__(config) + self.custom_id_max_length = 36 + + def create_ccxt_sessions(self): self.broker_code_spot = load_broker_code("binance_spot") for ccx, ccxt_module in [("cca", ccxt_async), ("ccp", ccxt_pro)]: exchange_class = getattr(ccxt_module, "binanceusdm") @@ -47,7 +50,6 @@ def __init__(self, config: dict): if self.broker_code_spot: for key in ["spot", "margin"]: getattr(self, ccx).options["broker"][key] = "x-" + self.broker_code_spot - self.custom_id_max_length = 36 async def print_new_user_suggestion(self): res = None diff --git a/src/exchanges/bitget.py b/src/exchanges/bitget.py index 5bc33049e..56fc1261f 100644 --- a/src/exchanges/bitget.py +++ b/src/exchanges/bitget.py @@ -13,6 +13,7 @@ calc_hash, determine_pos_side_ccxt, shorten_custom_id, + hysteresis_rounding, ) from njit_funcs import calc_diff from procedures import print_async_exception, utc_ms, assert_correct_ccxt_version @@ -23,6 +24,13 @@ class BitgetBot(Passivbot): def __init__(self, config: dict): super().__init__(config) + self.position_side_map = { + "buy": {"open": "long", "close": "short"}, + "sell": {"open": "short", "close": "long"}, + } + self.custom_id_max_length = 64 + + def create_ccxt_sessions(self): self.ccp = getattr(ccxt_pro, self.exchange)( { "apiKey": self.user_info["key"], @@ -39,11 +47,6 @@ def __init__(self, config: dict): } ) self.cca.options["defaultType"] = "swap" - self.position_side_map = { - "buy": {"open": "long", "close": "short"}, - "sell": {"open": "short", "close": "long"}, - } - self.custom_id_max_length = 64 async def determine_utc_offset(self, verbose=True): # returns millis to add to utc to get exchange timestamp @@ -70,24 +73,6 @@ def set_market_specific_settings(self): self.price_steps[symbol] = elm["precision"]["price"] self.c_mults[symbol] = elm["contractSize"] - async def watch_balance(self): - # bitget ccxt watch balance doesn't return required info. - # relying instead on periodic REST updates - while True: - try: - if self.stop_websocket: - break - res = await self.cca.fetch_balance() - res["USDT"]["total"] = float( - [x for x in res["info"] if x["marginCoin"] == self.quote][0]["available"] - ) - self.handle_balance_update(res) - await asyncio.sleep(10) - except Exception as e: - print(f"exception watch_balance", e) - traceback.print_exc() - await asyncio.sleep(1) - async def watch_orders(self): while True: try: @@ -144,9 +129,21 @@ async def fetch_positions(self) -> ([dict], float): self.cca.fetch_positions(), self.cca.fetch_balance(), ) - balance = float( - [x for x in fetched_balance["info"] if x["marginCoin"] == self.quote][0]["available"] - ) + balance_info = [x for x in fetched_balance["info"] if x["marginCoin"] == self.quote][0] + if ( + "assetMode" in balance_info + and "unionTotalMargin" in balance_info + and balance_info["assetMode"] == "union" + ): + balance = float(balance_info["unionTotalMargin"]) + if not hasattr(self, "previous_rounded_balance"): + self.previous_rounded_balance = balance + self.previous_rounded_balance = hysteresis_rounding( + balance, self.previous_rounded_balance, 0.02, 0.5 + ) + balance = self.previous_rounded_balance + else: + balance = float(balance_info["available"]) for i in range(len(fetched_positions)): fetched_positions[i]["position_side"] = fetched_positions[i]["side"] fetched_positions[i]["size"] = fetched_positions[i]["contracts"] diff --git a/src/exchanges/bybit.py b/src/exchanges/bybit.py index 787f718c5..9185b1d5e 100644 --- a/src/exchanges/bybit.py +++ b/src/exchanges/bybit.py @@ -25,6 +25,8 @@ class BybitBot(Passivbot): def __init__(self, config: dict): super().__init__(config) + + def create_ccxt_sessions(self): self.ccp = getattr(ccxt_pro, self.exchange)( { "apiKey": self.user_info["key"], diff --git a/src/exchanges/gateio.py b/src/exchanges/gateio.py index 1428473ad..e9bf2d1bf 100644 --- a/src/exchanges/gateio.py +++ b/src/exchanges/gateio.py @@ -34,6 +34,14 @@ class GateIOBot(Passivbot): def __init__(self, config: dict): super().__init__(config) + self.ohlcvs_1m_init_duration_seconds = ( + 120 # gateio has stricter rate limiting on fetching ohlcvs + ) + self.hedge_mode = False + self.max_n_creations_per_batch = 10 + self.max_n_cancellations_per_batch = 20 + + def create_ccxt_sessions(self): self.ccp = getattr(ccxt_pro, self.exchange)( { "apiKey": self.user_info["key"], @@ -50,12 +58,6 @@ def __init__(self, config: dict): } ) self.cca.options["defaultType"] = "swap" - self.ohlcvs_1m_init_duration_seconds = ( - 120 # gateio has stricter rate limiting on fetching ohlcvs - ) - self.hedge_mode = False - self.max_n_creations_per_batch = 10 - self.max_n_cancellations_per_batch = 20 def set_market_specific_settings(self): super().set_market_specific_settings() diff --git a/src/exchanges/hyperliquid.py b/src/exchanges/hyperliquid.py index 14c7dad08..ebedaeab5 100644 --- a/src/exchanges/hyperliquid.py +++ b/src/exchanges/hyperliquid.py @@ -34,6 +34,17 @@ class HyperliquidBot(Passivbot): def __init__(self, config: dict): super().__init__(config) + self.quote = "USDC" + self.hedge_mode = False + self.significant_digits = {} + if "is_vault" not in self.user_info or self.user_info["is_vault"] == "": + logging.info( + f"parameter 'is_vault' missing from api-keys.json for user {self.user}. Setting to false" + ) + self.user_info["is_vault"] = False + self.max_n_concurrent_ohlcvs_1m_updates = 2 + + def create_ccxt_sessions(self): self.ccp = getattr(ccxt_pro, self.exchange)( { "walletAddress": self.user_info["wallet_address"], @@ -48,15 +59,6 @@ def __init__(self, config: dict): } ) self.cca.options["defaultType"] = "swap" - self.quote = "USDC" - self.hedge_mode = False - self.significant_digits = {} - if "is_vault" not in self.user_info or self.user_info["is_vault"] == "": - logging.info( - f"parameter 'is_vault' missing from api-keys.json for user {self.user}. Setting to false" - ) - self.user_info["is_vault"] = False - self.max_n_concurrent_ohlcvs_1m_updates = 2 def set_market_specific_settings(self): super().set_market_specific_settings() diff --git a/src/exchanges/okx.py b/src/exchanges/okx.py index 31e1c0124..1937e99a2 100644 --- a/src/exchanges/okx.py +++ b/src/exchanges/okx.py @@ -24,6 +24,13 @@ class OKXBot(Passivbot): def __init__(self, config: dict): super().__init__(config) + self.order_side_map = { + "buy": {"long": "open_long", "short": "close_short"}, + "sell": {"long": "close_long", "short": "open_short"}, + } + self.custom_id_max_length = 32 + + def create_ccxt_sessions(self): self.ccp = getattr(ccxt_pro, self.exchange)( { "apiKey": self.user_info["key"], @@ -40,11 +47,6 @@ def __init__(self, config: dict): } ) self.cca.options["defaultType"] = "swap" - self.order_side_map = { - "buy": {"long": "open_long", "short": "close_short"}, - "sell": {"long": "close_long", "short": "open_short"}, - } - self.custom_id_max_length = 32 def set_market_specific_settings(self): super().set_market_specific_settings() diff --git a/src/optimize.py b/src/optimize.py index b7ac9a49a..fbd094737 100644 --- a/src/optimize.py +++ b/src/optimize.py @@ -19,6 +19,7 @@ denumpyize, sort_dict_keys, calc_hash, + flatten, ) from procedures import ( make_get_filepath, @@ -226,11 +227,18 @@ def individual_to_config(individual, template=None): return config -def config_to_individual(config): +def config_to_individual(config, param_bounds): individual = [] for pside in ["long", "short"]: - individual += [v for k, v in sorted(config["bot"][pside].items())] - return individual + is_enabled = ( + param_bounds[f"{pside}_n_positions"][1] > 0.0 + and param_bounds[f"{pside}_total_wallet_exposure_limit"][1] > 0.0 + ) + individual += [(v if is_enabled else 0.0) for k, v in sorted(config["bot"][pside].items())] + # adjust to bounds + bounds = [(low, high) for low, high in param_bounds.values()] + adjusted = [max(min(x, bounds[z][1]), bounds[z][0]) for z, x in enumerate(individual)] + return adjusted @contextmanager @@ -288,7 +296,8 @@ def evaluate(self, individual): def calc_fitness(self, analysis): modifier = 0.0 for i, key in [ - (4, "drawdown_worst"), + (5, "drawdown_worst"), + (4, "drawdown_worst_mean_1pct"), (3, "equity_balance_diff_mean"), (2, "loss_profit_ratio"), ]: @@ -334,38 +343,55 @@ def add_extra_options(parser): ) +def extract_configs(path): + print("debug extract_configs", path) + cfgs = [] + if os.path.exists(path): + if path.endswith("_all_results.txt"): + logging.info(f"Skipping {path}") + return [] + if path.endswith(".json"): + try: + cfgs.append(load_config(path, verbose=False)) + return cfgs + except: + return [] + if path.endswith("_pareto.txt"): + with open(path) as f: + for line in f.readlines(): + try: + cfg = json.loads(line) + cfgs.append(format_config(cfg, verbose=False)) + except Exception as e: + logging.error(f"Failed to load starting config {line} {e}") + return cfgs + + def get_starting_configs(starting_configs: str): if starting_configs is None: return [] - cfgs = [] if os.path.isdir(starting_configs): - filenames = [ - os.path.join(starting_configs, f) - for f in os.listdir(starting_configs) - if f.endswith("json") or f.endswith("hjson") - ] - else: - filenames = [starting_configs] - for path in filenames: - try: - cfgs.append(load_config(path, verbose=False)) - except Exception as e: - logging.error(f"failed to load live config {path} {e}") - return cfgs + return flatten( + [ + get_starting_configs(os.path.join(starting_configs, f)) + for f in os.listdir(starting_configs) + ] + ) + return extract_configs(starting_configs) -def configs_to_individuals(cfgs): +def configs_to_individuals(cfgs, param_bounds): inds = {} for cfg in cfgs: try: fcfg = format_config(cfg, verbose=False) - individual = config_to_individual(fcfg) + individual = config_to_individual(fcfg, param_bounds) inds[calc_hash(individual)] = individual # add duplicate of config, but with lowered total wallet exposure limit fcfg2 = deepcopy(fcfg) for pside in ["long", "short"]: fcfg2["bot"][pside]["total_wallet_exposure_limit"] *= 0.75 - individual2 = config_to_individual(fcfg2) + individual2 = config_to_individual(fcfg2, param_bounds) inds[calc_hash(individual2)] = individual2 except Exception as e: logging.error(f"error loading starting config: {e}") @@ -493,12 +519,14 @@ def create_individual(): # Create initial population logging.info(f"Creating initial population...") - starting_individuals = configs_to_individuals(get_starting_configs(args.starting_configs)) - if len(starting_individuals) > config["optimize"]["population_size"]: - logging.info( - f"increasing population size: {config['optimize']['population_size']} -> {len(starting_individuals)}" - ) - config["optimize"]["population_size"] = len(starting_individuals) + bounds = [(low, high) for low, high in param_bounds.values()] + starting_individuals = configs_to_individuals( + get_starting_configs(args.starting_configs), param_bounds + ) + if (nstart := len(starting_individuals)) > (popsize := config["optimize"]["population_size"]): + logging.info(f"Number of starting configs greater than population size.") + logging.info(f"Increasing population size: {popsize} -> {nstart}") + config["optimize"]["population_size"] = nstart population = toolbox.population(n=config["optimize"]["population_size"]) if starting_individuals: diff --git a/src/passivbot.py b/src/passivbot.py index 9ef4886ca..ec3fac4da 100644 --- a/src/passivbot.py +++ b/src/passivbot.py @@ -162,10 +162,12 @@ def __init__(self, config: dict): "long": "graceful_stop" if self.config["live"]["auto_gs"] else "manual", "short": "graceful_stop" if self.config["live"]["auto_gs"] else "manual", } + self.create_ccxt_sessions() + self.debug_mode = False - async def start_bot(self, debug_mode=False): + async def start_bot(self): logging.info(f"Starting bot...") - await self.hourly_cycle() + await self.init_markets() await asyncio.sleep(1) logging.info(f"Starting data maintainers...") await self.start_data_maintainers() @@ -175,14 +177,14 @@ async def start_bot(self, debug_mode=False): await self.prepare_for_execution() logging.info(f"starting execution loop...") - if not debug_mode: + if not self.debug_mode: await self.run_execution_loop() - async def hourly_cycle(self, verbose=True): + async def init_markets(self, verbose=True): # called at bot startup and once an hour thereafter + self.init_markets_last_update_ms = utc_ms() await self.update_exchange_config() # set hedge mode - self.hourly_cycle_last_update_ms = utc_ms() - self.markets_dict = {elm["symbol"]: elm for elm in (await self.cca.fetch_markets())} + self.markets_dict = await self.cca.load_markets(True) await self.determine_utc_offset(verbose) # ineligible symbols cannot open new positions self.ineligible_symbols = {} @@ -328,7 +330,7 @@ async def prepare_for_execution(self): ) await self.update_ohlcvs_1m_for_actives() - async def execute_to_exchange(self, debug_mode=False): + async def execute_to_exchange(self): await self.execution_cycle() await self.update_EMAs() await self.update_exchange_configs() @@ -351,7 +353,7 @@ async def execute_to_exchange(self, debug_mode=False): # format custom_id to_create = self.format_custom_ids(to_create) - if debug_mode: + if self.debug_mode: if to_cancel: print("would cancel:") for x in to_cancel[: self.config["live"]["max_n_cancellations_per_batch"]]: @@ -363,7 +365,7 @@ async def execute_to_exchange(self, debug_mode=False): if res: for elm in res: self.remove_cancelled_order(elm, source="POST") - if debug_mode: + if self.debug_mode: if to_create: print("would create:") for x in to_create[: self.config["live"]["max_n_creations_per_batch"]]: @@ -439,7 +441,7 @@ def set_live_configs(self): def pad_sym(self, symbol): return f"{symbol: <{self.sym_padding}}" - def stop_data_maintainers(self): + def stop_data_maintainers(self, verbose=True): if not hasattr(self, "maintainers"): return res = {} @@ -457,8 +459,10 @@ def stop_data_maintainers(self): except Exception as e: logging.error(f"error stopping WS_ohlcvs_1m_tasks {key} {e}") if res0s: - logging.info(f"stopped ohlcvs watcher tasks {res0s}") - logging.info(f"stopped data maintainers: {res}") + if verbose: + logging.info(f"stopped ohlcvs watcher tasks {res0s}") + if verbose: + logging.info(f"stopped data maintainers: {res}") return res def has_position(self, pside=None, symbol=None): @@ -1067,20 +1071,6 @@ async def update_pnls(self): self.upd_timestamps["pnls"] = utc_ms() return True - async def check_for_inactive_markets(self): - self.ineligible_symbols_with_pos = [ - elm["symbol"] - for elm in self.fetched_positions + self.fetched_open_orders - if elm["symbol"] not in self.markets_dict - ] - update = False - if self.ineligible_symbols_with_pos: - logging.info( - f"Caught symbol with pos for ineligible market: {self.ineligible_symbols_with_pos}" - ) - update = True - await self.init_markets_dict() - async def update_open_orders(self): if not hasattr(self, "open_orders"): self.open_orders = {} @@ -1090,7 +1080,6 @@ async def update_open_orders(self): if res in [None, False]: return False self.fetched_open_orders = res - await self.check_for_inactive_markets() open_orders = res oo_ids_old = {elm["id"] for sublist in self.open_orders.values() for elm in sublist} created_prints, cancelled_prints = [], [] @@ -1823,24 +1812,11 @@ async def update_ohlcvs_1m_for_actives(self): async def maintain_hourly_cycle(self): logging.info(f"Starting hourly_cycle...") - while not self.stop_signal_received: - try: - # update markets dict once every hour - if utc_ms() - self.hourly_cycle_last_update_ms > 1000 * 60 * 60: - await self.hourly_cycle(verbose=False) - await asyncio.sleep(1) - except Exception as e: - logging.error(f"error with {get_function_name()} {e}") - traceback.print_exc() - await asyncio.sleep(5) - - async def maintain_markets_info(self): - logging.info(f"starting maintain_markets_info") while not self.stop_signal_received: try: # update markets dict once every hour if utc_ms() - self.init_markets_last_update_ms > 1000 * 60 * 60: - await self.init_markets_dict(verbose=False) + await self.init_markets(verbose=False) await asyncio.sleep(1) except Exception as e: logging.error(f"error with {get_function_name()} {e}") @@ -1848,7 +1824,7 @@ async def maintain_markets_info(self): await asyncio.sleep(5) async def start_data_maintainers(self): - # maintains REST init_markets_dict and ohlcv_1m + # maintains REST hourly_cycle and ohlcv_1m if hasattr(self, "maintainers"): self.stop_data_maintainers() self.maintainers = { diff --git a/src/pure_funcs.py b/src/pure_funcs.py index 4db283803..00197cc7c 100644 --- a/src/pure_funcs.py +++ b/src/pure_funcs.py @@ -647,7 +647,8 @@ def get_template_live_config(passivbot_mode="neat_grid"): "crossover_probability": 0.7, "iters": 30000, "limits": { - "lower_bound_drawdown_worst": 0.5, + "lower_bound_drawdown_worst": 0.25, + "lower_bound_drawdown_worst_mean_1pct": 0.1, "lower_bound_equity_balance_diff_mean": 0.03, "lower_bound_loss_profit_ratio": 0.75, },