From 682d3afaf54f8dead947ba64d3d7a6d1ea5e1464 Mon Sep 17 00:00:00 2001 From: ukamoy Date: Tue, 11 Jun 2019 12:21:00 +0800 Subject: [PATCH 1/7] backtest- genetic algorithm --- example/run_ga.py | 36 +++ vnpy/trader/app/ctaStrategy/ctaBacktesting.py | 206 +++++++++++++++++- 2 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 example/run_ga.py diff --git a/example/run_ga.py b/example/run_ga.py new file mode 100644 index 0000000..89f1164 --- /dev/null +++ b/example/run_ga.py @@ -0,0 +1,36 @@ +from vnpy.trader.app.ctaStrategy.ctaBacktesting import OptimizationSetting +from vnpy.trader.app.ctaStrategy import BacktestingEngine +from StrategyBollBand import BollBandsStrategy as Strategy +import json +from datetime import datetime + +if __name__ == "__main__": + engine = BacktestingEngine() + # 设置回测用的数据起始日期 + engine.setStartDate('20190401 23:00:00') + engine.setEndDate('20190430 23:00:00') + # 设置产品相关参数 + contracts = [ + {"symbol":"eos.usd.q:okef", + "size" : 10, + "priceTick" : 0.001, + "rate" : 5/10000, + "slippage" : 0.005 + }] + + engine.setContracts(contracts) # 设置回测合约相关数据 + + # 设置使用的历史数据库 + engine.setDB_URI("mongodb://192.168.0.104:27017") + engine.setDatabase("VnTrader_1Min_Db") + engine.setCapital(100) # 设置起始资金，默认值是1,000,000 + + with open("CTA_setting.json") as parameterDict: + params = json.load(parameterDict) + engine.initStrategy(Strategy, params[0]) + + setting = OptimizationSetting() + setting.setOptimizeTarget("sharpe_ratio") + setting.addParameter('bBandPeriod', 12, 20, 2) # 增加第一个优化参数atrLength，起始12，结束20，步进2 + + engine.run_ga_optimization(setting) \ No newline at end of file diff --git a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py index 6a1ed8d..ffd5ccb 100644 --- a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py +++ b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py @@ -3,6 +3,7 @@ 可以使用和实盘相同的代码进行回测。 ''' from datetime import datetime, timedelta +from time import time from collections import OrderedDict, defaultdict from itertools import product import multiprocessing @@ -15,20 +16,22 @@ import json from vnpy.rpc import RpcClient, RpcServer, RemoteException import logging - +import random +from functools import lru_cache # 如果安装了seaborn则设置为白色风格 try: import seaborn as sns sns.set_style('whitegrid') except ImportError: pass - +from deap import creator, base, tools, algorithms from vnpy.trader.vtObject import VtTickData, VtBarData, VtLogData from vnpy.trader.language import constant from vnpy.trader.vtGateway import VtOrderData, VtTradeData from vnpy.trader.app.ctaStrategy.ctaBase import * - +creator.create("FitnessMax", base.Fitness, weights=(1.0,)) +creator.create("Individual", list, fitness=creator.FitnessMax) ######################################################################## class BacktestingEngine(object): """ @@ -54,6 +57,7 @@ def __init__(self): self.engineType = ENGINETYPE_BACKTESTING # 引擎类型为回测 self.strategy = None # 回测策略 + self.strategy_class = None self.mode = self.BAR_MODE # 回测模式，默认为K线 self.startDate = '' @@ -76,7 +80,8 @@ def __init__(self): self.cachePath = os.path.join(os.path.expanduser("~"), "vnpy_data") # 本地数据缓存地址 self.logActive = False # 回测日志开关 - self.logPath = os.path.join(os.getcwd(), "Backtest_Log") # 回测日志自定义路径 + self.path = os.path.join(os.getcwd(), "Backtest_Log") # 回测日志自定义路径 + self.logPath = "" self.strategy_setting = {} # 缓存策略配置 self.dataStartDate = None # 回测数据开始日期，datetime对象 @@ -160,7 +165,7 @@ def setDB_URI(self, dbURI): """设置历史数据所用的数据库""" self.dbURI = dbURI - def setDatabase(self, bardbName=None, tickdbName=None): + def setDatabase(self, bardbName="", tickdbName=""): self.bardbName = bardbName self.tickdbName = tickdbName @@ -170,14 +175,14 @@ def setCapital(self, capital): self.capital = capital # ---------------------------------------------------------------------- - def setContracts(self, contracts = {}): + def setContracts(self, contracts = []): self.contracts = contracts # ------------------------------------------------- def setLog(self, active=False, path=None): """设置是否出交割单和日志""" if path: - self.logPath = path + self.path = path self.logActive = active # ------------------------------------------------- @@ -403,7 +408,7 @@ def createFolder(self, symbolList): new_name = filter(lambda ch: ch in filter_text, str(symbolList)) symbol_name = ''.join(list(new_name)) Folder_Name = f'{self.strategy.name.replace("Strategy","")}_{symbol_name}_{datetime.now().strftime("%y%m%d%H%M")}' - self.logPath = os.path.join(self.logPath, Folder_Name[:50]) + self.logPath = os.path.join(self.path, Folder_Name[:50]) if not os.path.isdir(self.logPath): os.makedirs(self.logPath) @@ -422,6 +427,7 @@ def initStrategy(self, strategyClass, setting=None): symbolList.append(symbol_info["symbol"]) self.contracts_info.update({symbol_info["symbol"]:symbol_info}) setting['symbolList'] = symbolList + self.strategy_class = strategyClass self.strategy = strategyClass(self, setting) self.strategy.name = self.strategy.className self.initPosition(self.strategy) @@ -1204,6 +1210,133 @@ def runParallelOptimization(self, strategyClass, optimizationSetting, strategySe return resultList + def run_ga_optimization(self, optimization_setting, population_size=100, ngen_size=30, output=True): + """""" + # Get optimization setting and target + settings = optimization_setting.generate_setting_ga() + target_name = optimization_setting.optimizeTarget + + if not settings: + self.output("优化参数组合为空，请检查") + return + + if not target_name: + self.output("优化目标未设置，请检查") + return + + # Define parameter generation function + def generate_parameter(): + """""" + return random.choice(settings) + + def mutate_individual(individual, indpb): + """""" + size = len(individual) + paramlist = generate_parameter() + for i in range(size): + if random.random() < indpb: + individual[i] = paramlist[i] + return individual, + + # Create ga object function + global ga_engine_class + global ga_target_name + global ga_strategy_class + global ga_setting + global ga_start + global ga_init_hours + global ga_contracts + global ga_capital + global ga_end + global ga_mode + global ga_strategy_setting + global ga_dburi + global ga_db_bar + global ga_db_tick + + ga_engine_class = self.__class__ + ga_strategy_class = self.strategy_class + ga_setting = settings[0] + ga_target_name = target_name + ga_mode = self.mode + ga_start = self.startDate + ga_end = self.endDate + ga_capital = self.capital + ga_contracts = self.contracts + ga_init_hours = self.initHours + ga_strategy_setting = self.strategy_setting + ga_dburi = self.dbURI + ga_db_bar = self.bardbName + ga_db_tick = self.tickdbName + + # Set up genetic algorithem + toolbox = base.Toolbox() + toolbox.register("individual", tools.initIterate, creator.Individual, generate_parameter) + toolbox.register("population", tools.initRepeat, list, toolbox.individual) + toolbox.register("mate", tools.cxTwoPoint) + toolbox.register("mutate", mutate_individual, indpb=1) + toolbox.register("evaluate", ga_optimize) + toolbox.register("select", tools.selNSGA2) + + total_size = len(settings) + pop_size = population_size # number of individuals in each generation + lambda_ = pop_size # number of children to produce at each generation + mu = int(pop_size * 0.8) # number of individuals to select for the next generation + + cxpb = 0.95 # probability that an offspring is produced by crossover + mutpb = 1 - cxpb # probability that an offspring is produced by mutation + ngen = ngen_size # number of generation + + pop = toolbox.population(pop_size) + hof = tools.ParetoFront() # end result of pareto front + + stats = tools.Statistics(lambda ind: ind.fitness.values) + np.set_printoptions(suppress=True) + stats.register("mean", np.mean, axis=0) + stats.register("std", np.std, axis=0) + stats.register("min", np.min, axis=0) + stats.register("max", np.max, axis=0) + + # Multiprocessing is not supported yet. + # pool = multiprocessing.Pool(multiprocessing.cpu_count()) + # toolbox.register("map", pool.map) + + # Run ga optimization + self.output(f"参数优化空间：{total_size}") + self.output(f"每代族群总数：{pop_size}") + self.output(f"优良筛选个数：{mu}") + self.output(f"迭代次数：{ngen}") + self.output(f"交叉概率：{cxpb:.0%}") + self.output(f"突变概率：{mutpb:.0%}") + + start = time() + + algorithms.eaMuPlusLambda( + pop, + toolbox, + mu, + lambda_, + cxpb, + mutpb, + ngen, + stats, + halloffame=hof + ) + + end = time() + cost = int((end - start)) + + self.output(f"遗传算法优化完成，耗时{cost}秒") + + # Return result list + results = [] + + for parameter_values in hof: + setting = dict(parameter_values) + target_value = ga_optimize(parameter_values)[0] + results.append((setting, target_value, {})) + + return results # ---------------------------------------------------------------------- def updateDailyClose(self, symbol, dt, price): """更新每日收盘价""" @@ -1583,6 +1716,14 @@ def setOptimizeTarget(self, target): """设置优化目标字段""" self.optimizeTarget = target + def generate_setting_ga(self): + """""" + settings_ga = [] + settings = self.generateSetting() + for d in settings: + param = [tuple(i) for i in d.items()] + settings_ga.append(param) + return settings_ga ######################################################################## class HistoryDataServer(RpcServer): @@ -1646,8 +1787,8 @@ def formatNumber(n): # ---------------------------------------------------------------------- def optimize(backtestEngineClass, strategyClass, setting, targetName, mode, startDate, initHours, endDate, - db_URI, bardbName, tickdbName, - contracts = {}, prepared_data = []): + db_URI="", bardbName="", tickdbName="", + contracts = {}): """多进程优化时跑在每个进程中运行的函数""" engine = backtestEngineClass() engine.setBacktestingMode(mode) @@ -1658,7 +1799,7 @@ def optimize(backtestEngineClass, strategyClass, setting, targetName, engine.setDatabase(bardbName, tickdbName) engine.initStrategy(strategyClass, setting) - engine.runBacktesting(prepared_data) + engine.runBacktesting() df = engine.calculateDailyResult() df, d = engine.calculateDailyStatistics(df) @@ -1669,6 +1810,32 @@ def optimize(backtestEngineClass, strategyClass, setting, targetName, # return (str(setting), targetValue, d) return (setting, targetValue, d) +@lru_cache(maxsize=1000000) +def _ga_optimize(parameter_values): + """""" + setting = dict(parameter_values) + setting.update(ga_strategy_setting) + + result = optimize( + ga_engine_class, + ga_strategy_class, + setting, + ga_target_name, + ga_mode, + ga_start, + ga_init_hours, + ga_end, + ga_dburi, + ga_db_bar, + ga_db_tick, + ga_contracts + ) + return (result[1],) + + +def ga_optimize(parameter_values): + """""" + return _ga_optimize(tuple(parameter_values)) def gen_dates(b_date, days): day = timedelta(days=1) @@ -1717,6 +1884,23 @@ def get_minutes_list(start=None, end=None): data.append(d) return data +# GA related global value +ga_engine_class = None +ga_end = None +ga_mode = None +ga_target_name = None +ga_strategy_class = None +ga_setting = None +ga_start = None +ga_contracts = None +ga_capital = None +ga_engine_class = None +ga_init_hours = None +ga_strategy_setting = None +ga_dburi = None +ga_db_bar = None +ga_db_tick = None + class PatchedBacktestingEngine(BacktestingEngine): """ From e367db4c46015d216870c64ac10e02e18a4d37b0 Mon Sep 17 00:00:00 2001 From: tianrq10 Date: Wed, 19 Jun 2019 23:24:49 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E5=8F=8D=E5=90=91=E5=90=88=E7=BA=A6?= =?UTF-8?q?=E5=9B=9E=E6=B5=8B=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 + vnpy/trader/app/ctaStrategy/ctaBacktesting.py | 260 ++++++++++-------- 2 files changed, 153 insertions(+), 108 deletions(-) diff --git a/requirements.txt b/requirements.txt index 73e35c3..0454dcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ websocket-client==0.48.0 qdarkstyle SortedContainers pandas>=0.23 +deap flask-restful flask-socketio aiohttp diff --git a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py index ffd5ccb..53b7eb8 100644 --- a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py +++ b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py @@ -18,9 +18,11 @@ import logging import random from functools import lru_cache + # 如果安装了seaborn则设置为白色风格 try: import seaborn as sns + sns.set_style('whitegrid') except ImportError: pass @@ -30,8 +32,11 @@ from vnpy.trader.vtGateway import VtOrderData, VtTradeData from vnpy.trader.app.ctaStrategy.ctaBase import * + creator.create("FitnessMax", base.Fitness, weights=(1.0,)) creator.create("Individual", list, fitness=creator.FitnessMax) + + ######################################################################## class BacktestingEngine(object): """ @@ -66,18 +71,19 @@ def __init__(self): self.capital = 1000000 # 回测时的起始本金（默认100万） - self.dbClient = None # 数据库客户端 - self.dbURI = '' # 回测数据库地址 - self.bardbName = '' # bar数据库名 - self.tickdbName = '' # tick数据库名 - self.dbCursor = None # 数据库指针 - self.hdsClient = None # 历史数据服务器客户端 + self.dbClient = None # 数据库客户端 + self.dbURI = '' # 回测数据库地址 + self.bardbName = '' # bar数据库名 + self.tickdbName = '' # tick数据库名 + self.dbCursor = None # 数据库指针 + self.hdsClient = None # 历史数据服务器客户端 - self.initData = [] # 初始化用的数据 - self.contracts = [] # 回测集合名 - self.contracts_info = {}# portfolio + self.initData = [] # 初始化用的数据 + self.contracts = [] # 回测集合名 + self.contracts_info = {} # portfolio self.backtestData = [] # 回测用历史数据 + self.backtestResultType = "Linear" self.cachePath = os.path.join(os.path.expanduser("~"), "vnpy_data") # 本地数据缓存地址 self.logActive = False # 回测日志开关 self.path = os.path.join(os.getcwd(), "Backtest_Log") # 回测日志自定义路径 @@ -135,6 +141,13 @@ def output(self, content, carriageReturn=False): # ------------------------------------------------ # ---------------------------------------------------------------------- + def setBacktestResultType(self, _type): + self.backtestResultType = _type + if self.backtestResultType == "Linear" or self.backtestResultType == "Inverse": + pass + else: + raise ValueError("回测绩效类型只能为Linear/Inverse") + def setStartDate(self, startDate='20100416 01:00:00', initHours=0): """设置回测的启动日期""" self.startDate = startDate @@ -175,7 +188,7 @@ def setCapital(self, capital): self.capital = capital # ---------------------------------------------------------------------- - def setContracts(self, contracts = []): + def setContracts(self, contracts=[]): self.contracts = contracts # ------------------------------------------------- @@ -184,7 +197,7 @@ def setLog(self, active=False, path=None): if path: self.path = path self.logActive = active - + # ------------------------------------------------- def setCachePath(self, path): self.cachePath = path @@ -202,6 +215,7 @@ def parseData(self, dataClass, dataDict): 'gatewayName': '', 'high': 2374.4, 'low': 2374.1, 'open': 2374.1, 'openInterest': 0, 'rawData': None, 'symbol': 'tBTCUSD', 'time': '10:44:00.000000', 'volume': 12.18062789, 'vtSymbol': 'tBTCUSD:bitfinex'} """ + # ---------------------------------------------------------------------- def initHdsClient(self): """初始化历史数据服务器客户端""" @@ -217,7 +231,7 @@ def loadHistoryData(self, symbolList, startDate, endDate=None, dataMode=None): if not endDate: endDate = datetime.strptime(self.END_OF_THE_WORLD, constant.DATETIME) - modeMap = {self.BAR_MODE:"datetime",self.TICK_MODE:"date"} + modeMap = {self.BAR_MODE: "datetime", self.TICK_MODE: "date"} # 根据回测模式，确认要使用的数据类 if dataMode is None: @@ -242,7 +256,7 @@ def loadHistoryData(self, symbolList, startDate, endDate=None, dataMode=None): df_cached = {} # 优先从本地文件缓存读取数据 symbols_no_data = dict() # 本地缓存没有的数据 - + for symbol in symbolList: # 如果存在缓存文件，则读取日期列表和bar数据，否则初始化df_cached和dates_cached save_path = os.path.join(self.cachePath, dataMode, symbol.replace(":", "_")) @@ -271,7 +285,7 @@ def loadHistoryData(self, symbolList, startDate, endDate=None, dataMode=None): dbName = self.bardbName else: dbName = self.tickdbName - if self.dbURI and dbName is not None: # 有设置从指定数据库和表取数据 + if self.dbURI and dbName is not None: # 有设置从指定数据库和表取数据 import pymongo self.dbClient = pymongo.MongoClient(self.dbURI)[dbName] for symbol, need_datetimes in symbols_no_data.items(): @@ -284,8 +298,8 @@ def loadHistoryData(self, symbolList, startDate, endDate=None, dataMode=None): del data_df["_id"] # 筛选出需要的时间段 dataList += [self.parseData(dataClass, item) for item in - data_df[(data_df.datetime >= start) & (data_df.datetime < end)].to_dict( - "record")] + data_df[(data_df.datetime >= start) & (data_df.datetime < end)].to_dict( + "record")] # 缓存到本地文件 save_path = os.path.join(self.cachePath, dataMode, symbol.replace(":", "_")) if not os.path.isdir(save_path): @@ -297,7 +311,8 @@ def loadHistoryData(self, symbolList, startDate, endDate=None, dataMode=None): for date in symbols_no_data[symbol]: update_df = data_df[data_df["date"] == date] if update_df.size > 0: - update_df.to_hdf(f"{save_path}/{date}.hd5", "/", format = "table", append=True, complevel=9) + update_df.to_hdf(f"{save_path}/{date}.hd5", "/", format="table", append=True, + complevel=9) acq, need = len(list(set(data_df[modeMap[dataMode]]))), len(need_datetimes) self.output(f"{symbol}： 从数据库存取了{acq}, 应补{need}, 缺失了{need-acq}") @@ -375,7 +390,7 @@ def runBacktesting(self): filename = os.path.join(self.logPath, u"Backtest.log") f = open(filename, "w+") for line in self.logList: - print(f"{line}", file = f) + print(f"{line}", file=f) self.output(u'Backtest log Recorded') # ---------------------------------------------------------------------- @@ -384,7 +399,7 @@ def newBar(self, bar): self.barDict[bar.vtSymbol] = bar self.dt = bar.datetime - self.crossLimitOrder(bar) # 先撮合限价单 + self.crossLimitOrder(bar) # 先撮合限价单 self.crossStopOrder(bar) # 再撮合停止单 self.strategy.onBar(bar) # 推送K线到策略中 @@ -403,7 +418,7 @@ def newTick(self, tick): # ---------------------------------------------------------------------- def createFolder(self, symbolList): - alpha='abcdefghijklmnopqrstuvwxyz' + alpha = 'abcdefghijklmnopqrstuvwxyz' filter_text = "0123456789._-" + alpha + alpha.upper() new_name = filter(lambda ch: ch in filter_text, str(symbolList)) symbol_name = ''.join(list(new_name)) @@ -420,12 +435,12 @@ def initStrategy(self, strategyClass, setting=None): """ if not self.contracts: for symbol in setting['symbolList']: - self.contracts_info.update({symbol:{}}) + self.contracts_info.update({symbol: {}}) else: symbolList = [] for symbol_info in self.contracts: symbolList.append(symbol_info["symbol"]) - self.contracts_info.update({symbol_info["symbol"]:symbol_info}) + self.contracts_info.update({symbol_info["symbol"]: symbol_info}) setting['symbolList'] = symbolList self.strategy_class = strategyClass self.strategy = strategyClass(self, setting) @@ -451,7 +466,7 @@ def crossLimitOrder(self, data): sellCrossPrice = data.bidPrice1 buyBestCrossPrice = data.askPrice1 sellBestCrossPrice = data.bidPrice1 - + symbol = data.vtSymbol # 遍历限价单字典中的所有限价单 @@ -496,7 +511,8 @@ def crossLimitOrder(self, data): self.strategy.posDict[symbol + "_LONG"] += order.totalVolume self.strategy.eveningDict[symbol + "_LONG"] += order.totalVolume self.strategy.posDict[symbol + "_LONG"] = round(self.strategy.posDict[symbol + "_LONG"], 4) - self.strategy.eveningDict[symbol + "_LONG"] = round(self.strategy.eveningDict[symbol + "_LONG"], 4) + self.strategy.eveningDict[symbol + "_LONG"] = round(self.strategy.eveningDict[symbol + "_LONG"], + 4) elif buyCross and trade.offset == constant.OFFSET_CLOSE: trade.price = min(order.price, buyBestCrossPrice) self.strategy.posDict[symbol + "_SHORT"] -= order.totalVolume @@ -506,7 +522,8 @@ def crossLimitOrder(self, data): self.strategy.posDict[symbol + "_SHORT"] += order.totalVolume self.strategy.eveningDict[symbol + "_SHORT"] += order.totalVolume self.strategy.posDict[symbol + "_SHORT"] = round(self.strategy.posDict[symbol + "_SHORT"], 4) - self.strategy.eveningDict[symbol + "_SHORT"] = round(self.strategy.eveningDict[symbol + "_SHORT"], 4) + self.strategy.eveningDict[symbol + "_SHORT"] = round( + self.strategy.eveningDict[symbol + "_SHORT"], 4) elif sellCross and trade.offset == constant.OFFSET_CLOSE: trade.price = max(order.price, sellBestCrossPrice) self.strategy.posDict[symbol + "_LONG"] -= order.totalVolume @@ -646,7 +663,7 @@ def sendOrder(self, vtSymbol, orderType, price, volume, priceType, strategy): order.orderTime = self.dt.strftime(constant.DATETIME) order.orderDatetime = self.dt order.priceType = priceType - + # CTA委托类型映射 if orderType == CTAORDER_BUY: order.direction = constant.DIRECTION_LONG @@ -656,7 +673,7 @@ def sendOrder(self, vtSymbol, orderType, price, volume, priceType, strategy): order.offset = constant.OFFSET_CLOSE closable = self.strategy.eveningDict[order.vtSymbol + '_LONG'] if order.totalVolume > closable: - self.output(f"当前order：{order.orderTime}, 卖平{order.totalVolume}, 可平{closable}, 实盘下可能拒单, 请小心处理") + self.output(f"当前order：{order.orderTime}, 卖平{order.totalVolume}, 可平{closable}, 实盘下可能拒单, 请小心处理") closable -= order.totalVolume self.strategy.eveningDict[order.vtSymbol + '_LONG'] = round(closable, 4) elif orderType == CTAORDER_SHORT: @@ -720,6 +737,7 @@ def sendStopOrder(self, vtSymbol, orderType, price, volume, priceType, strategy) self.strategy.onStopOrder(so) return [stopOrderID] + # ---------------------------------------------------------------------- def cancelOrder(self, vtOrderID): """撤单""" @@ -733,11 +751,13 @@ def cancelOrder(self, vtOrderID): if order.offset == constant.OFFSET_CLOSE: if order.direction == constant.DIRECTION_LONG: self.strategy.eveningDict[order.vtSymbol + '_SHORT'] += order.totalVolume - self.strategy.eveningDict[order.vtSymbol + '_SHORT'] = round(self.strategy.posDict[order.vtSymbol + '_SHORT'], 4) + self.strategy.eveningDict[order.vtSymbol + '_SHORT'] = round( + self.strategy.posDict[order.vtSymbol + '_SHORT'], 4) elif order.direction == constant.DIRECTION_SHORT: self.strategy.eveningDict[order.vtSymbol + '_LONG'] += order.totalVolume - self.strategy.eveningDict[order.vtSymbol + '_LONG'] = round(self.strategy.posDict[order.vtSymbol + '_LONG'], 4) - + self.strategy.eveningDict[order.vtSymbol + '_LONG'] = round( + self.strategy.posDict[order.vtSymbol + '_LONG'], 4) + self.strategy.onOrder(order) del self.workingLimitOrderDict[vtOrderID] @@ -751,6 +771,7 @@ def cancelStopOrder(self, stopOrderID): so.status = STOPORDER_CANCELLED del self.workingStopOrderDict[stopOrderID] self.strategy.onStopOrder(so) + # ---------------------------------------------------------------------- def putStrategyEvent(self, name): """发送策略更新事件，回测中忽略""" @@ -782,7 +803,7 @@ def writeLog(self, content, level=logging.INFO): msg = "%s %s" % (logging.getLevelName(level), content) log = str(self.dt) + ' ' + msg self.logList.append(log) - + # ---------------------------------------------------------------------- def cancelAll(self, name): """全部撤单""" @@ -849,21 +870,21 @@ def calculateBacktestingResult(self): for trade in tradeDict.values(): if trade.direction == constant.DIRECTION_LONG: - if trade.offset in [constant.OFFSET_OPEN, constant.OFFSET_NONE]: + if trade.offset in [constant.OFFSET_OPEN, constant.OFFSET_NONE]: longTrade[trade.vtSymbol].append(trade) elif trade.offset == constant.OFFSET_CLOSE: while True: entryTrade = shortTrade[trade.vtSymbol][0] exitTrade = trade - + # 清算开平仓交易 closedVolume = min(exitTrade.volume, entryTrade.volume) result = TradingResult(entryTrade.price, entryTrade.tradeDatetime, entryTrade.orderID, - exitTrade.price, exitTrade.tradeDatetime,exitTrade.orderID, - -closedVolume, self.contracts_info[trade.vtSymbol]) + exitTrade.price, exitTrade.tradeDatetime, exitTrade.orderID, + -closedVolume, self.contracts_info[trade.vtSymbol], self.backtestResultType) resultList.append(result) r = result.__dict__ - r.update({"symbol":trade.vtSymbol}) + r.update({"symbol": trade.vtSymbol}) deliverSheet.append(r) posList.extend([-1, 0]) @@ -876,7 +897,6 @@ def calculateBacktestingResult(self): entryTrade.volume = round(entryTrade.volume, 4) exitTrade.volume = round(exitTrade.volume, 4) - # 如果开仓交易已经全部清算，则从列表中移除 if not entryTrade.volume: shortTrade[trade.vtSymbol].pop(0) @@ -897,7 +917,7 @@ def calculateBacktestingResult(self): pass elif trade.direction == constant.DIRECTION_SHORT: - if trade.offset == constant.OFFSET_OPEN: + if trade.offset == constant.OFFSET_OPEN: shortTrade[trade.vtSymbol].append(trade) elif trade.offset in [constant.OFFSET_CLOSE, constant.OFFSET_NONE]: while True: @@ -908,10 +928,10 @@ def calculateBacktestingResult(self): closedVolume = min(exitTrade.volume, entryTrade.volume) result = TradingResult(entryTrade.price, entryTrade.tradeDatetime, entryTrade.orderID, exitTrade.price, exitTrade.tradeDatetime, exitTrade.orderID, - closedVolume, self.contracts_info[trade.vtSymbol]) + closedVolume, self.contracts_info[trade.vtSymbol], self.backtestResultType) resultList.append(result) r = result.__dict__ - r.update({"symbol":trade.vtSymbol}) + r.update({"symbol": trade.vtSymbol}) deliverSheet.append(r) posList.extend([1, 0]) @@ -952,11 +972,11 @@ def calculateBacktestingResult(self): for trade in tradeList: result = TradingResult(trade.price, trade.tradeDatetime, trade.orderID, endPrice, self.dt, "LastDay", - trade.volume, self.contracts_info[symbol]) + trade.volume, self.contracts_info[symbol], self.backtestResultType) resultList.append(result) r = result.__dict__ - r.update({"symbol":symbol}) + r.update({"symbol": symbol}) deliverSheet.append(r) for symbol, tradeList in shortTrade.items(): @@ -968,10 +988,10 @@ def calculateBacktestingResult(self): for trade in tradeList: result = TradingResult(trade.price, trade.tradeDatetime, trade.orderID, endPrice, self.dt, "LastDay", - -trade.volume, self.contracts_info[symbol]) + -trade.volume, self.contracts_info[symbol], self.backtestResultType) resultList.append(result) r = result.__dict__ - r.update({"symbol":symbol}) + r.update({"symbol": symbol}) deliverSheet.append(r) # 检查是否有交易 @@ -1175,7 +1195,7 @@ def runOptimization(self, strategyClass, optimizationSetting): return resultList # ---------------------------------------------------------------------- - def runParallelOptimization(self, strategyClass, optimizationSetting, strategySetting = {}, prepared_data = []): + def runParallelOptimization(self, strategyClass, optimizationSetting, strategySetting={}, prepared_data=[]): """并行优化参数""" # 获取优化设置 settingList = optimizationSetting.generateSetting() @@ -1186,7 +1206,7 @@ def runParallelOptimization(self, strategyClass, optimizationSetting, strategySe self.output(u'优化设置有问题，请检查') # 多进程优化，启动一个对应CPU核心数量的进程池 - pool = multiprocessing.Pool(multiprocessing.cpu_count()-1) + pool = multiprocessing.Pool(multiprocessing.cpu_count() - 1) l = [] for setting in settingList: @@ -1195,7 +1215,7 @@ def runParallelOptimization(self, strategyClass, optimizationSetting, strategySe l.append(pool.apply_async(optimize, (self.__class__, strategyClass, setting, targetName, self.mode, self.startDate, self.initHours, self.endDate, - self.dbURI, self.bardbName, self.tickdbName, + self.dbURI, self.bardbName, self.tickdbName, self.contracts_info, prepared_data))) pool.close() pool.join() @@ -1228,7 +1248,7 @@ def run_ga_optimization(self, optimization_setting, population_size=100, ngen_si def generate_parameter(): """""" return random.choice(settings) - + def mutate_individual(individual, indpb): """""" size = len(individual) @@ -1268,27 +1288,27 @@ def mutate_individual(individual, indpb): ga_dburi = self.dbURI ga_db_bar = self.bardbName ga_db_tick = self.tickdbName - + # Set up genetic algorithem - toolbox = base.Toolbox() - toolbox.register("individual", tools.initIterate, creator.Individual, generate_parameter) - toolbox.register("population", tools.initRepeat, list, toolbox.individual) - toolbox.register("mate", tools.cxTwoPoint) - toolbox.register("mutate", mutate_individual, indpb=1) - toolbox.register("evaluate", ga_optimize) - toolbox.register("select", tools.selNSGA2) + toolbox = base.Toolbox() + toolbox.register("individual", tools.initIterate, creator.Individual, generate_parameter) + toolbox.register("population", tools.initRepeat, list, toolbox.individual) + toolbox.register("mate", tools.cxTwoPoint) + toolbox.register("mutate", mutate_individual, indpb=1) + toolbox.register("evaluate", ga_optimize) + toolbox.register("select", tools.selNSGA2) total_size = len(settings) - pop_size = population_size # number of individuals in each generation - lambda_ = pop_size # number of children to produce at each generation - mu = int(pop_size * 0.8) # number of individuals to select for the next generation + pop_size = population_size # number of individuals in each generation + lambda_ = pop_size # number of children to produce at each generation + mu = int(pop_size * 0.8) # number of individuals to select for the next generation - cxpb = 0.95 # probability that an offspring is produced by crossover - mutpb = 1 - cxpb # probability that an offspring is produced by mutation - ngen = ngen_size # number of generation - - pop = toolbox.population(pop_size) - hof = tools.ParetoFront() # end result of pareto front + cxpb = 0.95 # probability that an offspring is produced by crossover + mutpb = 1 - cxpb # probability that an offspring is produced by mutation + ngen = ngen_size # number of generation + + pop = toolbox.population(pop_size) + hof = tools.ParetoFront() # end result of pareto front stats = tools.Statistics(lambda ind: ind.fitness.values) np.set_printoptions(suppress=True) @@ -1312,22 +1332,22 @@ def mutate_individual(individual, indpb): start = time() algorithms.eaMuPlusLambda( - pop, - toolbox, - mu, - lambda_, - cxpb, - mutpb, - ngen, + pop, + toolbox, + mu, + lambda_, + cxpb, + mutpb, + ngen, stats, halloffame=hof - ) - + ) + end = time() cost = int((end - start)) self.output(f"遗传算法优化完成，耗时{cost}秒") - + # Return result list results = [] @@ -1335,8 +1355,9 @@ def mutate_individual(individual, indpb): setting = dict(parameter_values) target_value = ga_optimize(parameter_values)[0] results.append((setting, target_value, {})) - + return results + # ---------------------------------------------------------------------- def updateDailyClose(self, symbol, dt, price): """更新每日收盘价""" @@ -1376,7 +1397,7 @@ def calculateDailyResult(self): dailyResult.previousClose = previousClose previousClose = dailyResult.closePrice - dailyResult.calculatePnl(openPosition, self.contracts_info[symbol]) + dailyResult.calculatePnl(openPosition, self.contracts_info[symbol], self.backtestResultType) openPosition = dailyResult.closePosition # 生成DataFrame @@ -1541,12 +1562,12 @@ def showDailyResult(self): filename = os.path.join(self.logPath, u"回测绩效图.png") plt.savefig(filename) self.output(u'策略回测绩效图已保存') - + self.strategy_setting.update(result) filename = os.path.join(self.logPath, "BacktestingResult.json") - with open(filename,'w') as f: + with open(filename, 'w') as f: json.dump(self.strategy_setting, f, indent=4) - self.output(u'BacktestingResult saved') + self.output(u'BacktestingResult saved') plt.show() @@ -1556,8 +1577,8 @@ class TradingResult(object): """每笔交易的结果""" # ---------------------------------------------------------------------- - def __init__(self, entryPrice, entryDt, entryID, exitPrice, - exitDt, exitID, volume, contracts={}): + def __init__(self, entryPrice, entryDt, entryID, exitPrice, + exitDt, exitID, volume, contracts={}, backtestResultType="Linear"): """Constructor""" self.entryPrice = entryPrice # 开仓价格 self.exitPrice = exitPrice # 平仓价格 @@ -1576,11 +1597,16 @@ def __init__(self, entryPrice, entryDt, entryID, exitPrice, self.turnover = (self.entryPrice + self.exitPrice) * size * abs(volume) # 成交金额 - self.commission = self.turnover * rate # 手续费成本 - self.slippage = slippage * 2 * size * abs(volume) # 滑点成本 + if backtestResultType == "Linear": + self.commission = self.turnover * rate # 手续费成本 + self.slippage = slippage * 2 * size * abs(volume) # 滑点成本 + + self.pnl = (self.exitPrice - self.entryPrice) * volume * size - self.commission - self.slippage # 净盈亏 + if backtestResultType == "Inverse": + self.commission = rate * self.turnover/ self.exitPrice # 手续费成本 + self.slippage = slippage/self.entryPrice * size * abs(volume) + slippage/self.exitPrice * size * abs(volume) # 滑点成本 - self.pnl = ((self.exitPrice - self.entryPrice) * volume * size - - self.commission - self.slippage) # 净盈亏 + self.pnl = (self.exitPrice - self.entryPrice) * volume * size / self.exitPrice - self.commission - self.slippage# 净盈亏 ######################################################################## @@ -1616,7 +1642,7 @@ def addTrade(self, trade): self.tradeList.append(trade) # ---------------------------------------------------------------------- - def calculatePnl(self, openPosition=0, contracts = {}): + def calculatePnl(self, openPosition=0, contracts={}, backtestResultType="Linear"): """ 计算盈亏 size: 合约乘数 @@ -1629,7 +1655,10 @@ def calculatePnl(self, openPosition=0, contracts = {}): # 持仓部分 self.openPosition = openPosition - self.positionPnl = self.openPosition * (self.closePrice - self.previousClose) * size + if backtestResultType == "Linear": + self.positionPnl = self.openPosition * (self.closePrice - self.previousClose) * size + if backtestResultType == "Inverse": + self.positionPnl = self.openPosition * (self.closePrice - self.previousClose) * size / self.closePrice self.closePosition = self.openPosition # 交易部分 @@ -1640,14 +1669,19 @@ def calculatePnl(self, openPosition=0, contracts = {}): posChange = trade.volume else: posChange = -trade.volume - - self.tradingPnl += posChange * (self.closePrice - trade.price) * size + if backtestResultType == "Linear": + self.tradingPnl += posChange * (self.closePrice - trade.price) * size + if backtestResultType == "Inverse": + self.tradingPnl += posChange * (self.closePrice - trade.price) * size / self.closePrice self.turnover += trade.price * trade.volume * size self.closePosition += posChange - self.commission += trade.price * trade.volume * size * rate - self.slippage += trade.volume * size * slippage - + if backtestResultType == "Linear": + self.commission += trade.price * trade.volume * size * rate + self.slippage += trade.volume * size * slippage + if backtestResultType == "Inverse": + self.commission += trade.volume * size * rate # 这块只算了近似的手续费（平仓手续费应该为volume * 开仓价格/平仓价格 * rate， 这里只在二者变化不大时才成立） + self.slippage += trade.volume * size * slippage / trade.price # 汇总 self.totalPnl = self.tradingPnl + self.positionPnl self.netPnl = self.totalPnl - self.commission - self.slippage @@ -1717,14 +1751,15 @@ def setOptimizeTarget(self, target): self.optimizeTarget = target def generate_setting_ga(self): - """""" + """""" settings_ga = [] - settings = self.generateSetting() - for d in settings: + settings = self.generateSetting() + for d in settings: param = [tuple(i) for i in d.items()] settings_ga.append(param) return settings_ga + ######################################################################## class HistoryDataServer(RpcServer): """历史数据缓存服务器""" @@ -1777,6 +1812,7 @@ def runHistoryDataServer(): print(u'按任意键退出') hds.stop() + # ---------------------------------------------------------------------- def formatNumber(n): """格式化数字到字符串""" @@ -1788,7 +1824,7 @@ def formatNumber(n): def optimize(backtestEngineClass, strategyClass, setting, targetName, mode, startDate, initHours, endDate, db_URI="", bardbName="", tickdbName="", - contracts = {}): + contracts={}): """多进程优化时跑在每个进程中运行的函数""" engine = backtestEngineClass() engine.setBacktestingMode(mode) @@ -1810,6 +1846,7 @@ def optimize(backtestEngineClass, strategyClass, setting, targetName, # return (str(setting), targetValue, d) return (setting, targetValue, d) + @lru_cache(maxsize=1000000) def _ga_optimize(parameter_values): """""" @@ -1837,6 +1874,7 @@ def ga_optimize(parameter_values): """""" return _ga_optimize(tuple(parameter_values)) + def gen_dates(b_date, days): day = timedelta(days=1) for i in range(days): @@ -1861,7 +1899,7 @@ def get_date_list(start=None, end=None): def gen_minutes(b_date, days, minutes): - minute = timedelta(minutes = 1) + minute = timedelta(minutes=1) for i in range(days * 1440 + minutes): yield b_date + minute * i @@ -1884,6 +1922,7 @@ def get_minutes_list(start=None, end=None): data.append(d) return data + # GA related global value ga_engine_class = None ga_end = None @@ -1943,13 +1982,16 @@ def processCancelledOrders(self): if order.offset == constant.OFFSET_CLOSE: if order.direction == constant.DIRECTION_LONG: self.strategy.eveningDict[order.vtSymbol + '_SHORT'] += order.totalVolume - self.strategy.eveningDict[order.vtSymbol + "_SHORT"] = round(self.strategy.eveningDict[order.vtSymbol + "_SHORT"], 4) + self.strategy.eveningDict[order.vtSymbol + "_SHORT"] = round( + self.strategy.eveningDict[order.vtSymbol + "_SHORT"], 4) elif order.direction == constant.DIRECTION_SHORT: self.strategy.eveningDict[order.vtSymbol + '_LONG'] += order.totalVolume - self.strategy.eveningDict[order.vtSymbol + "_LONG"] = round(self.strategy.eveningDict[order.vtSymbol + "_LONG"], 4) + self.strategy.eveningDict[order.vtSymbol + "_LONG"] = round( + self.strategy.eveningDict[order.vtSymbol + "_LONG"], 4) del self.workingLimitOrderDict[vtOrderID] self.strategy.onOrder(order) del self._cancelledLimitOrderDict[vtOrderID] + def crossLimitOrder(self, data): # 先确定会撮合成交的价格 if self.mode == self.BAR_MODE: @@ -1962,13 +2004,13 @@ def crossLimitOrder(self, data): sellCrossPrice = data.bidPrice1 buyBestCrossPrice = data.askPrice1 sellBestCrossPrice = data.bidPrice1 - + symbol = data.vtSymbol # 遍历限价单字典中的所有限价单 for orderID in list(self.workingLimitOrderDict): order = self.workingLimitOrderDict.get(orderID, None) - if not order: # 已被撤销 + if not order: # 已被撤销 continue if order.vtSymbol == symbol: # 推送委托进入队列（未成交）的状态更新 @@ -2008,7 +2050,8 @@ def crossLimitOrder(self, data): self.strategy.posDict[symbol + "_LONG"] += order.totalVolume self.strategy.eveningDict[symbol + "_LONG"] += order.totalVolume self.strategy.posDict[symbol + "_LONG"] = round(self.strategy.posDict[symbol + "_LONG"], 4) - self.strategy.eveningDict[symbol + "_LONG"] = round(self.strategy.eveningDict[symbol + "_LONG"], 4) + self.strategy.eveningDict[symbol + "_LONG"] = round(self.strategy.eveningDict[symbol + "_LONG"], + 4) elif buyCross and trade.offset == constant.OFFSET_CLOSE: trade.price = min(order.price, buyBestCrossPrice) self.strategy.posDict[symbol + "_SHORT"] -= order.totalVolume @@ -2018,7 +2061,8 @@ def crossLimitOrder(self, data): self.strategy.posDict[symbol + "_SHORT"] += order.totalVolume self.strategy.eveningDict[symbol + "_SHORT"] += order.totalVolume self.strategy.posDict[symbol + "_SHORT"] = round(self.strategy.posDict[symbol + "_SHORT"], 4) - self.strategy.eveningDict[symbol + "_SHORT"] = round(self.strategy.eveningDict[symbol + "_SHORT"], 4) + self.strategy.eveningDict[symbol + "_SHORT"] = round( + self.strategy.eveningDict[symbol + "_SHORT"], 4) elif sellCross and trade.offset == constant.OFFSET_CLOSE: trade.price = max(order.price, sellBestCrossPrice) self.strategy.posDict[symbol + "_LONG"] -= order.totalVolume @@ -2037,14 +2081,14 @@ def crossLimitOrder(self, data): trade.volume = order.totalVolume trade.tradeTime = self.dt.strftime(constant.DATETIME) trade.tradeDatetime = self.dt - + # 提早到推送成交和订单状态前 # 从字典中删除该限价单 if orderID in self.workingLimitOrderDict: del self.workingLimitOrderDict[orderID] self.strategy.onTrade(trade) - + self.tradeDict[tradeID] = trade # 推送委托数据 @@ -2053,12 +2097,12 @@ def crossLimitOrder(self, data): order.price_avg = trade.price self.strategy.onOrder(order) self.processCancelledOrders() - def updateDailyClose(self, symbol, dt, price): # 为啥放在这个函数里，只是因为执行顺序刚好匹配而已，和这个函数干了啥没关系。 # 又不想改原来的newBar，改完子类也要动，只能这样trick，才能维护的了代码的样子。 - self.processCancelledOrders() + self.processCancelledOrders() super(PatchedBacktestingEngine, self).updateDailyClose(symbol, dt, price) + BacktestingEngine = PatchedBacktestingEngine From 4a0ea3bdddc822b4f8dafb48603bdd8bcb2f0e9a Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Thu, 20 Jun 2019 13:16:20 +0800 Subject: [PATCH 3/7] adjust --- vnpy/trader/app/ctaStrategy/ctaBacktesting.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py index 53b7eb8..9f6712f 100644 --- a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py +++ b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py @@ -1597,16 +1597,16 @@ def __init__(self, entryPrice, entryDt, entryID, exitPrice, self.turnover = (self.entryPrice + self.exitPrice) * size * abs(volume) # 成交金额 - if backtestResultType == "Linear": - self.commission = self.turnover * rate # 手续费成本 - self.slippage = slippage * 2 * size * abs(volume) # 滑点成本 - - self.pnl = (self.exitPrice - self.entryPrice) * volume * size - self.commission - self.slippage # 净盈亏 if backtestResultType == "Inverse": self.commission = rate * self.turnover/ self.exitPrice # 手续费成本 self.slippage = slippage/self.entryPrice * size * abs(volume) + slippage/self.exitPrice * size * abs(volume) # 滑点成本 self.pnl = (self.exitPrice - self.entryPrice) * volume * size / self.exitPrice - self.commission - self.slippage# 净盈亏 + else: + self.commission = self.turnover * rate # 手续费成本 + self.slippage = slippage * 2 * size * abs(volume) # 滑点成本 + + self.pnl = (self.exitPrice - self.entryPrice) * volume * size - self.commission - self.slippage # 净盈亏 ######################################################################## From 8c76ee66b13dd12ba35a4cb1fd547452a6ca67a5 Mon Sep 17 00:00:00 2001 From: tianrq10 Date: Tue, 25 Jun 2019 16:32:32 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E5=9B=9E=E6=B5=8B=E5=AE=8C=E6=AF=95?= =?UTF-8?q?=E5=8F=AF=E4=BF=9D=E5=AD=98=E6=97=A5=E5=87=80=E5=80=BC=E5=81=9A?= =?UTF-8?q?=E8=BF=9B=E4=B8=80=E6=AD=A5=E5=88=86=E6=9E=90=EF=BC=9B=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E7=8B=AC=E7=AB=8B=E7=9A=84=E5=87=80=E5=80=BC=E5=88=86?= =?UTF-8?q?=E6=9E=90=E6=A8=A1=E5=9D=97=EF=BC=9B=E5=B9=B6=E8=A1=8C=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E4=BC=98=E5=8C=96=E5=A2=9E=E5=8A=A0=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E5=87=80=E5=80=BC=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vnpy/trader/app/ctaStrategy/ctaBacktesting.py | 28 ++- vnpy/trader/utils/NVAnalysis.py | 211 ++++++++++++++++++ vnpy/trader/utils/optimize/__init__.py | 4 +- vnpy/trader/utils/optimize/optimization.py | 14 +- 4 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 vnpy/trader/utils/NVAnalysis.py diff --git a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py index 9f6712f..4774088 100644 --- a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py +++ b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py @@ -1423,6 +1423,7 @@ def calculateDailyStatistics(self, df): df['balance'] = df['netPnl'].cumsum() + self.capital df['return'] = df["netPnl"] / self.capital + df['retWithoutFee'] = df["totalPnl"] / self.capital df['highlevel'] = df['balance'].rolling(min_periods=1, window=len(df), center=False).max() df['drawdown'] = df['balance'] - df['highlevel'] df['ddPercent'] = df['drawdown'] / df['highlevel'] * 100 @@ -1458,11 +1459,19 @@ def calculateDailyStatistics(self, df): annualizedReturn = totalReturn / totalDays * 240 dailyReturn = df['return'].mean() * 100 returnStd = df['return'].std() * 100 + dailyReturnWithoutFee = df['retWithoutFee'].mean() * 100 + returnWithoutFeeStd = df['retWithoutFee'].std() * 100 if returnStd: sharpeRatio = dailyReturn / returnStd * np.sqrt(240) else: sharpeRatio = 0 + if returnWithoutFeeStd: + SRWithoutFee = dailyReturnWithoutFee / returnWithoutFeeStd * np.sqrt(240) + else: + SRWithoutFee = 0 + theoreticalSRWithoutFee = 0.1155 * np.sqrt(dailyTradeCount * 240) + calmarRatio = annualizedReturn/abs(maxDdPercent) # 返回结果 result = { @@ -1486,9 +1495,14 @@ def calculateDailyStatistics(self, df): 'dailyTradeCount': float(dailyTradeCount), 'totalReturn': float(totalReturn), 'annualizedReturn': float(annualizedReturn), + 'calmarRatio': float(calmarRatio), 'dailyReturn': float(dailyReturn), 'returnStd': float(returnStd), - 'sharpeRatio': float(sharpeRatio) + 'sharpeRatio': float(sharpeRatio), + 'dailyReturnWithoutFee': float(dailyReturnWithoutFee), + 'returnWithoutFeeStd': float(returnWithoutFeeStd), + 'SRWithoutFee': float(SRWithoutFee), + 'theoreticalSRWithoutFee': float(theoreticalSRWithoutFee) } return df, result @@ -1522,6 +1536,7 @@ def showDailyResult(self): self.output(u'总盈亏：\t%s' % formatNumber(result['totalNetPnl'])) self.output(u'最大回撤: \t%s' % formatNumber(result['maxDrawdown'])) self.output(u'百分比最大回撤: %s%%' % formatNumber(result['maxDdPercent'])) + self.output(u'卡玛比率：\t%s' % formatNumber(result['calmarRatio'])) self.output(u'总手续费：\t%s' % formatNumber(result['totalCommission'])) self.output(u'总滑点：\t%s' % formatNumber(result['totalSlippage'])) @@ -1538,6 +1553,11 @@ def showDailyResult(self): self.output(u'收益标准差：\t%s%%' % formatNumber(result['returnStd'])) self.output(u'Sharpe Ratio：\t%s' % formatNumber(result['sharpeRatio'])) + self.output(u'日均收益率(0交易成本)：\t%s%%' % formatNumber(result['dailyReturnWithoutFee'])) + self.output(u'收益标准差(0交易成本)：\t%s%%' % formatNumber(result['returnWithoutFeeStd'])) + self.output(u'Sharpe Ratio(0交易成本)：\t%s' % formatNumber(result['SRWithoutFee'])) + self.output(u'理论可实现Sharpe Ratio(0交易成本)：\t%s' % formatNumber(result['theoreticalSRWithoutFee'])) + # 绘图 fig = plt.figure(figsize=(10, 16)) @@ -1559,7 +1579,7 @@ def showDailyResult(self): # 输出回测绩效图 if self.logActive: - filename = os.path.join(self.logPath, u"回测绩效图.png") + filename = os.path.join(self.logPath, u"每日净值图.png") plt.savefig(filename) self.output(u'策略回测绩效图已保存') @@ -1569,6 +1589,10 @@ def showDailyResult(self): json.dump(self.strategy_setting, f, indent=4) self.output(u'BacktestingResult saved') + filename = os.path.join(self.logPath, u"每日净值.csv") + df.to_csv(filename, sep=',') + self.output(u'每日净值已保存') + plt.show() diff --git a/vnpy/trader/utils/NVAnalysis.py b/vnpy/trader/utils/NVAnalysis.py new file mode 100644 index 0000000..542168a --- /dev/null +++ b/vnpy/trader/utils/NVAnalysis.py @@ -0,0 +1,211 @@ +# encoding=utf-8 +''' +净值分析工具 +提供:净值分析、净值合并分析、相关性分析 +''' +import copy +from functools import reduce + +import pandas as pd +import numpy as np + + +def getWeight(nvDf_dict, weightMethod="equal"): + result = {} + if weightMethod == "equal": + result = {name: 1 for name in nvDf_dict.keys()} + elif weightMethod == "equal_vol": + for name in nvDf_dict.keys(): + result[name] = 1 / max(0.01, (nvDf_dict[name]["return"].std() * 100)) + elif weightMethod == "equal_maxdd": + for name in nvDf_dict.keys(): + maxDdPercent = abs(nvDf_dict[name]['ddPercent'].min()) + result[name] = 1 / max(0.01, maxDdPercent) + elif weightMethod == "sharpe": + for name in nvDf_dict.keys(): + dailyReturn = nvDf_dict[name]['return'].mean() * 100 + returnStd = nvDf_dict[name]['return'].std() * 100 + sharpeRatio = dailyReturn / returnStd * np.sqrt(240) + result[name] = max(0, sharpeRatio) + elif weightMethod == "calmar": + for name in nvDf_dict.keys(): + df = nvDf_dict[name] + totalDays = len(df) + endBalance = df['balance'].iloc[-1] + totalReturn = (endBalance - 1) * 100 + annualizedReturn = totalReturn / totalDays * 240 + maxDdPercent = abs(df['ddPercent'].min()) + calmarRatio = annualizedReturn / max(0.01, maxDdPercent) + result[name] = max(0, calmarRatio) + else: + raise ValueError("weightMethod can only choose equal:等权 equal_vol:波动性标准化 equal_maxdd:最大回撤标准化 sharpe:夏普比率加权 calmar：卡玛比率加权") + + # 权重值之和调整为0 + _sum = 0 + for name in result.keys(): + _sum += result[name] + for name in result.keys(): + result[name] = result[name] / _sum + return result + + +def combineNV(nvDf_dict, weightMethod="equal", weight=None): + ''' + :param nvDf_dict:各子策略净值表 + :param weightMethod: 内置加权方法 equal：等权 equal_vol:波动性标准化 equal_maxdd:最大回撤标准化 sharpe:夏普比率加权 calmar：卡玛比率加权 + :param weight:自定义权重。要求传入一个dict，key和nvDf_dict相同，值为权重值 + :return:合并净值表, 权重 + ''' + nvDf_dict = copy.deepcopy(nvDf_dict) + # 对齐数据 + _ = nvDf_dict[list(nvDf_dict.keys())[0]] + for name in nvDf_dict.keys(): + nvDf_dict[name] = nvDf_dict[name].reindex(_.index).replace([np.inf, -np.inf], np.nan) + nvDf_dict[name][ + ["netPnl", "slippage", "commission", "turnover", "tradeCount", "tradingPnl", "positionPnl", "totalPnl", + "return", "retWithoutFee"]] = \ + nvDf_dict[name][ + ["netPnl", "slippage", "commission", "turnover", "tradeCount", "tradingPnl", "positionPnl", "totalPnl", + "return", "retWithoutFee"]].fillna(0) + nvDf_dict[name] = nvDf_dict[name].fillna(method="ffill") + + # 计算权重 + if weight is None: + weight = getWeight(nvDf_dict, weightMethod) + else: + _sum = 0 + for name in weight.keys(): + _sum += weight[name] + for name in weight.keys(): + weight[name] = weight[name] / _sum + + # 净值归一化 + for name in nvDf_dict.keys(): + df = nvDf_dict[name] + capital = df['balance'].iloc[0] + df['netPnl'].iloc[0] + df["netPnl"] = df["netPnl"] / capital + df["slippage"] = df["slippage"] / capital + df["commission"] = df["commission"] / capital + df["turnover"] = df["turnover"] / capital + df["tradingPnl"] = df["tradingPnl"] / capital + df["positionPnl"] = df["positionPnl"] / capital + df["totalPnl"] = df["totalPnl"] / capital + df["balance"] = df["balance"] / capital + tradeCount = df["tradeCount"].copy() + nvDf_dict[name] = df * weight[name] + if weight[name] > 0: + nvDf_dict[name]["tradeCount"] = tradeCount + + # 计算合并净值表 + def _sum_table(x, y): + return x + y + + combined_NV_table = reduce(_sum_table, nvDf_dict.values()) + combined_NV_table['return'] = combined_NV_table["netPnl"] + combined_NV_table['retWithoutFee'] = combined_NV_table["totalPnl"] + combined_NV_table['highlevel'] = combined_NV_table['balance'].rolling(min_periods=1, window=len(combined_NV_table), + center=False).max() + combined_NV_table['drawdown'] = combined_NV_table['balance'] - combined_NV_table['highlevel'] + combined_NV_table['ddPercent'] = combined_NV_table['drawdown'] / combined_NV_table['highlevel'] * 100 + + return combined_NV_table, weight + + +def getPearsonrMatrix(nvDf_dict): + nvDf_dict = copy.deepcopy(nvDf_dict) + _ = nvDf_dict[list(nvDf_dict.keys())[0]] + for name in nvDf_dict.keys(): + nvDf_dict[name] = nvDf_dict[name].reindex(_.index).replace([np.inf, -np.inf], np.nan) + x1 = np.vstack([df["return"].fillna(0) for df in nvDf_dict.values()]) + x2 = np.vstack([df["retWithoutFee"].fillna(0) for df in nvDf_dict.values()]) + r1 = pd.DataFrame(np.corrcoef(x1), columns=nvDf_dict.keys(), index=nvDf_dict.keys()) + r2 = pd.DataFrame(np.corrcoef(x2), columns=nvDf_dict.keys(), index=nvDf_dict.keys()) + return {"return": r1, "retWithoutFee": r2} + + +# 净值分析 +def calculateDailyStatistics(df): + """计算按日统计的结果""" + if not isinstance(df, pd.DataFrame) or df.size <= 0: + return None, {} + + # 计算统计结果 + df.index = pd.to_datetime(df.index) + startDate = df.index[0] + endDate = df.index[-1] + + totalDays = len(df) + profitDays = len(df[df['netPnl'] > 0]) + lossDays = len(df[df['netPnl'] < 0]) + + capital = df['balance'].iloc[0] + df['netPnl'].iloc[0] + endBalance = df['balance'].iloc[-1] + maxDrawdown = df['drawdown'].min() + maxDdPercent = df['ddPercent'].min() + + totalNetPnl = df['netPnl'].sum() + dailyNetPnl = totalNetPnl / totalDays + + totalCommission = df['commission'].sum() + dailyCommission = totalCommission / totalDays + + totalSlippage = df['slippage'].sum() + dailySlippage = totalSlippage / totalDays + + totalTurnover = df['turnover'].sum() + dailyTurnover = totalTurnover / totalDays + + totalTradeCount = df['tradeCount'].sum() + dailyTradeCount = totalTradeCount / totalDays + + totalReturn = (endBalance / capital - 1) * 100 + annualizedReturn = totalReturn / totalDays * 240 + dailyReturn = df['return'].mean() * 100 + returnStd = df['return'].std() * 100 + dailyReturnWithoutFee = df['retWithoutFee'].mean() * 100 + returnWithoutFeeStd = df['retWithoutFee'].std() * 100 + + if returnStd: + sharpeRatio = dailyReturn / returnStd * np.sqrt(240) + else: + sharpeRatio = 0 + if returnWithoutFeeStd: + SRWithoutFee = dailyReturnWithoutFee / returnWithoutFeeStd * np.sqrt(240) + else: + SRWithoutFee = 0 + theoreticalSRWithoutFee = 0.1155 * np.sqrt(dailyTradeCount * 240) + calmarRatio = annualizedReturn / abs(maxDdPercent) + + # 返回结果 + result = { + 'startDate': startDate.strftime("%Y-%m-%d"), + 'endDate': endDate.strftime("%Y-%m-%d"), + 'totalDays': int(totalDays), + 'profitDays': int(profitDays), + 'lossDays': int(lossDays), + 'endBalance': float(endBalance), + 'maxDrawdown': float(maxDrawdown), + 'maxDdPercent': float(maxDdPercent), + 'totalNetPnl': float(totalNetPnl), + 'dailyNetPnl': float(dailyNetPnl), + 'totalCommission': float(totalCommission), + 'dailyCommission': float(dailyCommission), + 'totalSlippage': float(totalSlippage), + 'dailySlippage': float(dailySlippage), + 'totalTurnover': float(totalTurnover), + 'dailyTurnover': float(dailyTurnover), + 'totalTradeCount': int(totalTradeCount), + 'dailyTradeCount': float(dailyTradeCount), + 'totalReturn': float(totalReturn), + 'annualizedReturn': float(annualizedReturn), + 'calmarRatio': float(calmarRatio), + 'dailyReturn': float(dailyReturn), + 'returnStd': float(returnStd), + 'sharpeRatio': float(sharpeRatio), + 'dailyReturnWithoutFee': float(dailyReturnWithoutFee), + 'returnWithoutFeeStd': float(returnWithoutFeeStd), + 'SRWithoutFee': float(SRWithoutFee), + 'theoreticalSRWithoutFee': float(theoreticalSRWithoutFee) + } + + return result diff --git a/vnpy/trader/utils/optimize/__init__.py b/vnpy/trader/utils/optimize/__init__.py index 824d2cc..60eef98 100644 --- a/vnpy/trader/utils/optimize/__init__.py +++ b/vnpy/trader/utils/optimize/__init__.py @@ -79,11 +79,11 @@ def run(): return opt.report() -def runParallel(process=None): +def runParallel(process=None, save_path=None): if isinstance(_memory, OptMemory): _memory.save_report() - opt = getOpt().runParallel(process) + opt = getOpt().runParallel(process, save_path) if isinstance(_memory, OptMemory): return _memory.save_report() else: diff --git a/vnpy/trader/utils/optimize/optimization.py b/vnpy/trader/utils/optimize/optimization.py index 07f6815..5c4a1f9 100644 --- a/vnpy/trader/utils/optimize/optimization.py +++ b/vnpy/trader/utils/optimize/optimization.py @@ -42,16 +42,20 @@ def runStrategy(engineClass, strategyClass, engineSetting, globalSetting, strate return engine -def runPerformance(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number=0): +def runPerformance(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number=0, save_path=None): engine = runStrategy(engineClass, strategyClass, engineSetting, globalSetting, strategySetting.copy()) dr = engine.calculateDailyResult() ds, r = engine.calculateDailyStatistics(dr) + if save_path is not None: + if not os.path.isdir(save_path): + os.makedirs(save_path) + ds.to_hdf(f"{save_path}/{number}.hd5", "/", format="table", complevel=9) return {"setting": strategySetting, "result": r, INDEX_NAME: number} -def runPerformanceParallel(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number=0): +def runPerformanceParallel(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number=0, save_path=None): try: - r = runPerformance(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number) + r = runPerformance(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number, save_path) except: pe = ParallelError( number=number, @@ -189,7 +193,7 @@ def run(self): return self - def runParallel(self, processes=None): + def runParallel(self, processes=None, save_path=None): if not self.ready: return self @@ -199,7 +203,7 @@ def runParallel(self, processes=None): for index, strategySetting in self.iter_settings(): pool.apply_async( runPerformanceParallel, - (self.engineClass, self.strategyClass, self.engineSetting, self.globalSetting, strategySetting, index), + (self.engineClass, self.strategyClass, self.engineSetting, self.globalSetting, strategySetting, index, save_path), callback=self.callback, error_callback=self.error_callback ) From 4396e6cc1dbd2220564743f1a149bb3728095513 Mon Sep 17 00:00:00 2001 From: tianrq10 Date: Tue, 25 Jun 2019 18:14:43 +0800 Subject: [PATCH 5/7] update optimization --- vnpy/trader/utils/optimize/optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/trader/utils/optimize/optimization.py b/vnpy/trader/utils/optimize/optimization.py index 5c4a1f9..4a9ede2 100644 --- a/vnpy/trader/utils/optimize/optimization.py +++ b/vnpy/trader/utils/optimize/optimization.py @@ -49,7 +49,7 @@ def runPerformance(engineClass, strategyClass, engineSetting, globalSetting, st if save_path is not None: if not os.path.isdir(save_path): os.makedirs(save_path) - ds.to_hdf(f"{save_path}/{number}.hd5", "/", format="table", complevel=9) + ds.to_hdf(f"{save_path}/{number}.hd5", "/table", format="table", complevel=9) return {"setting": strategySetting, "result": r, INDEX_NAME: number} From 7d0e05781ff3077ff125fe20b1d3d9cde51fe315 Mon Sep 17 00:00:00 2001 From: tianrq10 Date: Wed, 3 Jul 2019 15:34:10 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8DNVanalysis=E5=AF=B9?= =?UTF-8?q?=E9=BD=90=E6=95=B0=E6=8D=AE=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vnpy/trader/utils/NVAnalysis.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/vnpy/trader/utils/NVAnalysis.py b/vnpy/trader/utils/NVAnalysis.py index 542168a..0f210c7 100644 --- a/vnpy/trader/utils/NVAnalysis.py +++ b/vnpy/trader/utils/NVAnalysis.py @@ -58,9 +58,12 @@ def combineNV(nvDf_dict, weightMethod="equal", weight=None): ''' nvDf_dict = copy.deepcopy(nvDf_dict) # 对齐数据 - _ = nvDf_dict[list(nvDf_dict.keys())[0]] + _index = set(nvDf_dict[list(nvDf_dict.keys())[0]].index) for name in nvDf_dict.keys(): - nvDf_dict[name] = nvDf_dict[name].reindex(_.index).replace([np.inf, -np.inf], np.nan) + _index = _index & set(nvDf_dict[name].index) + _index = sorted(list(_index)) + for name in nvDf_dict.keys(): + nvDf_dict[name] = nvDf_dict[name].reindex(_index).replace([np.inf, -np.inf], np.nan) nvDf_dict[name][ ["netPnl", "slippage", "commission", "turnover", "tradeCount", "tradingPnl", "positionPnl", "totalPnl", "return", "retWithoutFee"]] = \ @@ -92,7 +95,6 @@ def combineNV(nvDf_dict, weightMethod="equal", weight=None): df["totalPnl"] = df["totalPnl"] / capital df["balance"] = df["balance"] / capital tradeCount = df["tradeCount"].copy() - nvDf_dict[name] = df * weight[name] if weight[name] > 0: nvDf_dict[name]["tradeCount"] = tradeCount @@ -113,9 +115,13 @@ def _sum_table(x, y): def getPearsonrMatrix(nvDf_dict): nvDf_dict = copy.deepcopy(nvDf_dict) - _ = nvDf_dict[list(nvDf_dict.keys())[0]] + # 对齐数据 + _index = set(nvDf_dict[list(nvDf_dict.keys())[0]].index) + for name in nvDf_dict.keys(): + _index = _index & set(nvDf_dict[name].index) + _index = sorted(list(_index)) for name in nvDf_dict.keys(): - nvDf_dict[name] = nvDf_dict[name].reindex(_.index).replace([np.inf, -np.inf], np.nan) + nvDf_dict[name] = nvDf_dict[name].reindex(_index).replace([np.inf, -np.inf], np.nan) x1 = np.vstack([df["return"].fillna(0) for df in nvDf_dict.values()]) x2 = np.vstack([df["retWithoutFee"].fillna(0) for df in nvDf_dict.values()]) r1 = pd.DataFrame(np.corrcoef(x1), columns=nvDf_dict.keys(), index=nvDf_dict.keys()) From cb3cfecfe380a50ae0cb3b18e6393c3d96cf86db Mon Sep 17 00:00:00 2001 From: tianrq10 Date: Wed, 3 Jul 2019 16:15:44 +0800 Subject: [PATCH 7/7] =?UTF-8?q?combineNV=E5=A2=9E=E5=8A=A0=E5=8F=82?= =?UTF-8?q?=E6=95=B0=EF=BC=9A=E6=98=AF=E5=90=A6=E5=BD=92=E4=B8=80=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vnpy/trader/utils/NVAnalysis.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/vnpy/trader/utils/NVAnalysis.py b/vnpy/trader/utils/NVAnalysis.py index 0f210c7..f7d7b4d 100644 --- a/vnpy/trader/utils/NVAnalysis.py +++ b/vnpy/trader/utils/NVAnalysis.py @@ -49,7 +49,7 @@ def getWeight(nvDf_dict, weightMethod="equal"): return result -def combineNV(nvDf_dict, weightMethod="equal", weight=None): +def combineNV(nvDf_dict, weightMethod="equal", weight=None, normalized=True): ''' :param nvDf_dict:各子策略净值表 :param weightMethod: 内置加权方法 equal：等权 equal_vol:波动性标准化 equal_maxdd:最大回撤标准化 sharpe:夏普比率加权 calmar：卡玛比率加权 @@ -76,11 +76,13 @@ def combineNV(nvDf_dict, weightMethod="equal", weight=None): if weight is None: weight = getWeight(nvDf_dict, weightMethod) else: - _sum = 0 - for name in weight.keys(): - _sum += weight[name] - for name in weight.keys(): - weight[name] = weight[name] / _sum + weight = weight.copy() + if normalized: + _sum = 0 + for name in weight.keys(): + _sum += weight[name] + for name in weight.keys(): + weight[name] = weight[name] / _sum # 净值归一化 for name in nvDf_dict.keys(): @@ -95,6 +97,7 @@ def combineNV(nvDf_dict, weightMethod="equal", weight=None): df["totalPnl"] = df["totalPnl"] / capital df["balance"] = df["balance"] / capital tradeCount = df["tradeCount"].copy() + nvDf_dict[name] = df * weight[name] if weight[name] > 0: nvDf_dict[name]["tradeCount"] = tradeCount