Detect and resample intraday strategies; remove gross_lev (quantopian…

…#351)

* ENH: Detect and resample intraday strategies

* ENH: Remove gross_lev; just compute on the spot when needed using positions df

* Use random sample of N=16 for round trip lifetimes plot

pyfolio/examples/overview_slides.ipynb

@@ -604,7 +604,7 @@
    },
    "outputs": [],
    "source": [
-    "returns, positions, transactions, gross_lev = \\\n",
+    "returns, positions, transactions = \\\n",
     "    pf.utils.extract_rets_pos_txn_from_zipline(backtest)"
    ]
   },
@@ -1044,7 +1044,6 @@
     "pf.create_full_tear_sheet(returns,\n",
     "                          positions=positions,\n",
     "                          transactions=transactions,\n",
-    "                          gross_lev=gross_lev,\n",
     "                          live_start_date=oos_date,\n",
     "                          slippage=0.1,\n",
     "                          sector_mappings=sector_map)"

pyfolio/examples/sector_mapping_example.ipynb

@@ -42,8 +42,6 @@
     "positions = pd.read_csv(gzip.open('../tests/test_data/test_pos.csv.gz'),\n",
     "                    index_col=0, parse_dates=0)\n",
     "returns = pd.read_csv(gzip.open('../tests/test_data/test_returns.csv.gz'),\n",
-    "                    index_col=0, parse_dates=0, header=None)[1]\n",
-    "gross_lev = pd.read_csv(gzip.open('../tests/test_data/test_gross_lev.csv.gz'),\n",
     "                    index_col=0, parse_dates=0, header=None)[1]"
    ]
   },
@@ -57,8 +55,7 @@
    "source": [
     "returns.index = returns.index.tz_localize(\"UTC\")\n",
     "positions.index = positions.index.tz_localize(\"UTC\")\n",
-    "transactions.index = transactions.index.tz_localize(\"UTC\")\n",
-    "gross_lev.index = gross_lev.index.tz_localize(\"UTC\")\n"
+    "transactions.index = transactions.index.tz_localize(\"UTC\")"
    ]
   },
   {
@@ -215,7 +212,7 @@
     }
    ],
    "source": [
-    "pf.create_position_tear_sheet(returns, positions, gross_lev=gross_lev, sector_mappings=sect_map)\n"
+    "pf.create_position_tear_sheet(returns, positions, sector_mappings=sect_map)\n"
    ]
   },
   {
@@ -306,7 +303,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython2",
-   "version": "2.7.10"
+   "version": "2.7.12"
   }
  },
  "nbformat": 4,

pyfolio/examples/slippage_example.ipynb

@@ -48,12 +48,9 @@
     "                    index_col=0, parse_dates=0)\n",
     "returns = pd.read_csv(gzip.open('../tests/test_data/test_returns.csv.gz'),\n",
     "                    index_col=0, parse_dates=0, header=None)[1]\n",
-    "gross_lev = pd.read_csv(gzip.open('../tests/test_data/test_gross_lev.csv.gz'),\n",
-    "                    index_col=0, parse_dates=0, header=None)[1]\n",
     "returns.index = returns.index.tz_localize(\"UTC\")\n",
     "positions.index = positions.index.tz_localize(\"UTC\")\n",
-    "transactions.index = transactions.index.tz_localize(\"UTC\")\n",
-    "gross_lev.index = gross_lev.index.tz_localize(\"UTC\")"
+    "transactions.index = transactions.index.tz_localize(\"UTC\")"
    ]
   },
   {
@@ -545,7 +542,7 @@
     }
    ],
    "source": [
-    "pf.create_full_tear_sheet(returns, positions, transactions, gross_lev=gross_lev, slippage=0)"
+    "pf.create_full_tear_sheet(returns, positions, transactions, slippage=0)"
    ]
   }
  ],
@@ -565,7 +562,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.0"
+   "version": "3.4.3"
   }
  },
  "nbformat": 4,

pyfolio/examples/zipline_algo_example.ipynb

@@ -256,7 +256,7 @@
    },
    "outputs": [],
    "source": [
-    "returns, positions, transactions, gross_lev = pf.utils.extract_rets_pos_txn_from_zipline(results)"
+    "returns, positions, transactions = pf.utils.extract_rets_pos_txn_from_zipline(results)"
    ]
   },
   {
@@ -1381,7 +1381,7 @@
    ],
    "source": [
     "pf.create_full_tear_sheet(returns, positions=positions, transactions=transactions,\n",
-    "                          gross_lev=gross_lev, live_start_date='2009-10-22', round_trips=True)"
+    "                          live_start_date='2009-10-22', round_trips=True)"
    ]
   },
   {
@@ -1762,8 +1762,7 @@
    ],
    "source": [
     "pf.create_full_tear_sheet(returns, positions=positions, transactions=transactions,\n",
-    "                          gross_lev=gross_lev, live_start_date='2009-10-22',\n",
-    "                          hide_positions=True)"
+    "                          live_start_date='2009-10-22', hide_positions=True)"
    ]
   }
  ],
@@ -1783,7 +1782,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "version": "3.4.3"
   }
  },
  "nbformat": 4,

pyfolio/plotting.py

@@ -354,7 +354,7 @@ def plot_holdings(returns, positions, legend_loc='best', ax=None, **kwargs):
         ax = plt.gca()
 
     positions = positions.copy().drop('cash', axis='columns')
-    df_holdings = positions.apply(lambda x: np.sum(x != 0), axis='columns')
+    df_holdings = positions.fillna(0).count(axis=1)
     df_holdings_by_month = df_holdings.resample('1M').mean()
     df_holdings.plot(color='steelblue', alpha=0.6, lw=0.5, ax=ax, **kwargs)
     df_holdings_by_month.plot(
@@ -506,7 +506,7 @@ def plot_perf_stats(returns, factor_returns, ax=None):
     return ax
 
 
-def show_perf_stats(returns, factor_returns, gross_lev=None,
+def show_perf_stats(returns, factor_returns, positions=None,
                     live_start_date=None, bootstrap=False):
     """
     Prints some performance metrics of the strategy.
@@ -522,12 +522,15 @@ def show_perf_stats(returns, factor_returns, gross_lev=None,
     returns : pd.Series
         Daily returns of the strategy, noncumulative.
          - See full explanation in tears.create_full_tear_sheet.
-    live_start_date : datetime, optional
-        The point in time when the strategy began live trading, after
-        its backtest period.
     factor_returns : pd.Series
         Daily noncumulative returns of the benchmark.
          - This is in the same style as returns.
+    positions : pd.DataFrame
+        Daily net position values.
+         - See full explanation in create_full_tear_sheet.
+    live_start_date : datetime, optional
+        The point in time when the strategy began live trading, after
+        its backtest period.
     bootstrap : boolean (optional)
         Whether to perform bootstrap analysis for the performance
         metrics.
@@ -544,35 +547,35 @@ def show_perf_stats(returns, factor_returns, gross_lev=None,
         returns_backtest = returns[returns.index < live_start_date]
         returns_live = returns[returns.index > live_start_date]
 
-        gross_lev_backtest = None
-        gross_lev_live = None
-        if gross_lev is not None:
-            gross_lev_backtest = gross_lev[gross_lev.index < live_start_date]
-            gross_lev_live = gross_lev[gross_lev.index > live_start_date]
+        positions_backtest = None
+        positions_live = None
+        if positions is not None:
+            positions_backtest = positions[positions.index < live_start_date]
+            positions_live = positions[positions.index > live_start_date]
 
         perf_stats_live = perf_func(
             returns_live,
             factor_returns=factor_returns,
-            gross_lev=gross_lev_live)
+            positions=positions_live)
 
         perf_stats_all = perf_func(
             returns,
             factor_returns=factor_returns,
-            gross_lev=gross_lev)
+            positions=positions)
 
         print('Out-of-Sample Months: ' +
               str(int(len(returns_live) / APPROX_BDAYS_PER_MONTH)))
     else:
         returns_backtest = returns
-        gross_lev_backtest = gross_lev
+        positions_backtest = positions
 
     print('Backtest Months: ' +
           str(int(len(returns_backtest) / APPROX_BDAYS_PER_MONTH)))
 
     perf_stats = perf_func(
         returns_backtest,
         factor_returns=factor_returns,
-        gross_lev=gross_lev_backtest)
+        positions=positions_backtest)
 
     if live_start_date is not None:
         perf_stats = pd.concat(OrderedDict([
@@ -864,7 +867,7 @@ def plot_rolling_sharpe(returns, rolling_window=APPROX_BDAYS_PER_MONTH * 6,
     return ax
 
 
-def plot_gross_leverage(returns, gross_lev, ax=None, **kwargs):
+def plot_gross_leverage(returns, positions, ax=None, **kwargs):
     """
     Plots gross leverage versus date.
 
@@ -876,9 +879,9 @@ def plot_gross_leverage(returns, gross_lev, ax=None, **kwargs):
     returns : pd.Series
         Daily returns of the strategy, noncumulative.
          - See full explanation in tears.create_full_tear_sheet.
-    gross_lev : pd.Series, optional
-        The leverage of a strategy.
-         - See full explanation in tears.create_full_tear_sheet.
+    positions : pd.DataFrame
+        Daily net position values.
+         - See full explanation in create_full_tear_sheet.
     ax : matplotlib.Axes, optional
         Axes upon which to plot.
     **kwargs, optional
@@ -892,12 +895,10 @@ def plot_gross_leverage(returns, gross_lev, ax=None, **kwargs):
 
     if ax is None:
         ax = plt.gca()
+    gl = timeseries.gross_lev(positions)
+    gl.plot(alpha=0.8, lw=0.5, color='g', legend=False, ax=ax, **kwargs)
 
-    gross_lev.plot(alpha=0.8, lw=0.5, color='g', legend=False, ax=ax,
-                   **kwargs)
-
-    ax.axhline(gross_lev.mean(), color='g', linestyle='--', lw=3,
-               alpha=1.0)
+    ax.axhline(gl.mean(), color='g', linestyle='--', lw=3, alpha=1.0)
 
     ax.set_title('Gross leverage')
     ax.set_ylabel('Gross leverage')
@@ -1623,9 +1624,9 @@ def cumulate_returns(x):
     return ax
 
 
-def plot_round_trip_lifetimes(round_trips, disp_amount=12, lsize=24, ax=None):
+def plot_round_trip_lifetimes(round_trips, disp_amount=16, lsize=18, ax=None):
     """
-    Plots timespans and directions of round trip trades.
+    Plots timespans and directions of a sample of round trip trades.
 
     Parameters
     ----------
@@ -1644,14 +1645,12 @@ def plot_round_trip_lifetimes(round_trips, disp_amount=12, lsize=24, ax=None):
     if ax is None:
         ax = plt.subplot()
 
-    durations = round_trips.groupby('symbol').duration.apply(np.sum)
-    top_symbols = durations.sort_values(ascending=False).index[:disp_amount]
-    top_round_trips = round_trips.copy()[round_trips.symbol.isin(top_symbols)]
+    sample = round_trips.symbol.sample(n=disp_amount, random_state=1)
+    sample_round_trips = round_trips.copy()[round_trips.symbol.isin(sample)]
 
-    symbols = top_round_trips.symbol.unique()
-    symbol_idx = pd.Series(np.arange(len(symbols)), index=symbols)
+    symbol_idx = pd.Series(np.arange(len(sample)), index=sample)
 
-    for symbol, sym_round_trips in top_round_trips.groupby('symbol'):
+    for symbol, sym_round_trips in sample_round_trips.groupby('symbol'):
         for _, row in sym_round_trips.iterrows():
             c = 'b' if row.long else 'r'
             y_ix = symbol_idx[symbol] + 0.05
@@ -1660,9 +1659,9 @@ def plot_round_trip_lifetimes(round_trips, disp_amount=12, lsize=24, ax=None):
                     linewidth=lsize, solid_capstyle='butt')
 
     ax.set_yticks(range(disp_amount))
-    ax.set_yticklabels(symbols)
+    ax.set_yticklabels(sample)
 
-    ax.set_ylim((-0.5, min(len(symbols), disp_amount) - 0.5))
+    ax.set_ylim((-0.5, min(len(sample), disp_amount) - 0.5))
     blue = patches.Rectangle([0, 0], 1, 1, color='b', label='Long')
     red = patches.Rectangle([0, 0], 1, 1, color='r', label='Short')
     leg = ax.legend(handles=[blue, red], frameon=True, loc='lower left')