From ad7d3c801324732be9b6aaa83e9270290cbdf246 Mon Sep 17 00:00:00 2001 From: kdgutier Date: Thu, 27 Oct 2022 21:12:05 -0400 Subject: [PATCH 1/7] MinTrace's protection to Schafer-Strimmer covariance, eliminated statsmodels --- environment.yml | 1 - hierarchicalforecast/_modidx.py | 2 + hierarchicalforecast/core.py | 2 + hierarchicalforecast/methods.py | 58 ++++++++++++++++++++--- nbs/core.ipynb | 84 ++++++++++++++++++++++----------- nbs/methods.ipynb | 63 ++++++++++++++++++++++--- 6 files changed, 169 insertions(+), 41 deletions(-) diff --git a/environment.yml b/environment.yml index 9b349f4c..1c663b6c 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,6 @@ dependencies: - numba - pandas - scikit-learn - - statsmodels - pip - pip: - nbdev diff --git a/hierarchicalforecast/_modidx.py b/hierarchicalforecast/_modidx.py index 825986ea..24893419 100644 --- a/hierarchicalforecast/_modidx.py +++ b/hierarchicalforecast/_modidx.py @@ -56,6 +56,8 @@ 'hierarchicalforecast/methods.py'), 'hierarchicalforecast.methods._reconcile_fcst_proportions': ( 'methods.html#_reconcile_fcst_proportions', 'hierarchicalforecast/methods.py'), + 'hierarchicalforecast.methods.cov2corr': ( 'methods.html#cov2corr', + 'hierarchicalforecast/methods.py'), 'hierarchicalforecast.methods.crossprod': ( 'methods.html#crossprod', 'hierarchicalforecast/methods.py'), 'hierarchicalforecast.methods.is_strictly_hierarchical': ( 'methods.html#is_strictly_hierarchical', diff --git a/hierarchicalforecast/core.py b/hierarchicalforecast/core.py index 4b11d8af..81f751d4 100644 --- a/hierarchicalforecast/core.py +++ b/hierarchicalforecast/core.py @@ -92,6 +92,8 @@ def reconcile(self, uids = Y_hat_df.index.unique() # check if Y_hat_df has the same uids as S if len(S.index.difference(uids)) > 0 or len(Y_hat_df.index.difference(S.index.unique())) > 0: + print(len(S.index.difference(uids))) + print(len(Y_hat_df.index.difference(S.index.unique()))) raise Exception('Summing matrix `S` and `Y_hat_df` do not have the same time series, please check.') # same order of Y_hat_df to prevent errors S_ = S.loc[uids] diff --git a/hierarchicalforecast/methods.py b/hierarchicalforecast/methods.py index 460c3056..3903bca5 100644 --- a/hierarchicalforecast/methods.py +++ b/hierarchicalforecast/methods.py @@ -12,7 +12,6 @@ import numpy as np from numba import njit from quadprog import solve_qp -from statsmodels.stats.moment_helpers import cov2corr # %% ../nbs/methods.ipynb 5 def _reconcile(S: np.ndarray, @@ -336,6 +335,25 @@ def crossprod(x): return x.T @ x # %% ../nbs/methods.ipynb 33 +def cov2corr(cov, return_std=False): + """ convert covariance matrix to correlation matrix + + **Parameters:**
+ `cov`: array_like, 2d covariance matrix.
+ `return_std`: bool=False, if True returned std.
+ + **Returns:**
+ `corr`: ndarray (subclass) correlation matrix + """ + cov = np.asanyarray(cov) + std_ = np.sqrt(np.diag(cov)) + corr = cov / np.outer(std_, std_) + if return_std: + return corr, std_ + else: + return corr + +# %% ../nbs/methods.ipynb 34 class MinTrace: """MinTrace Reconciliation Class. @@ -398,27 +416,53 @@ def reconcile(self, elif self.method == 'wls_struct': W = np.diag(S @ np.ones((n_bottom,))) elif self.method in res_methods: - #we need residuals with shape (obs, n_hiers) + # Residuals with shape (obs, n_hiers) residuals = (y_insample - y_hat_insample).T n, _ = residuals.shape + + # Protection: against overfitted model + residuals_sum = np.sum(residuals, axis=0) + zero_residual_prc = np.abs(residuals_sum) < 1e-4 + zero_residual_prc = np.mean(zero_residual_prc) + if zero_residual_prc > .98: + raise Exception(f'Insample residuals close to 0, zero_residual_prc={zero_residual_prc}. Check `Y_df`') + + # Protection: cases where data is unavailable/nan masked_res = np.ma.array(residuals, mask=np.isnan(residuals)) covm = np.ma.cov(masked_res, rowvar=False, allow_masked=True).data + if self.method == 'wls_var': W = np.diag(np.diag(covm)) elif self.method == 'mint_cov': W = covm elif self.method == 'mint_shrink': + # Schäfer and Strimmer 2005, scale invariant shrinkage + # lasso or ridge might improve numerical stability but + # this version follows https://robjhyndman.com/papers/MinT.pdf tar = np.diag(np.diag(covm)) + + # Protection: constant's correlation set to 0 corm = cov2corr(covm) - xs = np.divide(residuals, np.sqrt(np.diag(covm))) + corm = np.nan_to_num(corm, nan=0.0) + + # Protection: standardized residuals 0 where residual_std=0 + residual_std = np.sqrt(np.diag(covm)) + xs = np.divide(residuals, residual_std, + out=np.zeros_like(residuals), where=residual_std!=0) + xs = xs[~np.isnan(xs).any(axis=1), :] v = (1 / (n * (n - 1))) * (crossprod(xs ** 2) - (1 / n) * (crossprod(xs) ** 2)) np.fill_diagonal(v, 0) + + # Protection: constant's correlation set to 0 corapn = cov2corr(tar) + corapn = np.nan_to_num(corapn, nan=0.0) d = (corm - corapn) ** 2 lmd = v.sum() / d.sum() lmd = max(min(lmd, 1), 0) - W = lmd * tar + (1 - lmd) * covm + + # Protection: final ridge diagonal protection + W = (lmd * tar + (1 - lmd) * covm) + 1e-8 else: raise ValueError(f'Unkown reconciliation method {self.method}') @@ -468,7 +512,7 @@ def reconcile(self, __call__ = reconcile -# %% ../nbs/methods.ipynb 40 +# %% ../nbs/methods.ipynb 41 class OptimalCombination(MinTrace): """Optimal Combination Reconciliation Class. @@ -503,7 +547,7 @@ def __init__(self, self.nonnegative = nonnegative self.insample = False -# %% ../nbs/methods.ipynb 46 +# %% ../nbs/methods.ipynb 47 @njit def lasso(X: np.ndarray, y: np.ndarray, lambda_reg: float, max_iters: int = 1_000, @@ -535,7 +579,7 @@ def lasso(X: np.ndarray, y: np.ndarray, #print(it) return beta -# %% ../nbs/methods.ipynb 47 +# %% ../nbs/methods.ipynb 48 class ERM: """Optimal Combination Reconciliation Class. diff --git a/nbs/core.ipynb b/nbs/core.ipynb index 72766713..0b654286 100644 --- a/nbs/core.ipynb +++ b/nbs/core.ipynb @@ -181,6 +181,8 @@ " uids = Y_hat_df.index.unique()\n", " # check if Y_hat_df has the same uids as S\n", " if len(S.index.difference(uids)) > 0 or len(Y_hat_df.index.difference(S.index.unique())) > 0:\n", + " print(len(S.index.difference(uids)))\n", + " print(len(Y_hat_df.index.difference(S.index.unique())))\n", " raise Exception('Summing matrix `S` and `Y_hat_df` do not have the same time series, please check.')\n", " # same order of Y_hat_df to prevent errors\n", " S_ = S.loc[uids]\n", @@ -333,7 +335,7 @@ "df.insert(0, 'Country', 'Australia')\n", "\n", "# non strictly hierarchical structure\n", - "hiers_grouped = [\n", + "hierS_grouped_df = [\n", " ['Country'],\n", " ['Country', 'State'], \n", " ['Country', 'Purpose'], \n", @@ -349,7 +351,7 @@ "]\n", "\n", "# getting df\n", - "hier_grouped_df, S_grouped, tags_grouped = aggregate(df, hiers_grouped)\n", + "hier_grouped_df, S_grouped_df, tags_grouped = aggregate(df, hierS_grouped_df)\n", "hier_strict_df, S_strict, tags_strict = aggregate(df, hiers_strictly)" ] }, @@ -362,8 +364,8 @@ "#| hide\n", "hier_grouped_df['y_model'] = hier_grouped_df['y']\n", "# we should be able to recover y using the methods\n", - "hier_grouped_df_h = hier_grouped_df.groupby('unique_id').tail(12)\n", - "ds_h = hier_grouped_df_h['ds'].unique()\n", + "hier_grouped_hat_df = hier_grouped_df.groupby('unique_id').tail(12)\n", + "ds_h = hier_grouped_hat_df['ds'].unique()\n", "hier_grouped_df = hier_grouped_df.query('~(ds in @ds_h)')\n", "#adding noise to `y_model` to avoid perfect fited values\n", "hier_grouped_df['y_model'] += np.random.uniform(-1, 1, len(hier_grouped_df))\n", @@ -383,8 +385,8 @@ " # ERM recovers but needs bigger eps\n", " #ERM(method='reg_bu', lambda_reg=None),\n", "])\n", - "reconciled = hrec.reconcile(Y_hat_df=hier_grouped_df_h, Y_df=hier_grouped_df, \n", - " S=S_grouped, tags=tags_grouped)\n", + "reconciled = hrec.reconcile(Y_hat_df=hier_grouped_hat_df, Y_df=hier_grouped_df, \n", + " S=S_grouped_df, tags=tags_grouped)\n", "for model in reconciled.drop(columns=['ds', 'y']).columns:\n", " if 'ERM' in model:\n", " eps = 3\n", @@ -407,19 +409,19 @@ "test_fail(\n", " hrec.reconcile,\n", " contains='do not have the same time series',\n", - " args=(hier_grouped_df_h.drop('Australia'), S_grouped, tags_grouped, hier_grouped_df),\n", + " args=(hier_grouped_hat_df.drop('Australia'), S_grouped_df, tags_grouped, hier_grouped_df),\n", " \n", ")\n", "test_fail(\n", " hrec.reconcile,\n", " contains='do not have the same time series',\n", - " args=(hier_grouped_df_h, S_grouped.drop('Australia'), tags_grouped, hier_grouped_df),\n", + " args=(hier_grouped_hat_df, S_grouped_df.drop('Australia'), tags_grouped, hier_grouped_df),\n", " \n", ")\n", "test_fail(\n", " hrec.reconcile,\n", " contains='do not have the same time series',\n", - " args=(hier_grouped_df_h, S_grouped, tags_grouped, hier_grouped_df.drop('Australia')),\n", + " args=(hier_grouped_hat_df, S_grouped_df, tags_grouped, hier_grouped_df.drop('Australia')),\n", " \n", ")" ] @@ -440,8 +442,8 @@ " MinTrace(method='ols', nonnegative=True),\n", " MinTrace(method='wls_struct', nonnegative=True),\n", "])\n", - "reconciled = hrec.reconcile(Y_hat_df=hier_grouped_df_h,\n", - " S=S_grouped, tags=tags_grouped)\n", + "reconciled = hrec.reconcile(Y_hat_df=hier_grouped_hat_df,\n", + " S=S_grouped_df, tags=tags_grouped)\n", "for model in reconciled.drop(columns=['ds', 'y']).columns:\n", " if 'ERM' in model:\n", " eps = 3\n", @@ -465,7 +467,7 @@ "test_fail(\n", " hrec.reconcile,\n", " contains='requires strictly hierarchical structures',\n", - " args=(hier_grouped_df_h, S_grouped, tags_grouped, hier_grouped_df,)\n", + " args=(hier_grouped_hat_df, S_grouped_df, tags_grouped, hier_grouped_df,)\n", ")" ] }, @@ -556,6 +558,27 @@ " test_close(reconciled['y'], reconciled[model], eps)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "# MinTrace should break\n", + "# with extremely overfitted model, y_model==y\n", + "\n", + "zero_df = hier_grouped_df.copy()\n", + "zero_df['y'] = 0\n", + "zero_df['y_model'] = 0\n", + "hrec = HierarchicalReconciliation([MinTrace(method='mint_shrink')])\n", + "test_fail(\n", + " hrec.reconcile,\n", + " contains='Insample residuals',\n", + " args=(hier_grouped_hat_df, S_grouped_df, tags_grouped, zero_df)\n", + ")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -568,9 +591,9 @@ "#that argument\n", "hrec = HierarchicalReconciliation([MinTrace(method='ols')])\n", "reconciled = hrec.reconcile(\n", - " Y_hat_df=hier_grouped_df_h, \n", + " Y_hat_df=hier_grouped_hat_df, \n", " Y_df=hier_grouped_df.drop(columns=['y_model']), \n", - " S=S_grouped, \n", + " S=S_grouped_df, \n", " tags=tags_grouped\n", ")\n", "for model in reconciled.drop(columns=['ds', 'y']).columns:\n", @@ -597,8 +620,8 @@ "#test methods bootstrap prediction\n", "#intervals\n", "hrec = HierarchicalReconciliation([BottomUp()])\n", - "reconciled = hrec.reconcile(hier_grouped_df_h, \n", - " Y_df=hier_grouped_df, S=S_grouped, tags=tags_grouped,\n", + "reconciled = hrec.reconcile(hier_grouped_hat_df, \n", + " Y_df=hier_grouped_df, S=S_grouped_df, tags=tags_grouped,\n", " level=[80, 90], \n", " intervals_method='bootstrap')\n", "total = reconciled.loc[tags_grouped['Country/State/Region/Purpose']].groupby('ds').sum().reset_index()\n", @@ -618,11 +641,11 @@ "#| hide\n", "# test methods that require prediction intervals\n", "# normality\n", - "hier_grouped_df_h['y_model-lo-80'] = hier_grouped_df_h['y_model'] - 1.96\n", - "hier_grouped_df_h['y_model-hi-80'] = hier_grouped_df_h['y_model'] + 1.96\n", + "hier_grouped_hat_df['y_model-lo-80'] = hier_grouped_hat_df['y_model'] - 1.96\n", + "hier_grouped_hat_df['y_model-hi-80'] = hier_grouped_hat_df['y_model'] + 1.96\n", "hrec = HierarchicalReconciliation([BottomUp()])\n", - "reconciled = hrec.reconcile(hier_grouped_df_h, \n", - " Y_df=hier_grouped_df, S=S_grouped, tags=tags_grouped,\n", + "reconciled = hrec.reconcile(hier_grouped_hat_df, \n", + " Y_df=hier_grouped_df, S=S_grouped_df, tags=tags_grouped,\n", " level=[80, 90], \n", " intervals_method='normality')\n", "total = reconciled.loc[tags_grouped['Country/State/Region/Purpose']].groupby('ds').sum().reset_index()\n", @@ -645,13 +668,13 @@ "\n", "# test expect error with grouped structure \n", "# (non strictly hierarchical)\n", - "hier_grouped_df_h['y_model-lo-80'] = hier_grouped_df_h['y_model'] - 1.96\n", - "hier_grouped_df_h['y_model-hi-80'] = hier_grouped_df_h['y_model'] + 1.96\n", + "hier_grouped_hat_df['y_model-lo-80'] = hier_grouped_hat_df['y_model'] - 1.96\n", + "hier_grouped_hat_df['y_model-hi-80'] = hier_grouped_hat_df['y_model'] + 1.96\n", "hrec = HierarchicalReconciliation([BottomUp()])\n", "test_fail(\n", " hrec.reconcile,\n", " contains='requires strictly hierarchical structures',\n", - " args=(hier_grouped_df_h, S_grouped, tags_grouped, hier_grouped_df, [80, 90], 'permbu',)\n", + " args=(hier_grouped_hat_df, S_grouped_df, tags_grouped, hier_grouped_df, [80, 90], 'permbu',)\n", ")\n", "\n", "# test PERMBU\n", @@ -708,7 +731,7 @@ " ['Country', 'State', 'Purpose'], \n", " ['Country', 'State', 'Region', 'Purpose']]\n", "\n", - "Y_df, S, tags = aggregate(df=df, spec=hierarchy_levels)\n", + "Y_df, S_df, tags = aggregate(df=df, spec=hierarchy_levels)\n", "qs = Y_df['ds'].str.replace(r'(\\d+) (Q\\d)', r'\\1-\\2', regex=True)\n", "Y_df['ds'] = pd.PeriodIndex(qs, freq='Q').to_timestamp()\n", "Y_df = Y_df.reset_index()\n", @@ -732,16 +755,23 @@ " MinTrace(method='ols')]\n", "hrec = HierarchicalReconciliation(reconcilers=reconcilers)\n", "Y_rec_df = hrec.reconcile(Y_hat_df=Y_hat_df, Y_df=Y_train_df,\n", - " S=S, tags=tags)\n", + " S=S_df, tags=tags)\n", "Y_rec_df.groupby('unique_id').head(2)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "hierarchicalforecast", "language": "python", - "name": "python3" + "name": "hierarchicalforecast" } }, "nbformat": 4, diff --git a/nbs/methods.ipynb b/nbs/methods.ipynb index 515f90e5..acc5d34b 100644 --- a/nbs/methods.ipynb +++ b/nbs/methods.ipynb @@ -44,8 +44,7 @@ "\n", "import numpy as np\n", "from numba import njit\n", - "from quadprog import solve_qp\n", - "from statsmodels.stats.moment_helpers import cov2corr" + "from quadprog import solve_qp" ] }, { @@ -803,6 +802,32 @@ " return x.T @ x" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| exporti\n", + "def cov2corr(cov, return_std=False):\n", + " \"\"\" convert covariance matrix to correlation matrix\n", + "\n", + " **Parameters:**
\n", + " `cov`: array_like, 2d covariance matrix.
\n", + " `return_std`: bool=False, if True returned std.
\n", + "\n", + " **Returns:**
\n", + " `corr`: ndarray (subclass) correlation matrix\n", + " \"\"\"\n", + " cov = np.asanyarray(cov)\n", + " std_ = np.sqrt(np.diag(cov))\n", + " corr = cov / np.outer(std_, std_)\n", + " if return_std:\n", + " return corr, std_\n", + " else:\n", + " return corr" + ] + }, { "cell_type": "code", "execution_count": null, @@ -872,27 +897,53 @@ " elif self.method == 'wls_struct':\n", " W = np.diag(S @ np.ones((n_bottom,)))\n", " elif self.method in res_methods:\n", - " #we need residuals with shape (obs, n_hiers)\n", + " # Residuals with shape (obs, n_hiers)\n", " residuals = (y_insample - y_hat_insample).T\n", " n, _ = residuals.shape\n", + "\n", + " # Protection: against overfitted model\n", + " residuals_sum = np.sum(residuals, axis=0)\n", + " zero_residual_prc = np.abs(residuals_sum) < 1e-4\n", + " zero_residual_prc = np.mean(zero_residual_prc)\n", + " if zero_residual_prc > .98:\n", + " raise Exception(f'Insample residuals close to 0, zero_residual_prc={zero_residual_prc}. Check `Y_df`')\n", + "\n", + " # Protection: cases where data is unavailable/nan\n", " masked_res = np.ma.array(residuals, mask=np.isnan(residuals))\n", " covm = np.ma.cov(masked_res, rowvar=False, allow_masked=True).data\n", + "\n", " if self.method == 'wls_var':\n", " W = np.diag(np.diag(covm))\n", " elif self.method == 'mint_cov':\n", " W = covm\n", " elif self.method == 'mint_shrink':\n", + " # Schäfer and Strimmer 2005, scale invariant shrinkage\n", + " # lasso or ridge might improve numerical stability but\n", + " # this version follows https://robjhyndman.com/papers/MinT.pdf\n", " tar = np.diag(np.diag(covm))\n", + "\n", + " # Protection: constant's correlation set to 0\n", " corm = cov2corr(covm)\n", - " xs = np.divide(residuals, np.sqrt(np.diag(covm)))\n", + " corm = np.nan_to_num(corm, nan=0.0)\n", + "\n", + " # Protection: standardized residuals 0 where residual_std=0\n", + " residual_std = np.sqrt(np.diag(covm))\n", + " xs = np.divide(residuals, residual_std, \n", + " out=np.zeros_like(residuals), where=residual_std!=0)\n", + "\n", " xs = xs[~np.isnan(xs).any(axis=1), :]\n", " v = (1 / (n * (n - 1))) * (crossprod(xs ** 2) - (1 / n) * (crossprod(xs) ** 2))\n", " np.fill_diagonal(v, 0)\n", + "\n", + " # Protection: constant's correlation set to 0\n", " corapn = cov2corr(tar)\n", + " corapn = np.nan_to_num(corapn, nan=0.0)\n", " d = (corm - corapn) ** 2\n", " lmd = v.sum() / d.sum()\n", " lmd = max(min(lmd, 1), 0)\n", - " W = lmd * tar + (1 - lmd) * covm\n", + "\n", + " # Protection: final ridge diagonal protection\n", + " W = (lmd * tar + (1 - lmd) * covm) + 1e-8\n", " else:\n", " raise ValueError(f'Unkown reconciliation method {self.method}')\n", "\n", @@ -1462,7 +1513,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.9.14 64-bit", "language": "python", "name": "python3" } From 7bb51d2db8fd2bde578db5f754fa90027965e169 Mon Sep 17 00:00:00 2001 From: kdgutier Date: Fri, 28 Oct 2022 09:46:40 -0400 Subject: [PATCH 2/7] Re runned replication examples --- nbs/examples/AustralianDomesticTourism.ipynb | 108 ++++++++++-------- nbs/examples/AustralianPrisonPopulation.ipynb | 91 ++++++++------- 2 files changed, 107 insertions(+), 92 deletions(-) diff --git a/nbs/examples/AustralianDomesticTourism.ipynb b/nbs/examples/AustralianDomesticTourism.ipynb index 0a607e07..cd2275e2 100644 --- a/nbs/examples/AustralianDomesticTourism.ipynb +++ b/nbs/examples/AustralianDomesticTourism.ipynb @@ -23,18 +23,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "%%capture\n", "!pip install hierarchicalforecast\n", - "!pip install -U statsforecast numba statsmodels" + "!pip install -U statsforecast numba" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -159,7 +159,7 @@ "4 Australia Adelaide South Australia Business 1999-01-01 137.448533" ] }, - "execution_count": null, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -183,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -206,17 +206,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ - "Y_df, S, tags = aggregate(Y_df, spec)\n", + "Y_df, S_df, tags = aggregate(Y_df, spec)\n", "Y_df = Y_df.reset_index()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -289,7 +289,7 @@ "4 Australia 1999-01-01 22087.353380" ] }, - "execution_count": null, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -300,7 +300,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -413,18 +413,18 @@ "Australia/Queensland 0.0 " ] }, - "execution_count": null, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "S.iloc[:5, :5]" + "S_df.iloc[:5, :5]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -434,7 +434,7 @@ " 'Australia/Visiting'], dtype=object)" ] }, - "execution_count": null, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -454,7 +454,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -464,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -474,7 +474,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -495,7 +495,7 @@ "Length: 425, dtype: int64" ] }, - "execution_count": null, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -515,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -537,7 +537,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -547,7 +547,7 @@ " MinTrace(method='ols')\n", "]\n", "hrec = HierarchicalReconciliation(reconcilers=reconcilers)\n", - "Y_rec_df = hrec.reconcile(Y_hat_df=Y_hat_df, Y_df=Y_fitted_df, S=S, tags=tags)" + "Y_rec_df = hrec.reconcile(Y_hat_df=Y_hat_df, Y_df=Y_fitted_df, S=S_df, tags=tags)" ] }, { @@ -559,7 +559,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -604,7 +604,7 @@ " 2016-01-01\n", " 25990.068359\n", " 24379.689453\n", - " 25438.888305\n", + " 25438.888350\n", " 25894.418893\n", " \n", " \n", @@ -612,7 +612,7 @@ " 2016-04-01\n", " 24458.490234\n", " 22902.675781\n", - " 23925.188497\n", + " 23925.188540\n", " 24357.230480\n", " \n", " \n", @@ -620,7 +620,7 @@ " 2016-07-01\n", " 23974.056641\n", " 22412.988281\n", - " 23440.310292\n", + " 23440.310337\n", " 23865.929521\n", " \n", " \n", @@ -628,7 +628,7 @@ " 2016-10-01\n", " 24563.455078\n", " 23127.646484\n", - " 24101.001789\n", + " 24101.001832\n", " 24470.783968\n", " \n", " \n", @@ -636,7 +636,7 @@ " 2017-01-01\n", " 25990.068359\n", " 24516.183594\n", - " 25556.667568\n", + " 25556.667615\n", " 25901.382401\n", " \n", " \n", @@ -654,14 +654,14 @@ "\n", " ETS/MinTrace_method-mint_shrink ETS/MinTrace_method-ols \n", "unique_id \n", - "Australia 25438.888305 25894.418893 \n", - "Australia 23925.188497 24357.230480 \n", - "Australia 23440.310292 23865.929521 \n", - "Australia 24101.001789 24470.783968 \n", - "Australia 25556.667568 25901.382401 " + "Australia 25438.888350 25894.418893 \n", + "Australia 23925.188540 24357.230480 \n", + "Australia 23440.310337 23865.929521 \n", + "Australia 24101.001832 24470.783968 \n", + "Australia 25556.667615 25901.382401 " ] }, - "execution_count": null, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -681,14 +681,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_5159/1077664338.py:22: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n", + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_90199/1077664338.py:22: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n", " evaluation = evaluation.drop('Overall')\n" ] } @@ -731,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -834,7 +834,7 @@ "All rmse 45.19 54.95 44.59 42.71" ] }, - "execution_count": null, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -855,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -958,7 +958,7 @@ "All mase 1.03 1.08 0.98 1.02" ] }, - "execution_count": null, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -989,20 +989,30 @@ "- [Rob Hyndman, Alan Lee, Earo Wang, Shanika Wickramasuriya, and Maintainer Earo Wang (2021). \"hts: Hierarchical and Grouped Time Series\". URL https://CRAN.R-project.org/package=hts. R package version 0.3.1.](https://cran.r-project.org/web/packages/hts/index.html)\n", "- [Mitchell O’Hara-Wild, Rob Hyndman, Earo Wang, Gabriel Caceres, Tim-Gunnar Hensel, and Timothy Hyndman (2021). \"fable: Forecasting Models for Tidy Time Series\". URL https://CRAN.R-project.org/package=fable. R package version 6.0.2.](https://CRAN.R-project.org/package=fable)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\"Open" - ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "hierarchicalforecast", "language": "python", - "name": "python3" + "name": "hierarchicalforecast" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b" + } } }, "nbformat": 4, diff --git a/nbs/examples/AustralianPrisonPopulation.ipynb b/nbs/examples/AustralianPrisonPopulation.ipynb index 76af513a..49ff6152 100644 --- a/nbs/examples/AustralianPrisonPopulation.ipynb +++ b/nbs/examples/AustralianPrisonPopulation.ipynb @@ -23,18 +23,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%%capture\n", "!pip install hierarchicalforecast\n", - "!pip install -U statsforecast numba statsmodels" + "!pip install -U statsforecast numba" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -165,7 +165,7 @@ "4 Australia ACT Male Remanded ATSI 2005-03-01 7" ] }, - "execution_count": null, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -188,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -210,17 +210,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "Y_df, S, tags = aggregate(Y_df, hiers, lambda x: np.sum(x) / 1e3)\n", + "Y_df, S_df, tags = aggregate(Y_df, hiers, lambda x: np.sum(x) / 1e3)\n", "Y_df = Y_df.reset_index()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -293,7 +293,7 @@ "4 Australia 2006-03-01 24.524" ] }, - "execution_count": null, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -304,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -403,18 +403,18 @@ "Australia/QLD 0.0 " ] }, - "execution_count": null, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "S.iloc[:5, :5]" + "S_df.iloc[:5, :5]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -445,7 +445,7 @@ " dtype=object)}" ] }, - "execution_count": null, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -465,7 +465,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -475,7 +475,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -494,7 +494,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -516,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -525,7 +525,7 @@ " MinTrace(method='mint_shrink')\n", "]\n", "hrec = HierarchicalReconciliation(reconcilers=reconcilers)\n", - "Y_rec_df = hrec.reconcile(Y_hat_df=Y_hat_df, Y_df=Y_fitted_df, S=S, tags=tags)" + "Y_rec_df = hrec.reconcile(Y_hat_df=Y_hat_df, Y_df=Y_fitted_df, S=S_df, tags=tags)" ] }, { @@ -537,7 +537,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -580,35 +580,35 @@ " 2015-01-01\n", " 34.799496\n", " 34.933872\n", - " 34.927252\n", + " 34.927248\n", " \n", " \n", " Australia\n", " 2015-04-01\n", " 35.192638\n", " 35.473522\n", - " 35.440869\n", + " 35.440866\n", " \n", " \n", " Australia\n", " 2015-07-01\n", " 35.188217\n", " 35.687302\n", - " 35.476417\n", + " 35.476424\n", " \n", " \n", " Australia\n", " 2015-10-01\n", " 35.888626\n", " 36.010601\n", - " 35.946141\n", + " 35.946150\n", " \n", " \n", " Australia\n", " 2016-01-01\n", " 36.045437\n", " 36.400002\n", - " 36.244691\n", + " 36.244702\n", " \n", " \n", "\n", @@ -617,14 +617,14 @@ "text/plain": [ " ds ETS ETS/BottomUp ETS/MinTrace_method-mint_shrink\n", "unique_id \n", - "Australia 2015-01-01 34.799496 34.933872 34.927252\n", - "Australia 2015-04-01 35.192638 35.473522 35.440869\n", - "Australia 2015-07-01 35.188217 35.687302 35.476417\n", - "Australia 2015-10-01 35.888626 36.010601 35.946141\n", - "Australia 2016-01-01 36.045437 36.400002 36.244691" + "Australia 2015-01-01 34.799496 34.933872 34.927248\n", + "Australia 2015-04-01 35.192638 35.473522 35.440866\n", + "Australia 2015-07-01 35.188217 35.687302 35.476424\n", + "Australia 2015-10-01 35.888626 36.010601 35.946150\n", + "Australia 2016-01-01 36.045437 36.400002 36.244702" ] }, - "execution_count": null, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -644,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -731,7 +731,7 @@ "All series 2.00 2.00 2.00" ] }, - "execution_count": null, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -783,20 +783,25 @@ "- [Rob Hyndman, Alan Lee, Earo Wang, Shanika Wickramasuriya, and Maintainer Earo Wang (2021). \"hts: Hierarchical and Grouped Time Series\". URL https://CRAN.R-project.org/package=hts. R package version 0.3.1.](https://cran.r-project.org/web/packages/hts/index.html)\n", "- [Mitchell O’Hara-Wild, Rob Hyndman, Earo Wang, Gabriel Caceres, Tim-Gunnar Hensel, and Timothy Hyndman (2021). \"fable: Forecasting Models for Tidy Time Series\". URL https://CRAN.R-project.org/package=fable. R package version 6.0.2.](https://CRAN.R-project.org/package=fable)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\"Open" - ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "hierarchicalforecast", "language": "python", - "name": "python3" + "name": "hierarchicalforecast" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" } }, "nbformat": 4, From ca92ce176db26c472547425df571904662c1663d Mon Sep 17 00:00:00 2001 From: kdgutier Date: Fri, 28 Oct 2022 09:47:28 -0400 Subject: [PATCH 3/7] Added series difference comments, minor reconcile code ordering --- hierarchicalforecast/core.py | 47 +- hierarchicalforecast/methods.py | 14 +- nbs/core.ipynb | 662 ++++++++++++++++++-- nbs/methods.ipynb | 1001 ++++++++++++++++++++++++++++--- 4 files changed, 1579 insertions(+), 145 deletions(-) diff --git a/hierarchicalforecast/core.py b/hierarchicalforecast/core.py index 81f751d4..4a888637 100644 --- a/hierarchicalforecast/core.py +++ b/hierarchicalforecast/core.py @@ -81,37 +81,50 @@ def reconcile(self, **Returns:**
`y_tilde`: pd.DataFrame, with reconciled predictions. """ + #----------------------------- Preliminary Wrangling/Protections -----------------------------# + # Check input's validity if intervals_method not in ['normality', 'bootstrap', 'permbu']: raise ValueError(f'Unkwon interval method: {intervals_method}') + + if self.insample or (intervals_method in ['bootstrap', 'permbu']): + if Y_df is None: + raise Exception('you need to pass `Y_df`') + + # Declare output names drop_cols = ['ds', 'y'] if 'y' in Y_hat_df.columns else ['ds'] model_names = Y_hat_df.drop(columns=drop_cols, axis=1).columns.to_list() - # store pi names pi_model_names = [name for name in model_names if ('-lo' in name or '-hi' in name)] - #remove prediction intervals model_names = [name for name in model_names if name not in pi_model_names] + uids = Y_hat_df.index.unique() - # check if Y_hat_df has the same uids as S - if len(S.index.difference(uids)) > 0 or len(Y_hat_df.index.difference(S.index.unique())) > 0: - print(len(S.index.difference(uids))) - print(len(Y_hat_df.index.difference(S.index.unique()))) - raise Exception('Summing matrix `S` and `Y_hat_df` do not have the same time series, please check.') - # same order of Y_hat_df to prevent errors + + # Check Y_hat_df\S_df series difference + S_diff = len(S.index.difference(uids)) + Y_hat_diff = len(Y_hat_df.index.difference(S.index.unique())) + if S_diff > 0 or Y_hat_diff > 0: + raise Exception(f'Check `S_df`, `Y_hat_df` series difference, S\Y_hat={S_diff}, Y_hat\S={Y_hat_diff}') + + if Y_df is not None: + # Check Y_hat_df\Y_df series difference + Y_diff = len(Y_df.index.difference(uids)) + Y_hat_diff = len(Y_hat_df.index.difference(Y_df.index.unique())) + if Y_diff > 0 or Y_hat_diff > 0: + raise Exception(f'Check `Y_hat_df`, `Y_df` series difference, Y_hat\Y={Y_hat_diff}, Y\Y_hat={Y_diff}') + + # Same Y_hat_df/S_df/Y_df's unique_id order to prevent errors S_ = S.loc[uids] + + + #---------------------------------------- Predictions ----------------------------------------# + # Initialize reconciler arguments reconciler_args = dict( S=S_.values.astype(np.float32), idx_bottom=S_.index.get_indexer(S.columns), tags={key: S_.index.get_indexer(val) for key, val in tags.items()} ) - # we need insample values if - # we are using a method that requires them - # or if we are performing boostrap - if self.insample or (intervals_method in ['bootstrap', 'permbu']): - if Y_df is None: - raise Exception('you need to pass `Y_df`') - # check if Y_hat_df has the same uids as Y_df - if len(Y_df.index.difference(uids)) > 0 or len(Y_hat_df.index.difference(Y_df.index.unique())) > 0: - raise Exception('Y_df` and `Y_hat_df` do not have the same time series, please check.') + if Y_df is not None: reconciler_args['y_insample'] = Y_df.pivot(columns='ds', values='y').loc[uids].values.astype(np.float32) + fcsts = Y_hat_df.copy() for reconcile_fn in self.reconcilers: reconcile_fn_name = _build_fn_name(reconcile_fn) diff --git a/hierarchicalforecast/methods.py b/hierarchicalforecast/methods.py index 3903bca5..0b896884 100644 --- a/hierarchicalforecast/methods.py +++ b/hierarchicalforecast/methods.py @@ -441,12 +441,10 @@ def reconcile(self, # this version follows https://robjhyndman.com/papers/MinT.pdf tar = np.diag(np.diag(covm)) - # Protection: constant's correlation set to 0 - corm = cov2corr(covm) + # Protections: constant's correlation set to 0 + # standardized residuals 0 where residual_std=0 + corm, residual_std = cov2corr(covm, return_std=True) corm = np.nan_to_num(corm, nan=0.0) - - # Protection: standardized residuals 0 where residual_std=0 - residual_std = np.sqrt(np.diag(covm)) xs = np.divide(residuals, residual_std, out=np.zeros_like(residuals), where=residual_std!=0) @@ -512,7 +510,7 @@ def reconcile(self, __call__ = reconcile -# %% ../nbs/methods.ipynb 41 +# %% ../nbs/methods.ipynb 42 class OptimalCombination(MinTrace): """Optimal Combination Reconciliation Class. @@ -547,7 +545,7 @@ def __init__(self, self.nonnegative = nonnegative self.insample = False -# %% ../nbs/methods.ipynb 47 +# %% ../nbs/methods.ipynb 48 @njit def lasso(X: np.ndarray, y: np.ndarray, lambda_reg: float, max_iters: int = 1_000, @@ -579,7 +577,7 @@ def lasso(X: np.ndarray, y: np.ndarray, #print(it) return beta -# %% ../nbs/methods.ipynb 48 +# %% ../nbs/methods.ipynb 49 class ERM: """Optimal Combination Reconciliation Class. diff --git a/nbs/core.ipynb b/nbs/core.ipynb index 0b654286..4fb0672e 100644 --- a/nbs/core.ipynb +++ b/nbs/core.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -27,7 +27,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -45,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -86,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -170,37 +170,50 @@ " **Returns:**
\n", " `y_tilde`: pd.DataFrame, with reconciled predictions. \n", " \"\"\"\n", + " #----------------------------- Preliminary Wrangling/Protections -----------------------------#\n", + " # Check input's validity\n", " if intervals_method not in ['normality', 'bootstrap', 'permbu']:\n", " raise ValueError(f'Unkwon interval method: {intervals_method}')\n", + "\n", + " if self.insample or (intervals_method in ['bootstrap', 'permbu']):\n", + " if Y_df is None:\n", + " raise Exception('you need to pass `Y_df`') \n", + "\n", + " # Declare output names\n", " drop_cols = ['ds', 'y'] if 'y' in Y_hat_df.columns else ['ds']\n", " model_names = Y_hat_df.drop(columns=drop_cols, axis=1).columns.to_list()\n", - " # store pi names\n", " pi_model_names = [name for name in model_names if ('-lo' in name or '-hi' in name)]\n", - " #remove prediction intervals\n", " model_names = [name for name in model_names if name not in pi_model_names]\n", + "\n", " uids = Y_hat_df.index.unique()\n", - " # check if Y_hat_df has the same uids as S\n", - " if len(S.index.difference(uids)) > 0 or len(Y_hat_df.index.difference(S.index.unique())) > 0:\n", - " print(len(S.index.difference(uids)))\n", - " print(len(Y_hat_df.index.difference(S.index.unique())))\n", - " raise Exception('Summing matrix `S` and `Y_hat_df` do not have the same time series, please check.')\n", - " # same order of Y_hat_df to prevent errors\n", + "\n", + " # Check Y_hat_df\\S_df series difference\n", + " S_diff = len(S.index.difference(uids))\n", + " Y_hat_diff = len(Y_hat_df.index.difference(S.index.unique()))\n", + " if S_diff > 0 or Y_hat_diff > 0:\n", + " raise Exception(f'Check `S_df`, `Y_hat_df` series difference, S\\Y_hat={S_diff}, Y_hat\\S={Y_hat_diff}')\n", + "\n", + " if Y_df is not None:\n", + " # Check Y_hat_df\\Y_df series difference\n", + " Y_diff = len(Y_df.index.difference(uids))\n", + " Y_hat_diff = len(Y_hat_df.index.difference(Y_df.index.unique()))\n", + " if Y_diff > 0 or Y_hat_diff > 0:\n", + " raise Exception(f'Check `Y_hat_df`, `Y_df` series difference, Y_hat\\Y={Y_hat_diff}, Y\\Y_hat={Y_diff}')\n", + "\n", + " # Same Y_hat_df/S_df/Y_df's unique_id order to prevent errors\n", " S_ = S.loc[uids]\n", + " \n", + "\n", + " #---------------------------------------- Predictions ----------------------------------------#\n", + " # Initialize reconciler arguments\n", " reconciler_args = dict(\n", " S=S_.values.astype(np.float32),\n", " idx_bottom=S_.index.get_indexer(S.columns),\n", " tags={key: S_.index.get_indexer(val) for key, val in tags.items()}\n", " )\n", - " # we need insample values if \n", - " # we are using a method that requires them\n", - " # or if we are performing boostrap\n", - " if self.insample or (intervals_method in ['bootstrap', 'permbu']):\n", - " if Y_df is None:\n", - " raise Exception('you need to pass `Y_df`')\n", - " # check if Y_hat_df has the same uids as Y_df\n", - " if len(Y_df.index.difference(uids)) > 0 or len(Y_hat_df.index.difference(Y_df.index.unique())) > 0:\n", - " raise Exception('Y_df` and `Y_hat_df` do not have the same time series, please check.')\n", + " if Y_df is not None:\n", " reconciler_args['y_insample'] = Y_df.pivot(columns='ds', values='y').loc[uids].values.astype(np.float32)\n", + "\n", " fcsts = Y_hat_df.copy()\n", " for reconcile_fn in self.reconcilers:\n", " reconcile_fn_name = _build_fn_name(reconcile_fn)\n", @@ -283,9 +296,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### init\n", + "\n", + "> init (reconcilers:List[Callable])\n", + "\n", + "Hierarchical Reconciliation Class.\n", + "\n", + "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", + "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", + "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", + "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", + "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", + "\n", + "**Parameters:**
\n", + "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", + "\n", + "**References:**
\n", + "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### init\n", + "\n", + "> init (reconcilers:List[Callable])\n", + "\n", + "Hierarchical Reconciliation Class.\n", + "\n", + "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", + "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", + "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", + "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", + "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", + "\n", + "**Parameters:**
\n", + "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", + "\n", + "**References:**
\n", + "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(HierarchicalReconciliation, \n", " name='init', title_level=3)" @@ -293,18 +360,160 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### HierarchicalReconciliation\n", + "\n", + "> HierarchicalReconciliation (reconcilers:List[Callable])\n", + "\n", + "Hierarchical Reconciliation Class.\n", + "\n", + "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", + "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", + "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", + "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", + "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", + "\n", + "**Parameters:**
\n", + "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", + "\n", + "**References:**
\n", + "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### HierarchicalReconciliation\n", + "\n", + "> HierarchicalReconciliation (reconcilers:List[Callable])\n", + "\n", + "Hierarchical Reconciliation Class.\n", + "\n", + "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", + "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", + "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", + "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", + "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", + "\n", + "**Parameters:**
\n", + "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", + "\n", + "**References:**
\n", + "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(HierarchicalReconciliation)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L50){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### reconcile\n", + "\n", + "> reconcile (Y_hat_df:pandas.core.frame.DataFrame,\n", + "> S:pandas.core.frame.DataFrame, tags:Dict[str,numpy.ndarray],\n", + "> Y_df:Optional[pandas.core.frame.DataFrame]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> intervals_method:str='normality')\n", + "\n", + "Hierarchical Reconciliation Method.\n", + "\n", + "The `reconcile` method is analogous to SKLearn `fit` method, it applies different \n", + "reconciliation methods instantiated in the `reconcilers` list. \n", + "\n", + "Most reconciliation methods can be described by the following convenient \n", + "linear algebra notation:\n", + "\n", + "$$\\tilde{\\mathbf{y}}_{[a,b],\\tau} = \\mathbf{S}_{[a,b][b]} \\mathbf{P}_{[b][a,b]} \\hat{\\mathbf{y}}_{[a,b],\\tau}$$\n", + "\n", + "where $a, b$ represent the aggregate and bottom levels, $\\mathbf{S}_{[a,b][b]}$ contains\n", + "the hierarchical aggregation constraints, and $\\mathbf{P}_{[b][a,b]}$ varies across \n", + "reconciliation methods. The reconciled predictions are $\\tilde{\\mathbf{y}}_{[a,b],\\tau}$, and the \n", + "base predictions $\\hat{\\mathbf{y}}_{[a,b],\\tau}$.\n", + "\n", + "**Parameters:**
\n", + "`Y_hat_df`: pd.DataFrame, base forecasts with columns `ds` and models to reconcile indexed by `unique_id`.
\n", + "`Y_df`: pd.DataFrame, training set of base time series with columns `['ds', 'y']` indexed by `unique_id`.\n", + "If a class of `self.reconciles` receives `y_hat_insample`, `Y_df` must include them as columns.
\n", + "`S`: pd.DataFrame with summing matrix of size `(base, bottom)`, see [aggregate method](https://nixtla.github.io/hierarchicalforecast/utils.html#aggregate).
\n", + "`tags`: Each key is a level and its value contains tags associated to that level.
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`intervals_method`: str, method used to calculate prediction intervals, one of `normality`, `bootstrap`, `permbu`.
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: pd.DataFrame, with reconciled predictions." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L50){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### reconcile\n", + "\n", + "> reconcile (Y_hat_df:pandas.core.frame.DataFrame,\n", + "> S:pandas.core.frame.DataFrame, tags:Dict[str,numpy.ndarray],\n", + "> Y_df:Optional[pandas.core.frame.DataFrame]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> intervals_method:str='normality')\n", + "\n", + "Hierarchical Reconciliation Method.\n", + "\n", + "The `reconcile` method is analogous to SKLearn `fit` method, it applies different \n", + "reconciliation methods instantiated in the `reconcilers` list. \n", + "\n", + "Most reconciliation methods can be described by the following convenient \n", + "linear algebra notation:\n", + "\n", + "$$\\tilde{\\mathbf{y}}_{[a,b],\\tau} = \\mathbf{S}_{[a,b][b]} \\mathbf{P}_{[b][a,b]} \\hat{\\mathbf{y}}_{[a,b],\\tau}$$\n", + "\n", + "where $a, b$ represent the aggregate and bottom levels, $\\mathbf{S}_{[a,b][b]}$ contains\n", + "the hierarchical aggregation constraints, and $\\mathbf{P}_{[b][a,b]}$ varies across \n", + "reconciliation methods. The reconciled predictions are $\\tilde{\\mathbf{y}}_{[a,b],\\tau}$, and the \n", + "base predictions $\\hat{\\mathbf{y}}_{[a,b],\\tau}$.\n", + "\n", + "**Parameters:**
\n", + "`Y_hat_df`: pd.DataFrame, base forecasts with columns `ds` and models to reconcile indexed by `unique_id`.
\n", + "`Y_df`: pd.DataFrame, training set of base time series with columns `['ds', 'y']` indexed by `unique_id`.\n", + "If a class of `self.reconciles` receives `y_hat_insample`, `Y_df` must include them as columns.
\n", + "`S`: pd.DataFrame with summing matrix of size `(base, bottom)`, see [aggregate method](https://nixtla.github.io/hierarchicalforecast/utils.html#aggregate).
\n", + "`tags`: Each key is a level and its value contains tags associated to that level.
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`intervals_method`: str, method used to calculate prediction intervals, one of `normality`, `bootstrap`, `permbu`.
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: pd.DataFrame, with reconciled predictions." + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(HierarchicalReconciliation.reconcile,\n", " name='reconcile', title_level=3)" @@ -312,7 +521,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +534,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -357,9 +566,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "hier_grouped_df['y_model'] = hier_grouped_df['y']\n", @@ -399,7 +623,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -408,19 +632,19 @@ "# different series S and Y_hat_df\n", "test_fail(\n", " hrec.reconcile,\n", - " contains='do not have the same time series',\n", + " contains='series difference',\n", " args=(hier_grouped_hat_df.drop('Australia'), S_grouped_df, tags_grouped, hier_grouped_df),\n", " \n", ")\n", "test_fail(\n", " hrec.reconcile,\n", - " contains='do not have the same time series',\n", + " contains='series difference',\n", " args=(hier_grouped_hat_df, S_grouped_df.drop('Australia'), tags_grouped, hier_grouped_df),\n", " \n", ")\n", "test_fail(\n", " hrec.reconcile,\n", - " contains='do not have the same time series',\n", + " contains='series difference',\n", " args=(hier_grouped_hat_df, S_grouped_df, tags_grouped, hier_grouped_df.drop('Australia')),\n", " \n", ")" @@ -428,9 +652,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "# test reconcile method without insample\n", @@ -456,7 +691,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -473,9 +708,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "# methods should work with\n", @@ -560,7 +810,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -581,7 +831,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -602,9 +852,163 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dsyy_modely_model/MinTrace_method-ols
unique_id
Australia/ACT2015 Q1566.135463566.135463566.135463
Australia/ACT2015 Q2516.870343516.870343516.870343
Australia/ACT2015 Q3688.203188688.203188688.203188
Australia/ACT2015 Q4597.245569597.245569597.245569
Australia/ACT2016 Q1625.141634625.141634625.141634
...............
Australia/Western Australia2016 Q42656.3307012656.3307012656.330701
Australia/Western Australia2017 Q12570.9116892570.9116892570.911689
Australia/Western Australia2017 Q22438.4879392438.4879392438.487939
Australia/Western Australia2017 Q32493.9550002493.9550002493.955000
Australia/Western Australia2017 Q42635.7542962635.7542962635.754296
\n", + "

96 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " ds y y_model \\\n", + "unique_id \n", + "Australia/ACT 2015 Q1 566.135463 566.135463 \n", + "Australia/ACT 2015 Q2 516.870343 516.870343 \n", + "Australia/ACT 2015 Q3 688.203188 688.203188 \n", + "Australia/ACT 2015 Q4 597.245569 597.245569 \n", + "Australia/ACT 2016 Q1 625.141634 625.141634 \n", + "... ... ... ... \n", + "Australia/Western Australia 2016 Q4 2656.330701 2656.330701 \n", + "Australia/Western Australia 2017 Q1 2570.911689 2570.911689 \n", + "Australia/Western Australia 2017 Q2 2438.487939 2438.487939 \n", + "Australia/Western Australia 2017 Q3 2493.955000 2493.955000 \n", + "Australia/Western Australia 2017 Q4 2635.754296 2635.754296 \n", + "\n", + " y_model/MinTrace_method-ols \n", + "unique_id \n", + "Australia/ACT 566.135463 \n", + "Australia/ACT 516.870343 \n", + "Australia/ACT 688.203188 \n", + "Australia/ACT 597.245569 \n", + "Australia/ACT 625.141634 \n", + "... ... \n", + "Australia/Western Australia 2656.330701 \n", + "Australia/Western Australia 2570.911689 \n", + "Australia/Western Australia 2438.487939 \n", + "Australia/Western Australia 2493.955000 \n", + "Australia/Western Australia 2635.754296 \n", + "\n", + "[96 rows x 4 columns]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "#| hide\n", "reconciled.loc[tags_grouped['Country/State']]" @@ -612,7 +1016,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -634,7 +1038,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -658,7 +1062,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -702,9 +1106,163 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dsNaiveNaive/BottomUpNaive/MinTrace_method-ols
unique_id
Australia2016-12-3126347.60156226347.59960926347.601460
Australia2017-03-3126347.60156226347.59960926347.601460
Australia/ACT2016-12-31667.214111667.214111667.214156
Australia/ACT2017-03-31667.214111667.214111667.214156
Australia/ACT/Business2016-12-31186.399078186.399078186.399069
...............
Australia/Western Australia/Holiday2017-03-31982.752563982.752563982.752600
Australia/Western Australia/Other2016-12-31121.279541121.279541121.279550
Australia/Western Australia/Other2017-03-31121.279541121.279541121.279550
Australia/Western Australia/Visiting2016-12-31787.030396787.030396787.030481
Australia/Western Australia/Visiting2017-03-31787.030396787.030396787.030481
\n", + "

850 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " ds Naive Naive/BottomUp \\\n", + "unique_id \n", + "Australia 2016-12-31 26347.601562 26347.599609 \n", + "Australia 2017-03-31 26347.601562 26347.599609 \n", + "Australia/ACT 2016-12-31 667.214111 667.214111 \n", + "Australia/ACT 2017-03-31 667.214111 667.214111 \n", + "Australia/ACT/Business 2016-12-31 186.399078 186.399078 \n", + "... ... ... ... \n", + "Australia/Western Australia/Holiday 2017-03-31 982.752563 982.752563 \n", + "Australia/Western Australia/Other 2016-12-31 121.279541 121.279541 \n", + "Australia/Western Australia/Other 2017-03-31 121.279541 121.279541 \n", + "Australia/Western Australia/Visiting 2016-12-31 787.030396 787.030396 \n", + "Australia/Western Australia/Visiting 2017-03-31 787.030396 787.030396 \n", + "\n", + " Naive/MinTrace_method-ols \n", + "unique_id \n", + "Australia 26347.601460 \n", + "Australia 26347.601460 \n", + "Australia/ACT 667.214156 \n", + "Australia/ACT 667.214156 \n", + "Australia/ACT/Business 186.399069 \n", + "... ... \n", + "Australia/Western Australia/Holiday 982.752600 \n", + "Australia/Western Australia/Other 121.279550 \n", + "Australia/Western Australia/Other 121.279550 \n", + "Australia/Western Australia/Visiting 787.030481 \n", + "Australia/Western Australia/Visiting 787.030481 \n", + "\n", + "[850 rows x 4 columns]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "#| eval: false\n", "import numpy as np\n", diff --git a/nbs/methods.ipynb b/nbs/methods.ipynb index acc5d34b..761d2f93 100644 --- a/nbs/methods.ipynb +++ b/nbs/methods.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -164,25 +164,133 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L53){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### BottomUp\n", + "\n", + "> BottomUp ()\n", + "\n", + "Bottom Up Reconciliation Class.\n", + "The most basic hierarchical reconciliation is performed using an Bottom-Up strategy. It was proposed for \n", + "the first time by Orcutt in 1968.\n", + "The corresponding hierarchical \"projection\" matrix is defined as:\n", + "$$\\mathbf{P}_{\\text{BU}} = [\\mathbf{0}_{\\mathrm{[b],[a]}}\\;|\\;\\mathbf{I}_{\\mathrm{[b][b]}}]$$\n", + "\n", + "**Parameters:**
\n", + "None\n", + "\n", + "**References:**
\n", + "- [Orcutt, G.H., Watts, H.W., & Edwards, J.B.(1968). \"Data aggregation and information loss\". The American \n", + "Economic Review, 58 , 773{787)](http://www.jstor.org/stable/1815532)." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L53){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### BottomUp\n", + "\n", + "> BottomUp ()\n", + "\n", + "Bottom Up Reconciliation Class.\n", + "The most basic hierarchical reconciliation is performed using an Bottom-Up strategy. It was proposed for \n", + "the first time by Orcutt in 1968.\n", + "The corresponding hierarchical \"projection\" matrix is defined as:\n", + "$$\\mathbf{P}_{\\text{BU}} = [\\mathbf{0}_{\\mathrm{[b],[a]}}\\;|\\;\\mathbf{I}_{\\mathrm{[b][b]}}]$$\n", + "\n", + "**Parameters:**
\n", + "None\n", + "\n", + "**References:**
\n", + "- [Orcutt, G.H., Watts, H.W., & Edwards, J.B.(1968). \"Data aggregation and information loss\". The American \n", + "Economic Review, 58 , 773{787)](http://www.jstor.org/stable/1815532)." + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(BottomUp, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L69){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### BottomUp.reconcile\n", + "\n", + "> BottomUp.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> idx_bottom:numpy.ndarray,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "Bottom Up Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the Bottom Up approach." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L69){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### BottomUp.reconcile\n", + "\n", + "> BottomUp.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> idx_bottom:numpy.ndarray,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "Bottom Up Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the Bottom Up approach." + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(BottomUp.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -208,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -227,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -259,7 +367,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -273,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -292,7 +400,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -327,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -352,7 +460,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -380,7 +488,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -412,7 +520,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -436,7 +544,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -523,25 +631,147 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L151){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### TopDown\n", + "\n", + "> TopDown (method:str)\n", + "\n", + "Top Down Reconciliation Class.\n", + "\n", + "The Top Down hierarchical reconciliation method, distributes the total aggregate predictions and decomposes \n", + "it down the hierarchy using proportions $\\mathbf{p}_{\\mathrm{[b]}}$ that can be actual historical values \n", + "or estimated.\n", + "\n", + "$$\\mathbf{P}=[\\mathbf{p}_{\\mathrm{[b]}}\\;|\\;\\mathbf{0}_{\\mathrm{[b][a,b\\;-1]}}]$$\n", + "**Parameters:**
\n", + "`method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", + "\n", + "**References:**
\n", + "- [CW. Gross (1990). \"Disaggregation methods to expedite product line forecasting\". Journal of Forecasting, 9 , 233–254. \n", + "doi:10.1002/for.3980090304](https://onlinelibrary.wiley.com/doi/abs/10.1002/for.3980090304).
\n", + "- [G. Fliedner (1999). \"An investigation of aggregate variable time series forecast strategies with specific subaggregate \n", + "time series statistical correlation\". Computers and Operations Research, 26 , 1133–1149. \n", + "doi:10.1016/S0305-0548(99)00017-9](https://doi.org/10.1016/S0305-0548(99)00017-9)." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L151){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### TopDown\n", + "\n", + "> TopDown (method:str)\n", + "\n", + "Top Down Reconciliation Class.\n", + "\n", + "The Top Down hierarchical reconciliation method, distributes the total aggregate predictions and decomposes \n", + "it down the hierarchy using proportions $\\mathbf{p}_{\\mathrm{[b]}}$ that can be actual historical values \n", + "or estimated.\n", + "\n", + "$$\\mathbf{P}=[\\mathbf{p}_{\\mathrm{[b]}}\\;|\\;\\mathbf{0}_{\\mathrm{[b][a,b\\;-1]}}]$$\n", + "**Parameters:**
\n", + "`method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", + "\n", + "**References:**
\n", + "- [CW. Gross (1990). \"Disaggregation methods to expedite product line forecasting\". Journal of Forecasting, 9 , 233–254. \n", + "doi:10.1002/for.3980090304](https://onlinelibrary.wiley.com/doi/abs/10.1002/for.3980090304).
\n", + "- [G. Fliedner (1999). \"An investigation of aggregate variable time series forecast strategies with specific subaggregate \n", + "time series statistical correlation\". Computers and Operations Research, 26 , 1133–1149. \n", + "doi:10.1016/S0305-0548(99)00017-9](https://doi.org/10.1016/S0305-0548(99)00017-9)." + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(TopDown, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L174){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### TopDown.reconcile\n", + "\n", + "> TopDown.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> tags:Dict[str,numpy.ndarray],\n", + "> y_insample:Optional[numpy.ndarray]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[numpy.ndarray]=None)\n", + "\n", + "Top Down Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`tags`: Each key is a level and each value its `S` indices.
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Optional for `forecast_proportions` method.
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the Top Down approach." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L174){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### TopDown.reconcile\n", + "\n", + "> TopDown.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> tags:Dict[str,numpy.ndarray],\n", + "> y_insample:Optional[numpy.ndarray]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[numpy.ndarray]=None)\n", + "\n", + "Top Down Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`tags`: Each key is a level and each value its `S` indices.
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Optional for `forecast_proportions` method.
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the Top Down approach." + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(TopDown.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -578,9 +808,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/2462090121.py:56: UserWarning: Prediction intervals not implement for `forecast_proportions`\n", + " warnings.warn('Prediction intervals not implement for `forecast_proportions`')\n" + ] + } + ], "source": [ "#| hide\n", "# test_levels\n", @@ -621,7 +860,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -731,25 +970,133 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L231){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MiddleOut\n", + "\n", + "> MiddleOut (middle_level:str, top_down_method:str)\n", + "\n", + "Middle Out Reconciliation Class.\n", + "\n", + "This method is only available for **strictly hierarchical structures**. It anchors the base predictions \n", + "in a middle level. The levels above the base predictions use the Bottom-Up approach, while the levels \n", + "below use a Top-Down.\n", + "\n", + "**Parameters:**
\n", + "`middle_level`: Middle level.
\n", + "`top_down_method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", + "\n", + "**References:**
\n", + "- [Hyndman, R.J., & Athanasopoulos, G. (2021). \"Forecasting: principles and practice, 3rd edition: \n", + "Chapter 11: Forecasting hierarchical and grouped series.\". OTexts: Melbourne, Australia. OTexts.com/fpp3 \n", + "Accessed on July 2022.](https://otexts.com/fpp3/hierarchical.html)" + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L231){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MiddleOut\n", + "\n", + "> MiddleOut (middle_level:str, top_down_method:str)\n", + "\n", + "Middle Out Reconciliation Class.\n", + "\n", + "This method is only available for **strictly hierarchical structures**. It anchors the base predictions \n", + "in a middle level. The levels above the base predictions use the Bottom-Up approach, while the levels \n", + "below use a Top-Down.\n", + "\n", + "**Parameters:**
\n", + "`middle_level`: Middle level.
\n", + "`top_down_method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", + "\n", + "**References:**
\n", + "- [Hyndman, R.J., & Athanasopoulos, G. (2021). \"Forecasting: principles and practice, 3rd edition: \n", + "Chapter 11: Forecasting hierarchical and grouped series.\". OTexts: Melbourne, Australia. OTexts.com/fpp3 \n", + "Accessed on July 2022.](https://otexts.com/fpp3/hierarchical.html)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(MiddleOut, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L255){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MiddleOut.reconcile\n", + "\n", + "> MiddleOut.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> tags:Dict[str,numpy.ndarray],\n", + "> y_insample:Optional[numpy.ndarray]=None)\n", + "\n", + "Middle Out Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`tags`: Each key is a level and each value its `S` indices.
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Only used for `forecast_proportions`
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the Middle Out approach." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L255){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MiddleOut.reconcile\n", + "\n", + "> MiddleOut.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> tags:Dict[str,numpy.ndarray],\n", + "> y_insample:Optional[numpy.ndarray]=None)\n", + "\n", + "Middle Out Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`tags`: Each key is a level and each value its `S` indices.
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Only used for `forecast_proportions`
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the Middle Out approach." + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(MiddleOut.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -793,7 +1140,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -804,7 +1151,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -830,7 +1177,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -922,12 +1269,10 @@ " # this version follows https://robjhyndman.com/papers/MinT.pdf\n", " tar = np.diag(np.diag(covm))\n", "\n", - " # Protection: constant's correlation set to 0\n", - " corm = cov2corr(covm)\n", + " # Protections: constant's correlation set to 0\n", + " # standardized residuals 0 where residual_std=0\n", + " corm, residual_std = cov2corr(covm, return_std=True)\n", " corm = np.nan_to_num(corm, nan=0.0)\n", - "\n", - " # Protection: standardized residuals 0 where residual_std=0\n", - " residual_std = np.sqrt(np.diag(covm))\n", " xs = np.divide(residuals, residual_std, \n", " out=np.zeros_like(residuals), where=residual_std!=0)\n", "\n", @@ -996,18 +1341,152 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L357){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MinTrace\n", + "\n", + "> MinTrace (method:str, nonnegative:bool=False)\n", + "\n", + "MinTrace Reconciliation Class.\n", + "\n", + "This reconciliation algorithm proposed by Wickramasuriya et al. depends on a generalized least squares estimator \n", + "and an estimator of the covariance matrix of the coherency errors $\\mathbf{W}_{h}$. The Min Trace algorithm \n", + "minimizes the squared errors for the coherent forecasts under an unbiasedness assumption; the solution has a \n", + "closed form.
\n", + "\n", + "$$\\mathbf{P}_{\\text{MinT}}=\\left(\\mathbf{S}^{\\intercal}\\mathbf{W}_{h}\\mathbf{S}\\right)^{-1}\n", + "\\mathbf{S}^{\\intercal}\\mathbf{W}^{-1}_{h}$$\n", + "\n", + "**Parameters:**
\n", + "`method`: str, one of `ols`, `wls_struct`, `wls_var`, `mint_shrink`, `mint_cov`.
\n", + "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", + "\n", + "**References:**
\n", + "- [Wickramasuriya, S. L., Athanasopoulos, G., & Hyndman, R. J. (2019). \"Optimal forecast reconciliation for\n", + "hierarchical and grouped time series through trace minimization\". Journal of the American Statistical Association, \n", + "114 , 804–819. doi:10.1080/01621459.2018.1448825.](https://robjhyndman.com/publications/mint/).\n", + "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", + "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", + "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L357){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MinTrace\n", + "\n", + "> MinTrace (method:str, nonnegative:bool=False)\n", + "\n", + "MinTrace Reconciliation Class.\n", + "\n", + "This reconciliation algorithm proposed by Wickramasuriya et al. depends on a generalized least squares estimator \n", + "and an estimator of the covariance matrix of the coherency errors $\\mathbf{W}_{h}$. The Min Trace algorithm \n", + "minimizes the squared errors for the coherent forecasts under an unbiasedness assumption; the solution has a \n", + "closed form.
\n", + "\n", + "$$\\mathbf{P}_{\\text{MinT}}=\\left(\\mathbf{S}^{\\intercal}\\mathbf{W}_{h}\\mathbf{S}\\right)^{-1}\n", + "\\mathbf{S}^{\\intercal}\\mathbf{W}^{-1}_{h}$$\n", + "\n", + "**Parameters:**
\n", + "`method`: str, one of `ols`, `wls_struct`, `wls_var`, `mint_shrink`, `mint_cov`.
\n", + "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", + "\n", + "**References:**
\n", + "- [Wickramasuriya, S. L., Athanasopoulos, G., & Hyndman, R. J. (2019). \"Optimal forecast reconciliation for\n", + "hierarchical and grouped time series through trace minimization\". Journal of the American Statistical Association, \n", + "114 , 804–819. doi:10.1080/01621459.2018.1448825.](https://robjhyndman.com/publications/mint/).\n", + "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", + "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", + "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(MinTrace, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MinTrace.reconcile\n", + "\n", + "> MinTrace.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> y_insample:Optional[numpy.ndarray]=None,\n", + "> y_hat_insample:Optional[numpy.ndarray]=None,\n", + "> idx_bottom:Optional[List[int]]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "MinTrace Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the MinTrace approach." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### MinTrace.reconcile\n", + "\n", + "> MinTrace.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> y_insample:Optional[numpy.ndarray]=None,\n", + "> y_hat_insample:Optional[numpy.ndarray]=None,\n", + "> idx_bottom:Optional[List[int]]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "MinTrace Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the MinTrace approach." + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(MinTrace.reconcile, title_level=3)" ] @@ -1025,9 +1504,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "for method in ['ols', 'wls_struct', 'wls_var', 'mint_shrink']:\n", @@ -1072,9 +1560,85 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[100. 50. 40. 20. 10.]\n", + " [ 30. 15. 12. 6. 3.]\n", + " [ 70. 35. 28. 14. 7.]\n", + " [ 20. 10. 8. 4. 2.]\n", + " [ 10. 5. 4. 2. 1.]\n", + " [ 30. 15. 12. 6. 3.]\n", + " [ nan nan nan nan 4.]]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/3472991109.py:14: RuntimeWarning: invalid value encountered in true_divide\n", + " corr = cov / np.outer(std_, std_)\n" + ] + }, + { + "data": { + "text/plain": [ + "{'mean': array([[9.99999988, 9.99999988],\n", + " [2.99999986, 2.99999986],\n", + " [7.00000002, 7.00000002],\n", + " [2.00000024, 2.00000024],\n", + " [0.99999962, 0.99999962],\n", + " [3.00000002, 3.00000002],\n", + " [4. , 4. ]])}" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#| hide\n", + "# MinTrace-shr covariance's stress test\n", + "diff_len_y_insample = S @ y_bottom\n", + "diff_len_y_hat_insample = S @ y_hat_bottom_insample\n", + "diff_len_y_insample[-1, :-1] = np.nan\n", + "diff_len_y_hat_insample[-1, :-1] = np.nan\n", + "print(diff_len_y_insample)\n", + "cls_min_trace = MinTrace(method='mint_shrink')\n", + "cls_min_trace(\n", + " S=S,\n", + " y_hat=S @ y_hat_bottom, \n", + " y_insample=diff_len_y_insample,\n", + " y_hat_insample=diff_len_y_hat_insample,\n", + " idx_bottom=idx_bottom\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "#test levels\n", @@ -1121,7 +1685,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -1163,27 +1727,170 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L514){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### OptimalCombination\n", + "\n", + "> OptimalCombination (method:str, nonnegative:bool=False)\n", + "\n", + "Optimal Combination Reconciliation Class.\n", + "\n", + "This reconciliation algorithm was proposed by Hyndman et al. 2011, the method uses generalized least squares \n", + "estimator using the coherency errors covariance matrix. Consider the covariance of the base forecast \n", + "$\\textrm{Var}(\\epsilon_{h}) = \\Sigma_{h}$, the $\\mathbf{P}$ matrix of this method is defined by:\n", + "$$ \\mathbf{P} = \\left(\\mathbf{S}^{\\intercal}\\Sigma_{h}^{\\dagger}\\mathbf{S}\\right)^{-1}\\mathbf{S}^{\\intercal}\\Sigma^{\\dagger}_{h}$$\n", + "where $\\Sigma_{h}^{\\dagger}$ denotes the variance pseudo-inverse. The method was later proven equivalent to \n", + "`MinTrace` variants.\n", + "\n", + "**Parameters:**
\n", + "`method`: str, allowed optimal combination methods: 'ols', 'wls_struct'.
\n", + "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", + "\n", + "**References:**
\n", + "- [Rob J. Hyndman, Roman A. Ahmed, George Athanasopoulos, Han Lin Shang (2010). \"Optimal Combination Forecasts for \n", + "Hierarchical Time Series\".](https://robjhyndman.com/papers/Hierarchical6.pdf).
\n", + "- [Shanika L. Wickramasuriya, George Athanasopoulos and Rob J. Hyndman (2010). \"Optimal Combination Forecasts for \n", + "Hierarchical Time Series\".](https://robjhyndman.com/papers/MinT.pdf).\n", + "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", + "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", + "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L514){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### OptimalCombination\n", + "\n", + "> OptimalCombination (method:str, nonnegative:bool=False)\n", + "\n", + "Optimal Combination Reconciliation Class.\n", + "\n", + "This reconciliation algorithm was proposed by Hyndman et al. 2011, the method uses generalized least squares \n", + "estimator using the coherency errors covariance matrix. Consider the covariance of the base forecast \n", + "$\\textrm{Var}(\\epsilon_{h}) = \\Sigma_{h}$, the $\\mathbf{P}$ matrix of this method is defined by:\n", + "$$ \\mathbf{P} = \\left(\\mathbf{S}^{\\intercal}\\Sigma_{h}^{\\dagger}\\mathbf{S}\\right)^{-1}\\mathbf{S}^{\\intercal}\\Sigma^{\\dagger}_{h}$$\n", + "where $\\Sigma_{h}^{\\dagger}$ denotes the variance pseudo-inverse. The method was later proven equivalent to \n", + "`MinTrace` variants.\n", + "\n", + "**Parameters:**
\n", + "`method`: str, allowed optimal combination methods: 'ols', 'wls_struct'.
\n", + "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", + "\n", + "**References:**
\n", + "- [Rob J. Hyndman, Roman A. Ahmed, George Athanasopoulos, Han Lin Shang (2010). \"Optimal Combination Forecasts for \n", + "Hierarchical Time Series\".](https://robjhyndman.com/papers/Hierarchical6.pdf).
\n", + "- [Shanika L. Wickramasuriya, George Athanasopoulos and Rob J. Hyndman (2010). \"Optimal Combination Forecasts for \n", + "Hierarchical Time Series\".](https://robjhyndman.com/papers/MinT.pdf).\n", + "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", + "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", + "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(OptimalCombination, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### OptimalCombination.reconcile\n", + "\n", + "> OptimalCombination.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> y_insample:Optional[numpy.ndarray]=None, y_\n", + "> hat_insample:Optional[numpy.ndarray]=None,\n", + "> idx_bottom:Optional[List[int]]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "MinTrace Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the MinTrace approach." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### OptimalCombination.reconcile\n", + "\n", + "> OptimalCombination.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> y_insample:Optional[numpy.ndarray]=None, y_\n", + "> hat_insample:Optional[numpy.ndarray]=None,\n", + "> idx_bottom:Optional[List[int]]=None,\n", + "> level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "MinTrace Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the MinTrace approach." + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(OptimalCombination.reconcile, title_level=3, name='OptimalCombination.reconcile')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "for method in ['ols', 'wls_struct']:\n", @@ -1204,9 +1911,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n", + "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", + " warnings.warn('Replacing negative forecasts with zero.')\n" + ] + } + ], "source": [ "#| hide\n", "#test intervals\n", @@ -1250,7 +1968,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1289,7 +2007,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -1390,25 +2108,155 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L581){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### ERM\n", + "\n", + "> ERM (method:str, lambda_reg:float=0.01)\n", + "\n", + "Optimal Combination Reconciliation Class.\n", + "\n", + "The Empirical Risk Minimization reconciliation strategy relaxes the unbiasedness assumptions from\n", + "previous reconciliation methods like MinT and optimizes square errors between the reconciled predictions\n", + "and the validation data to obtain an optimal reconciliation matrix P.\n", + "\n", + "The exact solution for $\\mathbf{P}$ (`method='closed'`) follows the expression:\n", + "$$\\mathbf{P}^{*} = \\left(\\mathbf{S}^{\\intercal}\\mathbf{S}\\right)^{-1}\\mathbf{Y}^{\\intercal}\\hat{\\mathbf{Y}}\\left(\\hat{\\mathbf{Y}}\\hat{\\mathbf{Y}}\\right)^{-1}$$\n", + "\n", + "The alternative Lasso regularized $\\mathbf{P}$ solution (`method='reg_bu'`) is useful when the observations \n", + "of validation data is limited or the exact solution has low numerical stability.\n", + "$$\\mathbf{P}^{*} = \\text{argmin}_{\\mathbf{P}} ||\\mathbf{Y}-\\mathbf{S} \\mathbf{P} \\hat{Y} ||^{2}_{2} + \\lambda ||\\mathbf{P}-\\mathbf{P}_{\\text{BU}}||_{1}$$\n", + "\n", + "**Parameters:**
\n", + "`method`: str, one of `closed`, `reg` and `reg_bu`.
\n", + "`lambda_reg`: float, l1 regularizer for `reg` and `reg_bu`.
\n", + "\n", + "**References:**
\n", + "- [Ben Taieb, S., & Koo, B. (2019). Regularized regression for hierarchical forecasting without \n", + "unbiasedness conditions. In Proceedings of the 25th ACM SIGKDD International Conference on Knowledge \n", + "Discovery & Data Mining KDD '19 (p. 1337{1347). New York, NY, USA: Association for Computing Machinery.](https://doi.org/10.1145/3292500.3330976).
" + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L581){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### ERM\n", + "\n", + "> ERM (method:str, lambda_reg:float=0.01)\n", + "\n", + "Optimal Combination Reconciliation Class.\n", + "\n", + "The Empirical Risk Minimization reconciliation strategy relaxes the unbiasedness assumptions from\n", + "previous reconciliation methods like MinT and optimizes square errors between the reconciled predictions\n", + "and the validation data to obtain an optimal reconciliation matrix P.\n", + "\n", + "The exact solution for $\\mathbf{P}$ (`method='closed'`) follows the expression:\n", + "$$\\mathbf{P}^{*} = \\left(\\mathbf{S}^{\\intercal}\\mathbf{S}\\right)^{-1}\\mathbf{Y}^{\\intercal}\\hat{\\mathbf{Y}}\\left(\\hat{\\mathbf{Y}}\\hat{\\mathbf{Y}}\\right)^{-1}$$\n", + "\n", + "The alternative Lasso regularized $\\mathbf{P}$ solution (`method='reg_bu'`) is useful when the observations \n", + "of validation data is limited or the exact solution has low numerical stability.\n", + "$$\\mathbf{P}^{*} = \\text{argmin}_{\\mathbf{P}} ||\\mathbf{Y}-\\mathbf{S} \\mathbf{P} \\hat{Y} ||^{2}_{2} + \\lambda ||\\mathbf{P}-\\mathbf{P}_{\\text{BU}}||_{1}$$\n", + "\n", + "**Parameters:**
\n", + "`method`: str, one of `closed`, `reg` and `reg_bu`.
\n", + "`lambda_reg`: float, l1 regularizer for `reg` and `reg_bu`.
\n", + "\n", + "**References:**
\n", + "- [Ben Taieb, S., & Koo, B. (2019). Regularized regression for hierarchical forecasting without \n", + "unbiasedness conditions. In Proceedings of the 25th ACM SIGKDD International Conference on Knowledge \n", + "Discovery & Data Mining KDD '19 (p. 1337{1347). New York, NY, USA: Association for Computing Machinery.](https://doi.org/10.1145/3292500.3330976).
" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(ERM, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L611){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### ERM.reconcile\n", + "\n", + "> ERM.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> y_insample:numpy.ndarray, y_hat_insample:numpy.ndarray,\n", + "> idx_bottom:numpy.ndarray, level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "ERM Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`y_insample`: Train values of size (`base`, `insample_size`).
\n", + "`y_hat_insample`: Insample train predictions of size (`base`, `insample_size`).
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the ERM approach." + ], + "text/plain": [ + "---\n", + "\n", + "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L611){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", + "\n", + "### ERM.reconcile\n", + "\n", + "> ERM.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", + "> y_insample:numpy.ndarray, y_hat_insample:numpy.ndarray,\n", + "> idx_bottom:numpy.ndarray, level:Optional[List[int]]=None,\n", + "> sampler:Optional[Callable]=None)\n", + "\n", + "ERM Reconciliation Method.\n", + "\n", + "**Parameters:**
\n", + "`S`: Summing matrix of size (`base`, `bottom`).
\n", + "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", + "`y_insample`: Train values of size (`base`, `insample_size`).
\n", + "`y_hat_insample`: Insample train predictions of size (`base`, `insample_size`).
\n", + "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", + "`level`: float list 0-100, confidence levels for prediction intervals.
\n", + "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", + "\n", + "**Returns:**
\n", + "`y_tilde`: Reconciliated y_hat using the ERM approach." + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show_doc(ERM.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -1429,7 +2277,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -1513,9 +2361,26 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9.14 64-bit", + "display_name": "hierarchicalforecast", "language": "python", - "name": "python3" + "name": "hierarchicalforecast" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b" + } } }, "nbformat": 4, From 50f854a794e8d123d6ec1b41f12b5ba5e226f8a6 Mon Sep 17 00:00:00 2001 From: kdgutier Date: Fri, 28 Oct 2022 09:47:53 -0400 Subject: [PATCH 4/7] Took statsmodels out of settings.ini --- settings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.ini b/settings.ini index e0e87a91..3f4e13ca 100644 --- a/settings.ini +++ b/settings.ini @@ -15,7 +15,7 @@ language = English custom_sidebar = True license = apache2 status = 2 -requirements = numpy numba pandas scikit-learn statsmodels mycolorpy quadprog +requirements = numpy numba pandas scikit-learn mycolorpy quadprog dev_requirements = datasetsforecast statsforecast>=1.0.0 requests matplotlib nbs_path = nbs doc_path = _docs From fd8d65d87f7b2950afd996c8cfcd48f9f08291eb Mon Sep 17 00:00:00 2001 From: kdgutier Date: Fri, 28 Oct 2022 09:50:12 -0400 Subject: [PATCH 5/7] cleaned metadata from examples --- nbs/core.ipynb | 609 +---------- nbs/examples/AustralianDomesticTourism.ipynb | 67 +- nbs/examples/AustralianPrisonPopulation.ipynb | 52 +- nbs/methods.ipynb | 970 ++---------------- 4 files changed, 140 insertions(+), 1558 deletions(-) diff --git a/nbs/core.ipynb b/nbs/core.ipynb index 4fb0672e..fc121c79 100644 --- a/nbs/core.ipynb +++ b/nbs/core.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -27,7 +27,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -45,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -86,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -296,63 +296,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### init\n", - "\n", - "> init (reconcilers:List[Callable])\n", - "\n", - "Hierarchical Reconciliation Class.\n", - "\n", - "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", - "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", - "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", - "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", - "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", - "\n", - "**Parameters:**
\n", - "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", - "\n", - "**References:**
\n", - "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### init\n", - "\n", - "> init (reconcilers:List[Callable])\n", - "\n", - "Hierarchical Reconciliation Class.\n", - "\n", - "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", - "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", - "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", - "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", - "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", - "\n", - "**Parameters:**
\n", - "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", - "\n", - "**References:**
\n", - "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(HierarchicalReconciliation, \n", " name='init', title_level=3)" @@ -360,160 +306,18 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### HierarchicalReconciliation\n", - "\n", - "> HierarchicalReconciliation (reconcilers:List[Callable])\n", - "\n", - "Hierarchical Reconciliation Class.\n", - "\n", - "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", - "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", - "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", - "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", - "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", - "\n", - "**Parameters:**
\n", - "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", - "\n", - "**References:**
\n", - "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L30){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### HierarchicalReconciliation\n", - "\n", - "> HierarchicalReconciliation (reconcilers:List[Callable])\n", - "\n", - "Hierarchical Reconciliation Class.\n", - "\n", - "The `core.HierarchicalReconciliation` class allows you to efficiently fit multiple \n", - "HierarchicaForecast methods for a collection of time series and base predictions stored in \n", - "pandas DataFrames. The `Y_df` dataframe identifies series and datestamps with the unique_id and ds columns while the\n", - "y column denotes the target time series variable. The `Y_h` dataframe stores the base predictions, \n", - "example ([AutoARIMA](https://nixtla.github.io/statsforecast/models.html#autoarima), [ETS](https://nixtla.github.io/statsforecast/models.html#autoets), etc.).\n", - "\n", - "**Parameters:**
\n", - "`reconcilers`: A list of instantiated classes of the [reconciliation methods](https://nixtla.github.io/hierarchicalforecast/methods.html) module .
\n", - "\n", - "**References:**
\n", - "[Rob J. Hyndman and George Athanasopoulos (2018). \"Forecasting principles and practice, Hierarchical and Grouped Series\".](https://otexts.com/fpp3/hierarchical.html)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(HierarchicalReconciliation)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L50){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### reconcile\n", - "\n", - "> reconcile (Y_hat_df:pandas.core.frame.DataFrame,\n", - "> S:pandas.core.frame.DataFrame, tags:Dict[str,numpy.ndarray],\n", - "> Y_df:Optional[pandas.core.frame.DataFrame]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> intervals_method:str='normality')\n", - "\n", - "Hierarchical Reconciliation Method.\n", - "\n", - "The `reconcile` method is analogous to SKLearn `fit` method, it applies different \n", - "reconciliation methods instantiated in the `reconcilers` list. \n", - "\n", - "Most reconciliation methods can be described by the following convenient \n", - "linear algebra notation:\n", - "\n", - "$$\\tilde{\\mathbf{y}}_{[a,b],\\tau} = \\mathbf{S}_{[a,b][b]} \\mathbf{P}_{[b][a,b]} \\hat{\\mathbf{y}}_{[a,b],\\tau}$$\n", - "\n", - "where $a, b$ represent the aggregate and bottom levels, $\\mathbf{S}_{[a,b][b]}$ contains\n", - "the hierarchical aggregation constraints, and $\\mathbf{P}_{[b][a,b]}$ varies across \n", - "reconciliation methods. The reconciled predictions are $\\tilde{\\mathbf{y}}_{[a,b],\\tau}$, and the \n", - "base predictions $\\hat{\\mathbf{y}}_{[a,b],\\tau}$.\n", - "\n", - "**Parameters:**
\n", - "`Y_hat_df`: pd.DataFrame, base forecasts with columns `ds` and models to reconcile indexed by `unique_id`.
\n", - "`Y_df`: pd.DataFrame, training set of base time series with columns `['ds', 'y']` indexed by `unique_id`.\n", - "If a class of `self.reconciles` receives `y_hat_insample`, `Y_df` must include them as columns.
\n", - "`S`: pd.DataFrame with summing matrix of size `(base, bottom)`, see [aggregate method](https://nixtla.github.io/hierarchicalforecast/utils.html#aggregate).
\n", - "`tags`: Each key is a level and its value contains tags associated to that level.
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`intervals_method`: str, method used to calculate prediction intervals, one of `normality`, `bootstrap`, `permbu`.
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: pd.DataFrame, with reconciled predictions." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/core.py#L50){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### reconcile\n", - "\n", - "> reconcile (Y_hat_df:pandas.core.frame.DataFrame,\n", - "> S:pandas.core.frame.DataFrame, tags:Dict[str,numpy.ndarray],\n", - "> Y_df:Optional[pandas.core.frame.DataFrame]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> intervals_method:str='normality')\n", - "\n", - "Hierarchical Reconciliation Method.\n", - "\n", - "The `reconcile` method is analogous to SKLearn `fit` method, it applies different \n", - "reconciliation methods instantiated in the `reconcilers` list. \n", - "\n", - "Most reconciliation methods can be described by the following convenient \n", - "linear algebra notation:\n", - "\n", - "$$\\tilde{\\mathbf{y}}_{[a,b],\\tau} = \\mathbf{S}_{[a,b][b]} \\mathbf{P}_{[b][a,b]} \\hat{\\mathbf{y}}_{[a,b],\\tau}$$\n", - "\n", - "where $a, b$ represent the aggregate and bottom levels, $\\mathbf{S}_{[a,b][b]}$ contains\n", - "the hierarchical aggregation constraints, and $\\mathbf{P}_{[b][a,b]}$ varies across \n", - "reconciliation methods. The reconciled predictions are $\\tilde{\\mathbf{y}}_{[a,b],\\tau}$, and the \n", - "base predictions $\\hat{\\mathbf{y}}_{[a,b],\\tau}$.\n", - "\n", - "**Parameters:**
\n", - "`Y_hat_df`: pd.DataFrame, base forecasts with columns `ds` and models to reconcile indexed by `unique_id`.
\n", - "`Y_df`: pd.DataFrame, training set of base time series with columns `['ds', 'y']` indexed by `unique_id`.\n", - "If a class of `self.reconciles` receives `y_hat_insample`, `Y_df` must include them as columns.
\n", - "`S`: pd.DataFrame with summing matrix of size `(base, bottom)`, see [aggregate method](https://nixtla.github.io/hierarchicalforecast/utils.html#aggregate).
\n", - "`tags`: Each key is a level and its value contains tags associated to that level.
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`intervals_method`: str, method used to calculate prediction intervals, one of `normality`, `bootstrap`, `permbu`.
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: pd.DataFrame, with reconciled predictions." - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(HierarchicalReconciliation.reconcile,\n", " name='reconcile', title_level=3)" @@ -521,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -534,7 +338,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -566,24 +370,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "hier_grouped_df['y_model'] = hier_grouped_df['y']\n", @@ -623,7 +412,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -652,20 +441,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "# test reconcile method without insample\n", @@ -691,7 +469,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -708,24 +486,9 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/Users/kingtzolivares/Desktop/hierarchicalforecast/hierarchicalforecast/methods.py:477: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "# methods should work with\n", @@ -810,7 +573,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -831,7 +594,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -852,163 +615,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
dsyy_modely_model/MinTrace_method-ols
unique_id
Australia/ACT2015 Q1566.135463566.135463566.135463
Australia/ACT2015 Q2516.870343516.870343516.870343
Australia/ACT2015 Q3688.203188688.203188688.203188
Australia/ACT2015 Q4597.245569597.245569597.245569
Australia/ACT2016 Q1625.141634625.141634625.141634
...............
Australia/Western Australia2016 Q42656.3307012656.3307012656.330701
Australia/Western Australia2017 Q12570.9116892570.9116892570.911689
Australia/Western Australia2017 Q22438.4879392438.4879392438.487939
Australia/Western Australia2017 Q32493.9550002493.9550002493.955000
Australia/Western Australia2017 Q42635.7542962635.7542962635.754296
\n", - "

96 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " ds y y_model \\\n", - "unique_id \n", - "Australia/ACT 2015 Q1 566.135463 566.135463 \n", - "Australia/ACT 2015 Q2 516.870343 516.870343 \n", - "Australia/ACT 2015 Q3 688.203188 688.203188 \n", - "Australia/ACT 2015 Q4 597.245569 597.245569 \n", - "Australia/ACT 2016 Q1 625.141634 625.141634 \n", - "... ... ... ... \n", - "Australia/Western Australia 2016 Q4 2656.330701 2656.330701 \n", - "Australia/Western Australia 2017 Q1 2570.911689 2570.911689 \n", - "Australia/Western Australia 2017 Q2 2438.487939 2438.487939 \n", - "Australia/Western Australia 2017 Q3 2493.955000 2493.955000 \n", - "Australia/Western Australia 2017 Q4 2635.754296 2635.754296 \n", - "\n", - " y_model/MinTrace_method-ols \n", - "unique_id \n", - "Australia/ACT 566.135463 \n", - "Australia/ACT 516.870343 \n", - "Australia/ACT 688.203188 \n", - "Australia/ACT 597.245569 \n", - "Australia/ACT 625.141634 \n", - "... ... \n", - "Australia/Western Australia 2656.330701 \n", - "Australia/Western Australia 2570.911689 \n", - "Australia/Western Australia 2438.487939 \n", - "Australia/Western Australia 2493.955000 \n", - "Australia/Western Australia 2635.754296 \n", - "\n", - "[96 rows x 4 columns]" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "#| hide\n", "reconciled.loc[tags_grouped['Country/State']]" @@ -1016,7 +625,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1038,7 +647,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1062,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1106,163 +715,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
dsNaiveNaive/BottomUpNaive/MinTrace_method-ols
unique_id
Australia2016-12-3126347.60156226347.59960926347.601460
Australia2017-03-3126347.60156226347.59960926347.601460
Australia/ACT2016-12-31667.214111667.214111667.214156
Australia/ACT2017-03-31667.214111667.214111667.214156
Australia/ACT/Business2016-12-31186.399078186.399078186.399069
...............
Australia/Western Australia/Holiday2017-03-31982.752563982.752563982.752600
Australia/Western Australia/Other2016-12-31121.279541121.279541121.279550
Australia/Western Australia/Other2017-03-31121.279541121.279541121.279550
Australia/Western Australia/Visiting2016-12-31787.030396787.030396787.030481
Australia/Western Australia/Visiting2017-03-31787.030396787.030396787.030481
\n", - "

850 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " ds Naive Naive/BottomUp \\\n", - "unique_id \n", - "Australia 2016-12-31 26347.601562 26347.599609 \n", - "Australia 2017-03-31 26347.601562 26347.599609 \n", - "Australia/ACT 2016-12-31 667.214111 667.214111 \n", - "Australia/ACT 2017-03-31 667.214111 667.214111 \n", - "Australia/ACT/Business 2016-12-31 186.399078 186.399078 \n", - "... ... ... ... \n", - "Australia/Western Australia/Holiday 2017-03-31 982.752563 982.752563 \n", - "Australia/Western Australia/Other 2016-12-31 121.279541 121.279541 \n", - "Australia/Western Australia/Other 2017-03-31 121.279541 121.279541 \n", - "Australia/Western Australia/Visiting 2016-12-31 787.030396 787.030396 \n", - "Australia/Western Australia/Visiting 2017-03-31 787.030396 787.030396 \n", - "\n", - " Naive/MinTrace_method-ols \n", - "unique_id \n", - "Australia 26347.601460 \n", - "Australia 26347.601460 \n", - "Australia/ACT 667.214156 \n", - "Australia/ACT 667.214156 \n", - "Australia/ACT/Business 186.399069 \n", - "... ... \n", - "Australia/Western Australia/Holiday 982.752600 \n", - "Australia/Western Australia/Other 121.279550 \n", - "Australia/Western Australia/Other 121.279550 \n", - "Australia/Western Australia/Visiting 787.030481 \n", - "Australia/Western Australia/Visiting 787.030481 \n", - "\n", - "[850 rows x 4 columns]" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "#| eval: false\n", "import numpy as np\n", diff --git a/nbs/examples/AustralianDomesticTourism.ipynb b/nbs/examples/AustralianDomesticTourism.ipynb index cd2275e2..6f4d7f03 100644 --- a/nbs/examples/AustralianDomesticTourism.ipynb +++ b/nbs/examples/AustralianDomesticTourism.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -159,7 +159,7 @@ "4 Australia Adelaide South Australia Business 1999-01-01 137.448533" ] }, - "execution_count": 20, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -183,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -206,7 +206,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -216,7 +216,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -289,7 +289,7 @@ "4 Australia 1999-01-01 22087.353380" ] }, - "execution_count": 23, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -300,7 +300,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -413,7 +413,7 @@ "Australia/Queensland 0.0 " ] }, - "execution_count": 24, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -424,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -434,7 +434,7 @@ " 'Australia/Visiting'], dtype=object)" ] }, - "execution_count": 25, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -454,7 +454,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -464,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -474,7 +474,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -495,7 +495,7 @@ "Length: 425, dtype: int64" ] }, - "execution_count": 28, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -515,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -537,7 +537,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -559,7 +559,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -661,7 +661,7 @@ "Australia 25556.667615 25901.382401 " ] }, - "execution_count": 31, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -681,7 +681,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -731,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -834,7 +834,7 @@ "All rmse 45.19 54.95 44.59 42.71" ] }, - "execution_count": 33, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -855,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -958,7 +958,7 @@ "All mase 1.03 1.08 0.98 1.02" ] }, - "execution_count": 34, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -996,23 +996,6 @@ "display_name": "hierarchicalforecast", "language": "python", "name": "hierarchicalforecast" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - }, - "vscode": { - "interpreter": { - "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b" - } } }, "nbformat": 4, diff --git a/nbs/examples/AustralianPrisonPopulation.ipynb b/nbs/examples/AustralianPrisonPopulation.ipynb index 49ff6152..938a2c76 100644 --- a/nbs/examples/AustralianPrisonPopulation.ipynb +++ b/nbs/examples/AustralianPrisonPopulation.ipynb @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -165,7 +165,7 @@ "4 Australia ACT Male Remanded ATSI 2005-03-01 7" ] }, - "execution_count": 3, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -188,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -210,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -220,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -293,7 +293,7 @@ "4 Australia 2006-03-01 24.524" ] }, - "execution_count": 6, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -304,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -403,7 +403,7 @@ "Australia/QLD 0.0 " ] }, - "execution_count": 7, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -414,7 +414,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -445,7 +445,7 @@ " dtype=object)}" ] }, - "execution_count": 8, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -465,7 +465,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -475,7 +475,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -494,7 +494,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -516,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -537,7 +537,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -624,7 +624,7 @@ "Australia 2016-01-01 36.045437 36.400002 36.244702" ] }, - "execution_count": 13, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -644,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -731,7 +731,7 @@ "All series 2.00 2.00 2.00" ] }, - "execution_count": 14, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -790,18 +790,6 @@ "display_name": "hierarchicalforecast", "language": "python", "name": "hierarchicalforecast" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" } }, "nbformat": 4, diff --git a/nbs/methods.ipynb b/nbs/methods.ipynb index 761d2f93..fc32a3b8 100644 --- a/nbs/methods.ipynb +++ b/nbs/methods.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -164,133 +164,25 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L53){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### BottomUp\n", - "\n", - "> BottomUp ()\n", - "\n", - "Bottom Up Reconciliation Class.\n", - "The most basic hierarchical reconciliation is performed using an Bottom-Up strategy. It was proposed for \n", - "the first time by Orcutt in 1968.\n", - "The corresponding hierarchical \"projection\" matrix is defined as:\n", - "$$\\mathbf{P}_{\\text{BU}} = [\\mathbf{0}_{\\mathrm{[b],[a]}}\\;|\\;\\mathbf{I}_{\\mathrm{[b][b]}}]$$\n", - "\n", - "**Parameters:**
\n", - "None\n", - "\n", - "**References:**
\n", - "- [Orcutt, G.H., Watts, H.W., & Edwards, J.B.(1968). \"Data aggregation and information loss\". The American \n", - "Economic Review, 58 , 773{787)](http://www.jstor.org/stable/1815532)." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L53){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### BottomUp\n", - "\n", - "> BottomUp ()\n", - "\n", - "Bottom Up Reconciliation Class.\n", - "The most basic hierarchical reconciliation is performed using an Bottom-Up strategy. It was proposed for \n", - "the first time by Orcutt in 1968.\n", - "The corresponding hierarchical \"projection\" matrix is defined as:\n", - "$$\\mathbf{P}_{\\text{BU}} = [\\mathbf{0}_{\\mathrm{[b],[a]}}\\;|\\;\\mathbf{I}_{\\mathrm{[b][b]}}]$$\n", - "\n", - "**Parameters:**
\n", - "None\n", - "\n", - "**References:**
\n", - "- [Orcutt, G.H., Watts, H.W., & Edwards, J.B.(1968). \"Data aggregation and information loss\". The American \n", - "Economic Review, 58 , 773{787)](http://www.jstor.org/stable/1815532)." - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(BottomUp, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L69){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### BottomUp.reconcile\n", - "\n", - "> BottomUp.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> idx_bottom:numpy.ndarray,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "Bottom Up Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the Bottom Up approach." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L69){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### BottomUp.reconcile\n", - "\n", - "> BottomUp.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> idx_bottom:numpy.ndarray,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "Bottom Up Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the Bottom Up approach." - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(BottomUp.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -316,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -335,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -367,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -381,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -400,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -435,7 +327,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -460,7 +352,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -488,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -520,7 +412,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -544,7 +436,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -631,147 +523,25 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L151){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### TopDown\n", - "\n", - "> TopDown (method:str)\n", - "\n", - "Top Down Reconciliation Class.\n", - "\n", - "The Top Down hierarchical reconciliation method, distributes the total aggregate predictions and decomposes \n", - "it down the hierarchy using proportions $\\mathbf{p}_{\\mathrm{[b]}}$ that can be actual historical values \n", - "or estimated.\n", - "\n", - "$$\\mathbf{P}=[\\mathbf{p}_{\\mathrm{[b]}}\\;|\\;\\mathbf{0}_{\\mathrm{[b][a,b\\;-1]}}]$$\n", - "**Parameters:**
\n", - "`method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", - "\n", - "**References:**
\n", - "- [CW. Gross (1990). \"Disaggregation methods to expedite product line forecasting\". Journal of Forecasting, 9 , 233–254. \n", - "doi:10.1002/for.3980090304](https://onlinelibrary.wiley.com/doi/abs/10.1002/for.3980090304).
\n", - "- [G. Fliedner (1999). \"An investigation of aggregate variable time series forecast strategies with specific subaggregate \n", - "time series statistical correlation\". Computers and Operations Research, 26 , 1133–1149. \n", - "doi:10.1016/S0305-0548(99)00017-9](https://doi.org/10.1016/S0305-0548(99)00017-9)." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L151){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### TopDown\n", - "\n", - "> TopDown (method:str)\n", - "\n", - "Top Down Reconciliation Class.\n", - "\n", - "The Top Down hierarchical reconciliation method, distributes the total aggregate predictions and decomposes \n", - "it down the hierarchy using proportions $\\mathbf{p}_{\\mathrm{[b]}}$ that can be actual historical values \n", - "or estimated.\n", - "\n", - "$$\\mathbf{P}=[\\mathbf{p}_{\\mathrm{[b]}}\\;|\\;\\mathbf{0}_{\\mathrm{[b][a,b\\;-1]}}]$$\n", - "**Parameters:**
\n", - "`method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", - "\n", - "**References:**
\n", - "- [CW. Gross (1990). \"Disaggregation methods to expedite product line forecasting\". Journal of Forecasting, 9 , 233–254. \n", - "doi:10.1002/for.3980090304](https://onlinelibrary.wiley.com/doi/abs/10.1002/for.3980090304).
\n", - "- [G. Fliedner (1999). \"An investigation of aggregate variable time series forecast strategies with specific subaggregate \n", - "time series statistical correlation\". Computers and Operations Research, 26 , 1133–1149. \n", - "doi:10.1016/S0305-0548(99)00017-9](https://doi.org/10.1016/S0305-0548(99)00017-9)." - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(TopDown, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L174){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### TopDown.reconcile\n", - "\n", - "> TopDown.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> tags:Dict[str,numpy.ndarray],\n", - "> y_insample:Optional[numpy.ndarray]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[numpy.ndarray]=None)\n", - "\n", - "Top Down Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`tags`: Each key is a level and each value its `S` indices.
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Optional for `forecast_proportions` method.
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the Top Down approach." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L174){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### TopDown.reconcile\n", - "\n", - "> TopDown.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> tags:Dict[str,numpy.ndarray],\n", - "> y_insample:Optional[numpy.ndarray]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[numpy.ndarray]=None)\n", - "\n", - "Top Down Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`tags`: Each key is a level and each value its `S` indices.
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Optional for `forecast_proportions` method.
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the Top Down approach." - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(TopDown.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -808,18 +578,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/2462090121.py:56: UserWarning: Prediction intervals not implement for `forecast_proportions`\n", - " warnings.warn('Prediction intervals not implement for `forecast_proportions`')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "# test_levels\n", @@ -860,7 +621,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -970,133 +731,25 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L231){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MiddleOut\n", - "\n", - "> MiddleOut (middle_level:str, top_down_method:str)\n", - "\n", - "Middle Out Reconciliation Class.\n", - "\n", - "This method is only available for **strictly hierarchical structures**. It anchors the base predictions \n", - "in a middle level. The levels above the base predictions use the Bottom-Up approach, while the levels \n", - "below use a Top-Down.\n", - "\n", - "**Parameters:**
\n", - "`middle_level`: Middle level.
\n", - "`top_down_method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", - "\n", - "**References:**
\n", - "- [Hyndman, R.J., & Athanasopoulos, G. (2021). \"Forecasting: principles and practice, 3rd edition: \n", - "Chapter 11: Forecasting hierarchical and grouped series.\". OTexts: Melbourne, Australia. OTexts.com/fpp3 \n", - "Accessed on July 2022.](https://otexts.com/fpp3/hierarchical.html)" - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L231){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MiddleOut\n", - "\n", - "> MiddleOut (middle_level:str, top_down_method:str)\n", - "\n", - "Middle Out Reconciliation Class.\n", - "\n", - "This method is only available for **strictly hierarchical structures**. It anchors the base predictions \n", - "in a middle level. The levels above the base predictions use the Bottom-Up approach, while the levels \n", - "below use a Top-Down.\n", - "\n", - "**Parameters:**
\n", - "`middle_level`: Middle level.
\n", - "`top_down_method`: One of `forecast_proportions`, `average_proportions` and `proportion_averages`.
\n", - "\n", - "**References:**
\n", - "- [Hyndman, R.J., & Athanasopoulos, G. (2021). \"Forecasting: principles and practice, 3rd edition: \n", - "Chapter 11: Forecasting hierarchical and grouped series.\". OTexts: Melbourne, Australia. OTexts.com/fpp3 \n", - "Accessed on July 2022.](https://otexts.com/fpp3/hierarchical.html)" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(MiddleOut, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L255){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MiddleOut.reconcile\n", - "\n", - "> MiddleOut.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> tags:Dict[str,numpy.ndarray],\n", - "> y_insample:Optional[numpy.ndarray]=None)\n", - "\n", - "Middle Out Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`tags`: Each key is a level and each value its `S` indices.
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Only used for `forecast_proportions`
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the Middle Out approach." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L255){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MiddleOut.reconcile\n", - "\n", - "> MiddleOut.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> tags:Dict[str,numpy.ndarray],\n", - "> y_insample:Optional[numpy.ndarray]=None)\n", - "\n", - "Middle Out Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`tags`: Each key is a level and each value its `S` indices.
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Only used for `forecast_proportions`
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the Middle Out approach." - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(MiddleOut.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1140,7 +793,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1151,7 +804,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1177,7 +830,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1341,152 +994,18 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L357){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MinTrace\n", - "\n", - "> MinTrace (method:str, nonnegative:bool=False)\n", - "\n", - "MinTrace Reconciliation Class.\n", - "\n", - "This reconciliation algorithm proposed by Wickramasuriya et al. depends on a generalized least squares estimator \n", - "and an estimator of the covariance matrix of the coherency errors $\\mathbf{W}_{h}$. The Min Trace algorithm \n", - "minimizes the squared errors for the coherent forecasts under an unbiasedness assumption; the solution has a \n", - "closed form.
\n", - "\n", - "$$\\mathbf{P}_{\\text{MinT}}=\\left(\\mathbf{S}^{\\intercal}\\mathbf{W}_{h}\\mathbf{S}\\right)^{-1}\n", - "\\mathbf{S}^{\\intercal}\\mathbf{W}^{-1}_{h}$$\n", - "\n", - "**Parameters:**
\n", - "`method`: str, one of `ols`, `wls_struct`, `wls_var`, `mint_shrink`, `mint_cov`.
\n", - "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", - "\n", - "**References:**
\n", - "- [Wickramasuriya, S. L., Athanasopoulos, G., & Hyndman, R. J. (2019). \"Optimal forecast reconciliation for\n", - "hierarchical and grouped time series through trace minimization\". Journal of the American Statistical Association, \n", - "114 , 804–819. doi:10.1080/01621459.2018.1448825.](https://robjhyndman.com/publications/mint/).\n", - "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", - "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", - "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L357){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MinTrace\n", - "\n", - "> MinTrace (method:str, nonnegative:bool=False)\n", - "\n", - "MinTrace Reconciliation Class.\n", - "\n", - "This reconciliation algorithm proposed by Wickramasuriya et al. depends on a generalized least squares estimator \n", - "and an estimator of the covariance matrix of the coherency errors $\\mathbf{W}_{h}$. The Min Trace algorithm \n", - "minimizes the squared errors for the coherent forecasts under an unbiasedness assumption; the solution has a \n", - "closed form.
\n", - "\n", - "$$\\mathbf{P}_{\\text{MinT}}=\\left(\\mathbf{S}^{\\intercal}\\mathbf{W}_{h}\\mathbf{S}\\right)^{-1}\n", - "\\mathbf{S}^{\\intercal}\\mathbf{W}^{-1}_{h}$$\n", - "\n", - "**Parameters:**
\n", - "`method`: str, one of `ols`, `wls_struct`, `wls_var`, `mint_shrink`, `mint_cov`.
\n", - "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", - "\n", - "**References:**
\n", - "- [Wickramasuriya, S. L., Athanasopoulos, G., & Hyndman, R. J. (2019). \"Optimal forecast reconciliation for\n", - "hierarchical and grouped time series through trace minimization\". Journal of the American Statistical Association, \n", - "114 , 804–819. doi:10.1080/01621459.2018.1448825.](https://robjhyndman.com/publications/mint/).\n", - "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", - "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", - "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(MinTrace, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MinTrace.reconcile\n", - "\n", - "> MinTrace.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> y_insample:Optional[numpy.ndarray]=None,\n", - "> y_hat_insample:Optional[numpy.ndarray]=None,\n", - "> idx_bottom:Optional[List[int]]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "MinTrace Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the MinTrace approach." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### MinTrace.reconcile\n", - "\n", - "> MinTrace.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> y_insample:Optional[numpy.ndarray]=None,\n", - "> y_hat_insample:Optional[numpy.ndarray]=None,\n", - "> idx_bottom:Optional[List[int]]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "MinTrace Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the MinTrace approach." - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(MinTrace.reconcile, title_level=3)" ] @@ -1504,18 +1023,9 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "for method in ['ols', 'wls_struct', 'wls_var', 'mint_shrink']:\n", @@ -1560,47 +1070,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[[100. 50. 40. 20. 10.]\n", - " [ 30. 15. 12. 6. 3.]\n", - " [ 70. 35. 28. 14. 7.]\n", - " [ 20. 10. 8. 4. 2.]\n", - " [ 10. 5. 4. 2. 1.]\n", - " [ 30. 15. 12. 6. 3.]\n", - " [ nan nan nan nan 4.]]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/3472991109.py:14: RuntimeWarning: invalid value encountered in true_divide\n", - " corr = cov / np.outer(std_, std_)\n" - ] - }, - { - "data": { - "text/plain": [ - "{'mean': array([[9.99999988, 9.99999988],\n", - " [2.99999986, 2.99999986],\n", - " [7.00000002, 7.00000002],\n", - " [2.00000024, 2.00000024],\n", - " [0.99999962, 0.99999962],\n", - " [3.00000002, 3.00000002],\n", - " [4. , 4. ]])}" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "#| hide\n", "# MinTrace-shr covariance's stress test\n", @@ -1621,24 +1093,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "#test levels\n", @@ -1685,7 +1142,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1727,170 +1184,27 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L514){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### OptimalCombination\n", - "\n", - "> OptimalCombination (method:str, nonnegative:bool=False)\n", - "\n", - "Optimal Combination Reconciliation Class.\n", - "\n", - "This reconciliation algorithm was proposed by Hyndman et al. 2011, the method uses generalized least squares \n", - "estimator using the coherency errors covariance matrix. Consider the covariance of the base forecast \n", - "$\\textrm{Var}(\\epsilon_{h}) = \\Sigma_{h}$, the $\\mathbf{P}$ matrix of this method is defined by:\n", - "$$ \\mathbf{P} = \\left(\\mathbf{S}^{\\intercal}\\Sigma_{h}^{\\dagger}\\mathbf{S}\\right)^{-1}\\mathbf{S}^{\\intercal}\\Sigma^{\\dagger}_{h}$$\n", - "where $\\Sigma_{h}^{\\dagger}$ denotes the variance pseudo-inverse. The method was later proven equivalent to \n", - "`MinTrace` variants.\n", - "\n", - "**Parameters:**
\n", - "`method`: str, allowed optimal combination methods: 'ols', 'wls_struct'.
\n", - "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", - "\n", - "**References:**
\n", - "- [Rob J. Hyndman, Roman A. Ahmed, George Athanasopoulos, Han Lin Shang (2010). \"Optimal Combination Forecasts for \n", - "Hierarchical Time Series\".](https://robjhyndman.com/papers/Hierarchical6.pdf).
\n", - "- [Shanika L. Wickramasuriya, George Athanasopoulos and Rob J. Hyndman (2010). \"Optimal Combination Forecasts for \n", - "Hierarchical Time Series\".](https://robjhyndman.com/papers/MinT.pdf).\n", - "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", - "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", - "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L514){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### OptimalCombination\n", - "\n", - "> OptimalCombination (method:str, nonnegative:bool=False)\n", - "\n", - "Optimal Combination Reconciliation Class.\n", - "\n", - "This reconciliation algorithm was proposed by Hyndman et al. 2011, the method uses generalized least squares \n", - "estimator using the coherency errors covariance matrix. Consider the covariance of the base forecast \n", - "$\\textrm{Var}(\\epsilon_{h}) = \\Sigma_{h}$, the $\\mathbf{P}$ matrix of this method is defined by:\n", - "$$ \\mathbf{P} = \\left(\\mathbf{S}^{\\intercal}\\Sigma_{h}^{\\dagger}\\mathbf{S}\\right)^{-1}\\mathbf{S}^{\\intercal}\\Sigma^{\\dagger}_{h}$$\n", - "where $\\Sigma_{h}^{\\dagger}$ denotes the variance pseudo-inverse. The method was later proven equivalent to \n", - "`MinTrace` variants.\n", - "\n", - "**Parameters:**
\n", - "`method`: str, allowed optimal combination methods: 'ols', 'wls_struct'.
\n", - "`nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", - "\n", - "**References:**
\n", - "- [Rob J. Hyndman, Roman A. Ahmed, George Athanasopoulos, Han Lin Shang (2010). \"Optimal Combination Forecasts for \n", - "Hierarchical Time Series\".](https://robjhyndman.com/papers/Hierarchical6.pdf).
\n", - "- [Shanika L. Wickramasuriya, George Athanasopoulos and Rob J. Hyndman (2010). \"Optimal Combination Forecasts for \n", - "Hierarchical Time Series\".](https://robjhyndman.com/papers/MinT.pdf).\n", - "- [Wickramasuriya, S.L., Turlach, B.A. & Hyndman, R.J. (2020). \"Optimal non-negative\n", - "forecast reconciliation\". Stat Comput 30, 1167–1182, \n", - "https://doi.org/10.1007/s11222-020-09930-0](https://robjhyndman.com/publications/nnmint/)." - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(OptimalCombination, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### OptimalCombination.reconcile\n", - "\n", - "> OptimalCombination.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> y_insample:Optional[numpy.ndarray]=None, y_\n", - "> hat_insample:Optional[numpy.ndarray]=None,\n", - "> idx_bottom:Optional[List[int]]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "MinTrace Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the MinTrace approach." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L387){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### OptimalCombination.reconcile\n", - "\n", - "> OptimalCombination.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> y_insample:Optional[numpy.ndarray]=None, y_\n", - "> hat_insample:Optional[numpy.ndarray]=None,\n", - "> idx_bottom:Optional[List[int]]=None,\n", - "> level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "MinTrace Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`y_insample`: Insample values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`y_hat_insample`: Insample fitted values of size (`base`, `insample_size`). Only used by `wls_var`, `mint_cov`, `mint_shrink`
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the MinTrace approach." - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(OptimalCombination.reconcile, title_level=3, name='OptimalCombination.reconcile')" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "for method in ['ols', 'wls_struct']:\n", @@ -1911,20 +1225,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n", - "/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_89646/574583412.py:122: UserWarning: Replacing negative forecasts with zero.\n", - " warnings.warn('Replacing negative forecasts with zero.')\n" - ] - } - ], + "outputs": [], "source": [ "#| hide\n", "#test intervals\n", @@ -1968,7 +1271,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2007,7 +1310,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2108,155 +1411,25 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L581){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### ERM\n", - "\n", - "> ERM (method:str, lambda_reg:float=0.01)\n", - "\n", - "Optimal Combination Reconciliation Class.\n", - "\n", - "The Empirical Risk Minimization reconciliation strategy relaxes the unbiasedness assumptions from\n", - "previous reconciliation methods like MinT and optimizes square errors between the reconciled predictions\n", - "and the validation data to obtain an optimal reconciliation matrix P.\n", - "\n", - "The exact solution for $\\mathbf{P}$ (`method='closed'`) follows the expression:\n", - "$$\\mathbf{P}^{*} = \\left(\\mathbf{S}^{\\intercal}\\mathbf{S}\\right)^{-1}\\mathbf{Y}^{\\intercal}\\hat{\\mathbf{Y}}\\left(\\hat{\\mathbf{Y}}\\hat{\\mathbf{Y}}\\right)^{-1}$$\n", - "\n", - "The alternative Lasso regularized $\\mathbf{P}$ solution (`method='reg_bu'`) is useful when the observations \n", - "of validation data is limited or the exact solution has low numerical stability.\n", - "$$\\mathbf{P}^{*} = \\text{argmin}_{\\mathbf{P}} ||\\mathbf{Y}-\\mathbf{S} \\mathbf{P} \\hat{Y} ||^{2}_{2} + \\lambda ||\\mathbf{P}-\\mathbf{P}_{\\text{BU}}||_{1}$$\n", - "\n", - "**Parameters:**
\n", - "`method`: str, one of `closed`, `reg` and `reg_bu`.
\n", - "`lambda_reg`: float, l1 regularizer for `reg` and `reg_bu`.
\n", - "\n", - "**References:**
\n", - "- [Ben Taieb, S., & Koo, B. (2019). Regularized regression for hierarchical forecasting without \n", - "unbiasedness conditions. In Proceedings of the 25th ACM SIGKDD International Conference on Knowledge \n", - "Discovery & Data Mining KDD '19 (p. 1337{1347). New York, NY, USA: Association for Computing Machinery.](https://doi.org/10.1145/3292500.3330976).
" - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L581){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### ERM\n", - "\n", - "> ERM (method:str, lambda_reg:float=0.01)\n", - "\n", - "Optimal Combination Reconciliation Class.\n", - "\n", - "The Empirical Risk Minimization reconciliation strategy relaxes the unbiasedness assumptions from\n", - "previous reconciliation methods like MinT and optimizes square errors between the reconciled predictions\n", - "and the validation data to obtain an optimal reconciliation matrix P.\n", - "\n", - "The exact solution for $\\mathbf{P}$ (`method='closed'`) follows the expression:\n", - "$$\\mathbf{P}^{*} = \\left(\\mathbf{S}^{\\intercal}\\mathbf{S}\\right)^{-1}\\mathbf{Y}^{\\intercal}\\hat{\\mathbf{Y}}\\left(\\hat{\\mathbf{Y}}\\hat{\\mathbf{Y}}\\right)^{-1}$$\n", - "\n", - "The alternative Lasso regularized $\\mathbf{P}$ solution (`method='reg_bu'`) is useful when the observations \n", - "of validation data is limited or the exact solution has low numerical stability.\n", - "$$\\mathbf{P}^{*} = \\text{argmin}_{\\mathbf{P}} ||\\mathbf{Y}-\\mathbf{S} \\mathbf{P} \\hat{Y} ||^{2}_{2} + \\lambda ||\\mathbf{P}-\\mathbf{P}_{\\text{BU}}||_{1}$$\n", - "\n", - "**Parameters:**
\n", - "`method`: str, one of `closed`, `reg` and `reg_bu`.
\n", - "`lambda_reg`: float, l1 regularizer for `reg` and `reg_bu`.
\n", - "\n", - "**References:**
\n", - "- [Ben Taieb, S., & Koo, B. (2019). Regularized regression for hierarchical forecasting without \n", - "unbiasedness conditions. In Proceedings of the 25th ACM SIGKDD International Conference on Knowledge \n", - "Discovery & Data Mining KDD '19 (p. 1337{1347). New York, NY, USA: Association for Computing Machinery.](https://doi.org/10.1145/3292500.3330976).
" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(ERM, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L611){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### ERM.reconcile\n", - "\n", - "> ERM.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> y_insample:numpy.ndarray, y_hat_insample:numpy.ndarray,\n", - "> idx_bottom:numpy.ndarray, level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "ERM Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`y_insample`: Train values of size (`base`, `insample_size`).
\n", - "`y_hat_insample`: Insample train predictions of size (`base`, `insample_size`).
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the ERM approach." - ], - "text/plain": [ - "---\n", - "\n", - "[source](https://github.com/Nixtla/hierarchicalforecast/tree/main/blob/main/hierarchicalforecast/methods.py#L611){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### ERM.reconcile\n", - "\n", - "> ERM.reconcile (S:numpy.ndarray, y_hat:numpy.ndarray,\n", - "> y_insample:numpy.ndarray, y_hat_insample:numpy.ndarray,\n", - "> idx_bottom:numpy.ndarray, level:Optional[List[int]]=None,\n", - "> sampler:Optional[Callable]=None)\n", - "\n", - "ERM Reconciliation Method.\n", - "\n", - "**Parameters:**
\n", - "`S`: Summing matrix of size (`base`, `bottom`).
\n", - "`y_hat`: Forecast values of size (`base`, `horizon`).
\n", - "`y_insample`: Train values of size (`base`, `insample_size`).
\n", - "`y_hat_insample`: Insample train predictions of size (`base`, `insample_size`).
\n", - "`idx_bottom`: Indices corresponding to the bottom level of `S`, size (`bottom`).
\n", - "`level`: float list 0-100, confidence levels for prediction intervals.
\n", - "`sampler`: Sampler for prediction intevals, one of Normality(), Bootstrap(), PERMBU().
\n", - "\n", - "**Returns:**
\n", - "`y_tilde`: Reconciliated y_hat using the ERM approach." - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_doc(ERM.reconcile, title_level=3)" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2277,7 +1450,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2364,23 +1537,6 @@ "display_name": "hierarchicalforecast", "language": "python", "name": "hierarchicalforecast" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - }, - "vscode": { - "interpreter": { - "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b" - } } }, "nbformat": 4, From 9d13e75c2950750d6f2577bf008f7651d90f0304 Mon Sep 17 00:00:00 2001 From: kdgutier Date: Fri, 28 Oct 2022 09:59:01 -0400 Subject: [PATCH 6/7] Eliminated statsmodels dependency from probabilistic_methods --- hierarchicalforecast/probabilistic_methods.py | 4 ++-- nbs/probabilistic_methods.ipynb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hierarchicalforecast/probabilistic_methods.py b/hierarchicalforecast/probabilistic_methods.py index f7be3e1d..a0626e3d 100644 --- a/hierarchicalforecast/probabilistic_methods.py +++ b/hierarchicalforecast/probabilistic_methods.py @@ -9,9 +9,9 @@ import numpy as np from scipy.stats import norm from sklearn.preprocessing import OneHotEncoder -from statsmodels.stats.moment_helpers import cov2corr -from .methods import is_strictly_hierarchical + +from .methods import is_strictly_hierarchical, cov2corr # %% ../nbs/probabilistic_methods.ipynb 6 class Normality: diff --git a/nbs/probabilistic_methods.ipynb b/nbs/probabilistic_methods.ipynb index aaa89091..51306e4a 100644 --- a/nbs/probabilistic_methods.ipynb +++ b/nbs/probabilistic_methods.ipynb @@ -38,9 +38,9 @@ "import numpy as np\n", "from scipy.stats import norm\n", "from sklearn.preprocessing import OneHotEncoder\n", - "from statsmodels.stats.moment_helpers import cov2corr\n", "\n", - "from hierarchicalforecast.methods import is_strictly_hierarchical" + "\n", + "from hierarchicalforecast.methods import is_strictly_hierarchical, cov2corr" ] }, { @@ -474,9 +474,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "hierarchicalforecast", "language": "python", - "name": "python3" + "name": "hierarchicalforecast" } }, "nbformat": 4, From 03a9e786348d2cc5d4aac3ad0a30976acfcffe6a Mon Sep 17 00:00:00 2001 From: kdgutier Date: Fri, 28 Oct 2022 11:14:32 -0400 Subject: [PATCH 7/7] minor numeric increase of MinTrace's ridge and added parameter --- hierarchicalforecast/methods.py | 8 ++++++-- nbs/methods.ipynb | 10 +++++++--- nbs/probabilistic_methods.ipynb | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hierarchicalforecast/methods.py b/hierarchicalforecast/methods.py index 0b896884..d0492767 100644 --- a/hierarchicalforecast/methods.py +++ b/hierarchicalforecast/methods.py @@ -368,6 +368,7 @@ class MinTrace: **Parameters:**
`method`: str, one of `ols`, `wls_struct`, `wls_var`, `mint_shrink`, `mint_cov`.
`nonnegative`: bool, reconciled forecasts should be nonnegative?
+ `mint_shr_ridge`: float, ridge numeric protection to MinTrace-shr covariance estimator.
**References:**
- [Wickramasuriya, S. L., Athanasopoulos, G., & Hyndman, R. J. (2019). \"Optimal forecast reconciliation for @@ -379,10 +380,13 @@ class MinTrace: """ def __init__(self, method: str, - nonnegative: bool = False): + nonnegative: bool = False, + mint_shr_ridge: Optional[float] = 2e-8): self.method = method self.nonnegative = nonnegative self.insample = method in ['wls_var', 'mint_cov', 'mint_shrink'] + if method == 'mint_shrink': + self.mint_shr_ridge = mint_shr_ridge def reconcile(self, S: np.ndarray, @@ -460,7 +464,7 @@ def reconcile(self, lmd = max(min(lmd, 1), 0) # Protection: final ridge diagonal protection - W = (lmd * tar + (1 - lmd) * covm) + 1e-8 + W = (lmd * tar + (1 - lmd) * covm) + self.mint_shr_ridge else: raise ValueError(f'Unkown reconciliation method {self.method}') diff --git a/nbs/methods.ipynb b/nbs/methods.ipynb index fc32a3b8..9666132f 100644 --- a/nbs/methods.ipynb +++ b/nbs/methods.ipynb @@ -849,6 +849,7 @@ " **Parameters:**
\n", " `method`: str, one of `ols`, `wls_struct`, `wls_var`, `mint_shrink`, `mint_cov`.
\n", " `nonnegative`: bool, reconciled forecasts should be nonnegative?
\n", + " `mint_shr_ridge`: float, ridge numeric protection to MinTrace-shr covariance estimator.
\n", "\n", " **References:**
\n", " - [Wickramasuriya, S. L., Athanasopoulos, G., & Hyndman, R. J. (2019). \\\"Optimal forecast reconciliation for\n", @@ -860,10 +861,13 @@ " \"\"\"\n", " def __init__(self, \n", " method: str,\n", - " nonnegative: bool = False):\n", + " nonnegative: bool = False,\n", + " mint_shr_ridge: Optional[float] = 2e-8):\n", " self.method = method\n", " self.nonnegative = nonnegative\n", " self.insample = method in ['wls_var', 'mint_cov', 'mint_shrink']\n", + " if method == 'mint_shrink':\n", + " self.mint_shr_ridge = mint_shr_ridge\n", "\n", " def reconcile(self, \n", " S: np.ndarray,\n", @@ -941,7 +945,7 @@ " lmd = max(min(lmd, 1), 0)\n", "\n", " # Protection: final ridge diagonal protection\n", - " W = (lmd * tar + (1 - lmd) * covm) + 1e-8\n", + " W = (lmd * tar + (1 - lmd) * covm) + self.mint_shr_ridge\n", " else:\n", " raise ValueError(f'Unkown reconciliation method {self.method}')\n", "\n", @@ -1084,7 +1088,7 @@ "cls_min_trace = MinTrace(method='mint_shrink')\n", "cls_min_trace(\n", " S=S,\n", - " y_hat=S @ y_hat_bottom, \n", + " y_hat=S @ y_hat_bottom,\n", " y_insample=diff_len_y_insample,\n", " y_hat_insample=diff_len_y_hat_insample,\n", " idx_bottom=idx_bottom\n", diff --git a/nbs/probabilistic_methods.ipynb b/nbs/probabilistic_methods.ipynb index 51306e4a..934d9f13 100644 --- a/nbs/probabilistic_methods.ipynb +++ b/nbs/probabilistic_methods.ipynb @@ -13,7 +13,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Probabilistic Methods" + "# Probabilistic Methods " ] }, {