Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FEAT] Faster creation of ProbReconciler's ordered levels #137

Merged
merged 15 commits into from
Dec 17, 2022
Merged

[FEAT] Faster creation of ProbReconciler's ordered levels #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
2c65ae0
Added MSSE, sCRPS, and EScore placeholder
kdgutier Dec 15, 2022
aa9942c
Default [N,H,samples] shape for easier future handling
kdgutier Dec 15, 2022
366e1e4
Default [N,H,samples] shape for easier future handling
kdgutier Dec 15, 2022
ffb968b
Working energy score
kdgutier Dec 15, 2022
77c8fbf
Improved docstrings
kdgutier Dec 16, 2022
fcd8613
Added Exception messages and QL unit test
kdgutier Dec 16, 2022
3a91dc4
TourismL end to end experiment placeholder
kdgutier Dec 16, 2022
4e457be
Partial y_hat_insample protections
kdgutier Dec 16, 2022
c632334
adding tqdm dependency
kdgutier Dec 16, 2022
dce5d44
adding tqdm to reconcilers for loop
kdgutier Dec 16, 2022
70e8fb2
adding timer utility
kdgutier Dec 16, 2022
7158415
solving merge
kdgutier Dec 16, 2022
8597624
Informative Except Y_hat_df and Y_df column mismatch message
kdgutier Dec 17, 2022
b1adfc7
Cleaning notebooks
kdgutier Dec 17, 2022
94e7873
solving merge
kdgutier Dec 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions hierarchicalforecast/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Bootstrap.get_prediction_levels': ( 'probabilistic_methods.html#bootstrap.get_prediction_levels',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Bootstrap.get_prediction_quantiles': ( 'probabilistic_methods.html#bootstrap.get_prediction_quantiles',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Bootstrap.get_samples': ( 'probabilistic_methods.html#bootstrap.get_samples',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Normality': ( 'probabilistic_methods.html#normality',
Expand All @@ -118,6 +120,8 @@
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Normality.get_prediction_levels': ( 'probabilistic_methods.html#normality.get_prediction_levels',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Normality.get_prediction_quantiles': ( 'probabilistic_methods.html#normality.get_prediction_quantiles',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.Normality.get_samples': ( 'probabilistic_methods.html#normality.get_samples',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.PERMBU': ( 'probabilistic_methods.html#permbu',
Expand All @@ -134,6 +138,8 @@
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.PERMBU.get_prediction_levels': ( 'probabilistic_methods.html#permbu.get_prediction_levels',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.PERMBU.get_prediction_quantiles': ( 'probabilistic_methods.html#permbu.get_prediction_quantiles',
'hierarchicalforecast/probabilistic_methods.py'),
'hierarchicalforecast.probabilistic_methods.PERMBU.get_samples': ( 'probabilistic_methods.html#permbu.get_samples',
'hierarchicalforecast/probabilistic_methods.py')},
'hierarchicalforecast.utils': { 'hierarchicalforecast.utils.CodeTimer': ( 'utils.html#codetimer',
Expand Down
24 changes: 15 additions & 9 deletions hierarchicalforecast/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def reconcile(self,
# TODO: Complete y_hat_insample protection
if intervals_method in ['bootstrap', 'permbu']:
if not (set(model_names) <= set(Y_df.columns)):
raise Exception('Check `Y_hat_df`, `Y_df` columns difference')
raise Exception('Check `Y_hat_df`s models are included in `Y_df` columns')

# Same Y_hat_df/S_df/Y_df's unique_id order to prevent errors
S_ = S.loc[uids]
Expand Down Expand Up @@ -206,15 +206,21 @@ def reconcile(self,
# Parse final outputs
fcsts[f'{model_name}/{reconcile_fn_name}'] = fcsts_model['mean'].flatten()
if intervals_method in ['bootstrap', 'normality', 'permbu'] and level is not None:
for lv in level:
fcsts[f'{model_name}/{reconcile_fn_name}-lo-{lv}'] = fcsts_model[f'lo-{lv}'].flatten()
fcsts[f'{model_name}/{reconcile_fn_name}-hi-{lv}'] = fcsts_model[f'hi-{lv}'].flatten()

end = time.time()
self.execution_times[f'{model_name}/{reconcile_fn_name}'] = (end - start)

level.sort()
hi_names = [f'{model_name}/{reconcile_fn_name}-hi-{lv}' for lv in level]
lo_names = [f'{model_name}/{reconcile_fn_name}-lo-{lv}' for lv in reversed(level)]
sorted_quantiles = np.reshape(fcsts_model['quantiles'], (len(fcsts),-1))
intervals_df = pd.DataFrame(sorted_quantiles,
columns=(lo_names+hi_names), index=fcsts.index)
fcsts = pd.concat([fcsts, intervals_df], axis=1)

del sorted_quantiles
del intervals_df
if self.insample and has_fitted:
del reconciler_args['y_hat_insample']
del y_hat_insample
gc.collect()

end = time.time()
self.execution_times[f'{model_name}/{reconcile_fn_name}'] = (end - start)

return fcsts
7 changes: 6 additions & 1 deletion hierarchicalforecast/methods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,12 @@ def _reconcile(S: np.ndarray,

# Probabilistic reconciliation
if (level is not None) and (sampler is not None):
res = sampler.get_prediction_levels(res=res, level=level)
# Update results dictionary within
# Vectorized quantiles
quantiles = np.concatenate(
[[(100 - lv) / 200, ((100 - lv) / 200) + lv / 100] for lv in level])
quantiles = np.sort(quantiles)
res = sampler.get_prediction_quantiles(res, quantiles)

return res

Expand Down
26 changes: 26 additions & 0 deletions hierarchicalforecast/probabilistic_methods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ def get_prediction_levels(self, res, level):
res[f'hi-{lv}'] = res['mean'] + zs * self.sigmah_rec
return res

def get_prediction_quantiles(self, res, quantiles):
""" Adds reconciled forecast quantiles to results dictionary """
# [N,H,None] + [None None,Q] * [N,H,None] -> [N,H,Q]
z = norm.ppf(quantiles)
res['sigmah'] = self.sigmah_rec
res['quantiles'] = res['mean'][:,:,None] + z[None,None,:] * self.sigmah_rec[:,:,None]
return res

# %% ../nbs/probabilistic_methods.ipynb 10
class Bootstrap:
""" Bootstrap Probabilistic Reconciliation Class.
Expand Down Expand Up @@ -187,6 +195,15 @@ def get_prediction_levels(self, res, level):
res[f'hi-{lv}'] = np.quantile(samples, max_q, axis=2)
return res

def get_prediction_quantiles(self, res, quantiles):
""" Adds reconciled forecast quantiles to results dictionary """
samples = self.get_samples(num_samples=self.num_samples)

# [Q, N, H] -> [N, H, Q]
sample_quantiles = np.quantile(samples, quantiles, axis=2)
res['quantiles'] = sample_quantiles.transpose((1, 2, 0))
return res

# %% ../nbs/probabilistic_methods.ipynb 14
class PERMBU:
""" PERMBU Probabilistic Reconciliation Class.
Expand Down Expand Up @@ -400,3 +417,12 @@ def get_prediction_levels(self, res, level):
res[f'lo-{lv}'] = np.quantile(samples, min_q, axis=2)
res[f'hi-{lv}'] = np.quantile(samples, max_q, axis=2)
return res

def get_prediction_quantiles(self, res, quantiles):
""" Adds reconciled forecast quantiles to results dictionary """
samples = self.get_samples(num_samples=self.num_samples)

# [Q, N, H] -> [N, H, Q]
sample_quantiles = np.quantile(samples, quantiles, axis=2)
res['quantiles'] = sample_quantiles.transpose((1, 2, 0))
return res
24 changes: 15 additions & 9 deletions nbs/core.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@
" # TODO: Complete y_hat_insample protection\n",
" if intervals_method in ['bootstrap', 'permbu']:\n",
" if not (set(model_names) <= set(Y_df.columns)):\n",
" raise Exception('Check `Y_hat_df`, `Y_df` columns difference')\n",
" raise Exception('Check `Y_hat_df`s models are included in `Y_df` columns')\n",
"\n",
" # Same Y_hat_df/S_df/Y_df's unique_id order to prevent errors\n",
" S_ = S.loc[uids]\n",
Expand Down Expand Up @@ -306,17 +306,23 @@
" # Parse final outputs\n",
" fcsts[f'{model_name}/{reconcile_fn_name}'] = fcsts_model['mean'].flatten()\n",
" if intervals_method in ['bootstrap', 'normality', 'permbu'] and level is not None:\n",
" for lv in level:\n",
" fcsts[f'{model_name}/{reconcile_fn_name}-lo-{lv}'] = fcsts_model[f'lo-{lv}'].flatten()\n",
" fcsts[f'{model_name}/{reconcile_fn_name}-hi-{lv}'] = fcsts_model[f'hi-{lv}'].flatten()\n",
" \n",
" end = time.time()\n",
" self.execution_times[f'{model_name}/{reconcile_fn_name}'] = (end - start)\n",
"\n",
" level.sort()\n",
" hi_names = [f'{model_name}/{reconcile_fn_name}-hi-{lv}' for lv in level]\n",
" lo_names = [f'{model_name}/{reconcile_fn_name}-lo-{lv}' for lv in reversed(level)]\n",
" sorted_quantiles = np.reshape(fcsts_model['quantiles'], (len(fcsts),-1))\n",
" intervals_df = pd.DataFrame(sorted_quantiles, \n",
" columns=(lo_names+hi_names), index=fcsts.index)\n",
" fcsts = pd.concat([fcsts, intervals_df], axis=1)\n",
"\n",
" del sorted_quantiles\n",
" del intervals_df\n",
" if self.insample and has_fitted:\n",
" del reconciler_args['y_hat_insample']\n",
" del y_hat_insample\n",
" gc.collect()\n",
"\n",
" end = time.time()\n",
" self.execution_times[f'{model_name}/{reconcile_fn_name}'] = (end - start)\n",
"\n",
" return fcsts"
]
},
Expand Down
11 changes: 8 additions & 3 deletions nbs/methods.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,12 @@
"\n",
" # Probabilistic reconciliation\n",
" if (level is not None) and (sampler is not None):\n",
" res = sampler.get_prediction_levels(res=res, level=level)\n",
" # Update results dictionary within\n",
" # Vectorized quantiles\n",
" quantiles = np.concatenate(\n",
" [[(100 - lv) / 200, ((100 - lv) / 200) + lv / 100] for lv in level])\n",
" quantiles = np.sort(quantiles)\n",
" res = sampler.get_prediction_quantiles(res, quantiles)\n",
"\n",
" return res"
]
Expand Down Expand Up @@ -1673,15 +1678,15 @@
"cls_bottom_up = BottomUp()\n",
"bu_bootstrap_intervals = cls_bottom_up(**reconciler_args)\n",
"test_eq(\n",
" ['mean', 'sigmah', 'lo-80', 'hi-80', 'lo-90', 'hi-90'],\n",
" ['mean', 'sigmah', 'quantiles'],\n",
" list(bu_bootstrap_intervals.keys())\n",
")\n",
"\n",
"# test PERMBU interval's names\n",
"reconciler_args['intervals_method'] = 'permbu'\n",
"bu_permbu_intervals = cls_bottom_up(**reconciler_args)\n",
"test_eq(\n",
" ['mean', 'lo-80', 'hi-80', 'lo-90', 'hi-90'],\n",
" ['mean', 'quantiles'],\n",
" list(bu_permbu_intervals.keys())\n",
")"
]
Expand Down
37 changes: 37 additions & 0 deletions nbs/probabilistic_methods.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,14 @@
" for zs, lv in zip(z, level):\n",
" res[f'lo-{lv}'] = res['mean'] - zs * self.sigmah_rec\n",
" res[f'hi-{lv}'] = res['mean'] + zs * self.sigmah_rec\n",
" return res\n",
"\n",
" def get_prediction_quantiles(self, res, quantiles):\n",
" \"\"\" Adds reconciled forecast quantiles to results dictionary \"\"\"\n",
" # [N,H,None] + [None None,Q] * [N,H,None] -> [N,H,Q]\n",
" z = norm.ppf(quantiles)\n",
" res['sigmah'] = self.sigmah_rec\n",
" res['quantiles'] = res['mean'][:,:,None] + z[None,None,:] * self.sigmah_rec[:,:,None]\n",
" return res"
]
},
Expand Down Expand Up @@ -273,6 +281,15 @@
" max_q = min_q + lv / 100\n",
" res[f'lo-{lv}'] = np.quantile(samples, min_q, axis=2)\n",
" res[f'hi-{lv}'] = np.quantile(samples, max_q, axis=2)\n",
" return res\n",
"\n",
" def get_prediction_quantiles(self, res, quantiles):\n",
" \"\"\" Adds reconciled forecast quantiles to results dictionary \"\"\"\n",
" samples = self.get_samples(num_samples=self.num_samples)\n",
"\n",
" # [Q, N, H] -> [N, H, Q]\n",
" sample_quantiles = np.quantile(samples, quantiles, axis=2)\n",
" res['quantiles'] = sample_quantiles.transpose((1, 2, 0))\n",
" return res"
]
},
Expand Down Expand Up @@ -519,6 +536,15 @@
" max_q = min_q + lv / 100\n",
" res[f'lo-{lv}'] = np.quantile(samples, min_q, axis=2)\n",
" res[f'hi-{lv}'] = np.quantile(samples, max_q, axis=2)\n",
" return res\n",
"\n",
" def get_prediction_quantiles(self, res, quantiles):\n",
" \"\"\" Adds reconciled forecast quantiles to results dictionary \"\"\"\n",
" samples = self.get_samples(num_samples=self.num_samples)\n",
"\n",
" # [Q, N, H] -> [N, H, Q]\n",
" sample_quantiles = np.quantile(samples, quantiles, axis=2)\n",
" res['quantiles'] = sample_quantiles.transpose((1, 2, 0))\n",
" return res"
]
},
Expand Down Expand Up @@ -671,6 +697,17 @@
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"level = [1,2,3,0]\n",
"level.sort()\n",
"level"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down