Feat/conformal prediction

darts/ad/anomaly_model/forecasting_am.py

@@ -120,10 +120,9 @@ def fit(
             If set to 'value', `start` corresponds to the index value/label of the first predicted point. Will raise
             an error if the value is not in `series`' index. Default: `'value'`
         num_samples
-            Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for
-            deterministic models.
+            Number of times a prediction is sampled from a probabilistic model. Must be `1` for deterministic models.
         verbose
-            Whether to print progress.
+            Whether to print the progress.
         show_warnings
             Whether to show warnings related to historical forecasts optimization, or parameters `start` and
             `train_length`.
@@ -201,10 +200,9 @@ def score(
             If set to 'value', `start` corresponds to the index value/label of the first predicted point. Will raise
             an error if the value is not in `series`' index. Default: `'value'`
         num_samples
-            Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for
-            deterministic models.
+            Number of times a prediction is sampled from a probabilistic model. Must be `1` for deterministic models.
         verbose
-            Whether to print progress.
+            Whether to print the progress.
         show_warnings
             Whether to show warnings related to historical forecasts optimization, or parameters `start` and
             `train_length`.
@@ -289,10 +287,9 @@ def predict_series(
             If set to 'value', `start` corresponds to the index value/label of the first predicted point. Will raise
             an error if the value is not in `series`' index. Default: `'value'`
         num_samples
-            Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for
-            deterministic models.
+            Number of times a prediction is sampled from a probabilistic model. Must be `1` for deterministic models.
         verbose
-            Whether to print progress.
+            Whether to print the progress.
         show_warnings
             Whether to show warnings related to historical forecasts optimization, or parameters `start` and
             `train_length`.
@@ -385,10 +382,9 @@ def eval_metric(
             If set to 'value', `start` corresponds to the index value/label of the first predicted point. Will raise
             an error if the value is not in `series`' index. Default: `'value'`
         num_samples
-            Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for
-            deterministic models.
+            Number of times a prediction is sampled from a probabilistic model. Must be `1` for deterministic models.
         verbose
-            Whether to print progress.
+            Whether to print the progress.
         show_warnings
             Whether to show warnings related to historical forecasts optimization, or parameters `start` and
             `train_length`.
@@ -491,10 +487,9 @@ def show_anomalies(
             If set to 'value', `start` corresponds to the index value/label of the first predicted point. Will raise
             an error if the value is not in `series`' index. Default: `'value'`
         num_samples
-            Number of times a prediction is sampled from a probabilistic model. Should be left set to 1 for
-            deterministic models.
+            Number of times a prediction is sampled from a probabilistic model. Must be `1` for deterministic models.
         verbose
-            Whether to print progress.
+            Whether to print the progress.
         show_warnings
             Whether to show warnings related to historical forecasts optimization, or parameters `start` and
             `train_length`.

darts/metrics/__init__.py

@@ -6,52 +6,67 @@
 and quantile forecasts. For probabilistic and quantile forecasts, use parameter `q` to define the quantile(s) to
 compute the deterministic metrics on:
 
-    - Aggregated over time:
-        Absolute metrics:
-            - :func:`MERR <darts.metrics.metrics.merr>`: Mean Error
-            - :func:`MAE <darts.metrics.metrics.mae>`: Mean Absolute Error
-            - :func:`MSE <darts.metrics.metrics.mse>`: Mean Squared Error
-            - :func:`RMSE <darts.metrics.metrics.rmse>`: Root Mean Squared Error
-            - :func:`RMSLE <darts.metrics.metrics.rmsle>`: Root Mean Squared Log Error
-
-        Relative metrics:
-            - :func:`MASE <darts.metrics.metrics.mase>`: Mean Absolute Scaled Error
-            - :func:`MSSE <darts.metrics.metrics.msse>`: Mean Squared Scaled Error
-            - :func:`RMSSE <darts.metrics.metrics.rmsse>`: Root Mean Squared Scaled Error
-            - :func:`MAPE <darts.metrics.metrics.mape>`: Mean Absolute Percentage Error
-            - :func:`sMAPE <darts.metrics.metrics.smape>`: symmetric Mean Absolute Percentage Error
-            - :func:`OPE <darts.metrics.metrics.ope>`: Overall Percentage Error
-            - :func:`MARRE <darts.metrics.metrics.marre>`: Mean Absolute Ranged Relative Error
-
-        Other metrics:
-            - :func:`R2 <darts.metrics.metrics.r2_score>`: Coefficient of Determination
-            - :func:`CV <darts.metrics.metrics.coefficient_of_variation>`: Coefficient of Variation
-
-    - Per time step:
-        Absolute metrics:
-            - :func:`ERR <darts.metrics.metrics.err>`: Error
-            - :func:`AE <darts.metrics.metrics.ae>`: Absolute Error
-            - :func:`SE <darts.metrics.metrics.se>`: Squared Error
-            - :func:`SLE <darts.metrics.metrics.sle>`: Squared Log Error
-
-        Relative metrics:
-            - :func:`ASE <darts.metrics.metrics.ase>`: Absolute Scaled Error
-            - :func:`SSE <darts.metrics.metrics.sse>`: Squared Scaled Error
-            - :func:`APE <darts.metrics.metrics.ape>`: Absolute Percentage Error
-            - :func:`sAPE <darts.metrics.metrics.sape>`: symmetric Absolute Percentage Error
-            - :func:`ARRE <darts.metrics.metrics.arre>`: Absolute Ranged Relative Error
-
-For probabilistic forecasts (storchastic predictions with `num_samples >> 1`):
-    - Aggregated over time:
+- Aggregated over time:
+    Absolute metrics:
+        - :func:`MERR <darts.metrics.metrics.merr>`: Mean Error
+        - :func:`MAE <darts.metrics.metrics.mae>`: Mean Absolute Error
+        - :func:`MSE <darts.metrics.metrics.mse>`: Mean Squared Error
+        - :func:`RMSE <darts.metrics.metrics.rmse>`: Root Mean Squared Error
+        - :func:`RMSLE <darts.metrics.metrics.rmsle>`: Root Mean Squared Log Error
+
+    Relative metrics:
+        - :func:`MASE <darts.metrics.metrics.mase>`: Mean Absolute Scaled Error
+        - :func:`MSSE <darts.metrics.metrics.msse>`: Mean Squared Scaled Error
+        - :func:`RMSSE <darts.metrics.metrics.rmsse>`: Root Mean Squared Scaled Error
+        - :func:`MAPE <darts.metrics.metrics.mape>`: Mean Absolute Percentage Error
+        - :func:`sMAPE <darts.metrics.metrics.smape>`: symmetric Mean Absolute Percentage Error
+        - :func:`OPE <darts.metrics.metrics.ope>`: Overall Percentage Error
+        - :func:`MARRE <darts.metrics.metrics.marre>`: Mean Absolute Ranged Relative Error
+
+    Other metrics:
+        - :func:`R2 <darts.metrics.metrics.r2_score>`: Coefficient of Determination
+        - :func:`CV <darts.metrics.metrics.coefficient_of_variation>`: Coefficient of Variation
+
+- Per time step:
+    Absolute metrics:
+        - :func:`ERR <darts.metrics.metrics.err>`: Error
+        - :func:`AE <darts.metrics.metrics.ae>`: Absolute Error
+        - :func:`SE <darts.metrics.metrics.se>`: Squared Error
+        - :func:`SLE <darts.metrics.metrics.sle>`: Squared Log Error
+
+    Relative metrics:
+        - :func:`ASE <darts.metrics.metrics.ase>`: Absolute Scaled Error
+        - :func:`SSE <darts.metrics.metrics.sse>`: Squared Scaled Error
+        - :func:`APE <darts.metrics.metrics.ape>`: Absolute Percentage Error
+        - :func:`sAPE <darts.metrics.metrics.sape>`: symmetric Absolute Percentage Error
+        - :func:`ARRE <darts.metrics.metrics.arre>`: Absolute Ranged Relative Error
+
+For probabilistic forecasts (storchastic predictions with `num_samples >> 1`) and quantile forecasts:
+
+- Aggregated over time:
+    Quantile metrics:
         - :func:`MQL <darts.metrics.metrics.mql>`: Mean Quantile Loss
         - :func:`QR <darts.metrics.metrics.qr>`: Quantile Risk
+
+    Quantile interval metrics:
         - :func:`MIW <darts.metrics.metrics.miw>`: Mean Interval Width
-    - Per time step:
+        - :func:`MWS <darts.metrics.metrics.miws>`: Mean Interval Winkler Score
+        - :func:`MIC <darts.metrics.metrics.mic>`: Mean Interval Coverage
+        - :func:`MINCS_QR <darts.metrics.metrics.mincs_qr>`: Mean Interval Non-Conformity Score for Quantile Regression
+
+- Per time step:
+    Quantile metrics:
         - :func:`QL <darts.metrics.metrics.ql>`: Quantile Loss
+
+    Quantile interval metrics:
         - :func:`IW <darts.metrics.metrics.iw>`: Interval Width
+        - :func:`WS <darts.metrics.metrics.iws>`: Interval Winkler Score
+        - :func:`IC <darts.metrics.metrics.ic>`: Interval Coverage
+        - :func:`INCS_QR <darts.metrics.metrics.incs_qr>`: Interval Non-Conformity Score for Quantile Regression
 
 For Dynamic Time Warping (DTW) (aggregated over time):
-    - :func:`DTW <darts.metrics.metrics.dtw_metric>`: Dynamic Time Warping Metric
+
+- :func:`DTW <darts.metrics.metrics.dtw_metric>`: Dynamic Time Warping Metric
 """
 
 from darts.metrics.metrics import (
@@ -62,13 +77,19 @@
     coefficient_of_variation,
     dtw_metric,
     err,
+    ic,
+    incs_qr,
     iw,
+    iws,
     mae,
     mape,
     marre,
     mase,
     merr,
+    mic,
+    mincs_qr,
     miw,
+    miws,
     mql,
     mse,
     msse,
@@ -86,6 +107,44 @@
     sse,
 )
 
+ALL_METRICS = {
+    ae,
+    ape,
+    arre,
+    ase,
+    coefficient_of_variation,
+    dtw_metric,
+    err,
+    iw,
+    iws,
+    mae,
+    mape,
+    marre,
+    mase,
+    merr,
+    miw,
+    miws,
+    mql,
+    mse,
+    msse,
+    ope,
+    ql,
+    qr,
+    r2_score,
+    rmse,
+    rmsle,
+    rmsse,
+    sape,
+    se,
+    sle,
+    smape,
+    sse,
+    ic,
+    mic,
+    incs_qr,
+    mincs_qr,
+}
+
 TIME_DEPENDENT_METRICS = {
     ae,
     ape,
@@ -98,8 +157,23 @@
     sle,
     sse,
     iw,
+    iws,
+    ic,
+    incs_qr,
 }
 
+Q_INTERVAL_METRICS = {
+    iw,
+    iws,
+    miw,
+    miws,
+    ic,
+    mic,
+    incs_qr,
+}
+
+NON_Q_METRICS = {dtw_metric}
+
 __all__ = [
     "ae",
     "ape",
@@ -130,4 +204,10 @@
     "sse",
     "iw",
     "miw",
+    "iws",
+    "miws",
+    "ic",
+    "mic",
+    "incs_qr",
+    "mincs_qr",
 ]