[FIX] Numerical stability scaling for timeseries forecasting tasks

autoPyTorch/constants.py

@@ -58,6 +58,10 @@
                                          "forecasting tasks! Please run \n pip install autoPyTorch[forecasting] \n to "\
                                          "install the corresponding dependencies!"
 
+# This value is applied to ensure numerical stability: Sometimes we want to rescale some values: value / scale.
+# We make the scale value to be 1 if it is smaller than this value to ensure that the scaled value will not resutl in
+# overflow
+VERY_SMALL_VALUE = 1e-12
 
 # The constant values for time series forecasting comes from
 # https://github.com/rakshitha123/TSForecasting/blob/master/experiments/deep_learning_experiments.py

autoPyTorch/pipeline/components/preprocessing/time_series_preprocessing/scaling/utils.py

@@ -6,6 +6,8 @@
 
 from sklearn.base import BaseEstimator
 
+from autoPyTorch.constants import VERY_SMALL_VALUE
+
 
 # Similar to / inspired by
 # https://github.com/tslearn-team/tslearn/blob/a3cf3bf/tslearn/preprocessing/preprocessing.py
@@ -41,7 +43,7 @@ def fit(self, X: Union[pd.DataFrame, np.ndarray], y: Any = None) -> "TimeSeriesS
                 self.loc[self.static_features] = X[self.static_features].mean()
 
                 # ensure that if all the values are the same in a group, we could still normalize them correctly
-                self.scale[self.scale == 0] = 1.
+                self.scale[self.scale < VERY_SMALL_VALUE] = 1.
 
             elif self.mode == "min_max":
                 X_grouped = X.groupby(X.index)
@@ -55,14 +57,14 @@ def fit(self, X: Union[pd.DataFrame, np.ndarray], y: Any = None) -> "TimeSeriesS
                 self.loc = min_
                 self.scale = diff_
                 self.scale.mask(self.scale == 0.0, self.loc)
-                self.scale[self.scale == 0.0] = 1.0
+                self.scale[self.scale < VERY_SMALL_VALUE] = 1.0
 
             elif self.mode == "max_abs":
                 X_abs = X.transform("abs")
                 max_abs_ = X_abs.groupby(X_abs.index).agg("max")
                 max_abs_[self.static_features] = max_abs_[self.static_features].max()
 
-                max_abs_[max_abs_ == 0.0] = 1.0
+                max_abs_[max_abs_ < VERY_SMALL_VALUE] = 1.0
                 self.loc = None
                 self.scale = max_abs_
 
@@ -73,7 +75,7 @@ def fit(self, X: Union[pd.DataFrame, np.ndarray], y: Any = None) -> "TimeSeriesS
                 mean_abs_[self.static_features] = mean_abs_[self.static_features].mean()
                 self.scale = mean_abs_.mask(mean_abs_ == 0.0, X_abs.agg("max"))
 
-                self.scale[self.scale == 0] = 1
+                self.scale[self.scale < VERY_SMALL_VALUE] = 1
                 self.loc = None
 
             elif self.mode == "none":
@@ -108,7 +110,7 @@ def transform(self, X: Union[pd.DataFrame, np.ndarray]) -> Union[pd.DataFrame, n
                 loc = X.mean(axis=0, keepdims=True)
                 scale = np.nan_to_num(X.std(axis=0, ddof=1, keepdims=True))
                 scale = np.where(scale == 0, loc, scale)
-                scale[scale == 0] = 1.
+                scale[scale < VERY_SMALL_VALUE] = 1.
                 return (X - loc) / scale
 
             elif self.mode == 'min_max':
@@ -119,21 +121,21 @@ def transform(self, X: Union[pd.DataFrame, np.ndarray]) -> Union[pd.DataFrame, n
                 loc = min_
                 scale = diff_
                 scale = np.where(scale == 0., loc, scale)
-                scale[scale == 0.0] = 1.0
+                scale[scale < VERY_SMALL_VALUE] = 1.0
                 return (X - loc) / scale
 
             elif self.mode == "max_abs":
                 X_abs = np.abs(X)
                 max_abs_ = X_abs.max(0, keepdims=True)
-                max_abs_[max_abs_ == 0.0] = 1.0
+                max_abs_[max_abs_ < VERY_SMALL_VALUE] = 1.0
                 scale = max_abs_
                 return X / scale
 
             elif self.mode == 'mean_abs':
                 X_abs = np.abs(X)
                 mean_abs_ = X_abs.mean(0, keepdims=True)
                 scale = np.where(mean_abs_ == 0.0, np.max(X_abs), mean_abs_)
-                scale[scale == 0] = 1
+                scale[scale < VERY_SMALL_VALUE] = 1
                 return X / scale
 
             elif self.mode == "none":

autoPyTorch/pipeline/components/setup/forecasting_target_scaling/utils.py

@@ -4,6 +4,8 @@
 
 import torch
 
+from autoPyTorch.constants import VERY_SMALL_VALUE
+
 
 # Similar to / inspired by
 # https://github.com/tslearn-team/tslearn/blob/a3cf3bf/tslearn/preprocessing/preprocessing.py
@@ -30,7 +32,7 @@ def transform(self,
 
                 offset_targets = past_targets - loc
                 scale = torch.where(torch.logical_or(scale == 0.0, scale == torch.nan), offset_targets[:, [-1]], scale)
-                scale[scale == 0.0] = 1.0
+                scale[scale < VERY_SMALL_VALUE] = 1.0
                 if future_targets is not None:
                     future_targets = (future_targets - loc) / scale
                 return (past_targets - loc) / scale, future_targets, loc, scale
@@ -42,14 +44,14 @@ def transform(self,
                 diff_ = max_ - min_
                 loc = min_
                 scale = torch.where(diff_ == 0, past_targets[:, [-1]], diff_)
-                scale[scale == 0.0] = 1.0
+                scale[scale < VERY_SMALL_VALUE] = 1.0
                 if future_targets is not None:
                     future_targets = (future_targets - loc) / scale
                 return (past_targets - loc) / scale, future_targets, loc, scale
 
             elif self.mode == "max_abs":
                 max_abs_ = torch.max(torch.abs(past_targets), dim=1, keepdim=True)[0]
-                max_abs_[max_abs_ == 0.0] = 1.0
+                max_abs_[max_abs_ < VERY_SMALL_VALUE] = 1.0
                 scale = max_abs_
                 if future_targets is not None:
                     future_targets = future_targets / scale
@@ -58,7 +60,7 @@ def transform(self,
             elif self.mode == 'mean_abs':
                 mean_abs = torch.mean(torch.abs(past_targets), dim=1, keepdim=True)
                 scale = torch.where(mean_abs == 0.0, past_targets[:, [-1]], mean_abs)
-                scale[scale == 0.0] = 1.0
+                scale[scale < VERY_SMALL_VALUE] = 1.0
                 if future_targets is not None:
                     future_targets = future_targets / scale
                 return past_targets / scale, future_targets, None, scale
@@ -82,7 +84,7 @@ def transform(self,
                 offset_targets = past_targets - loc
                 # ensure that all the targets are scaled properly
                 scale = torch.where(torch.logical_or(scale == 0.0, scale == torch.nan), offset_targets[:, [-1]], scale)
-                scale[scale == 0.0] = 1.0
+                scale[scale < VERY_SMALL_VALUE] = 1.0
 
                 if future_targets is not None:
                     future_targets = (future_targets - loc) / scale
@@ -100,7 +102,7 @@ def transform(self,
                 diff_ = max_ - min_
                 loc = min_
                 scale = torch.where(diff_ == 0, past_targets[:, [-1]], diff_)
-                scale[scale == 0.0] = 1.0
+                scale[scale < VERY_SMALL_VALUE] = 1.0
 
                 if future_targets is not None:
                     future_targets = (future_targets - loc) / scale
@@ -110,7 +112,7 @@ def transform(self,
 
             elif self.mode == "max_abs":
                 max_abs_ = torch.max(torch.abs(valid_past_targets), dim=1, keepdim=True)[0]
-                max_abs_[max_abs_ == 0.0] = 1.0
+                max_abs_[max_abs_ < VERY_SMALL_VALUE] = 1.0
                 scale = max_abs_
                 if future_targets is not None:
                     future_targets = future_targets / scale
@@ -122,8 +124,8 @@ def transform(self,
             elif self.mode == 'mean_abs':
                 mean_abs = torch.sum(torch.abs(valid_past_targets), dim=1, keepdim=True) / valid_past_obs
                 scale = torch.where(mean_abs == 0.0, valid_past_targets[:, [-1]], mean_abs)
-                # in case that all values in the tensor is 0
-                scale[scale == 0.0] = 1.0
+                # in case that all values in the tensor is too small
+                scale[scale < VERY_SMALL_VALUE] = 1.0
                 if future_targets is not None:
                     future_targets = future_targets / scale
 

test/test_pipeline/components/preprocessing/forecasting/test_scaling.py

@@ -26,6 +26,7 @@ def setUp(self) -> None:
 
         columns = ['f1', 's', 'f2']
         self.raw_data = [data_seq_1, data_seq_2]
+
         self.data = pd.DataFrame(np.concatenate([data_seq_1, data_seq_2]), columns=columns, index=[0] * 3 + [1] * 4)
         self.static_features = ('s',)
         self.static_features_column = (1, )
@@ -38,6 +39,9 @@ def setUp(self) -> None:
                                    'static_features': self.static_features,
                                    'is_small_preprocess': True}
 
+        self.small_data = pd.DataFrame(np.array([[1e-10, 0., 1e-15],
+                                                [-1e-10, 0., +1e-15]]), columns=columns, index=[0] * 2)
+
     def test_base_and_standard_scaler(self):
         scaler_component = BaseScaler(scaling_mode='standard')
         X = {
@@ -82,6 +86,10 @@ def test_base_and_standard_scaler(self):
         transformed_test = np.concatenate([scaler.transform(raw_data) for raw_data in self.raw_data])
         self.assertTrue(np.allclose(transformed_test[:, [0, -1]], transformed_test[:, [0, -1]]))
 
+        scaler.dataset_is_small_preprocess = True
+        scaler.fit(self.small_data)
+        self.assertTrue(np.allclose(scaler.scale.values.flatten(), np.array([1.41421356e-10, 1., 1.])))
+
     def test_min_max(self):
         scaler = TimeSeriesScaler(mode='min_max',
                                   static_features=self.static_features
@@ -109,6 +117,10 @@ def test_min_max(self):
         transformed_test = np.concatenate([scaler.transform(raw_data) for raw_data in self.raw_data])
         self.assertTrue(np.allclose(transformed_test[:, [0, -1]], transformed_test[:, [0, -1]]))
 
+        scaler.dataset_is_small_preprocess = True
+        scaler.fit(self.small_data)
+        self.assertTrue(np.all(scaler.scale.values.flatten() == np.array([2e-10, 1., 1.])))
+
     def test_max_abs_scaler(self):
         scaler = TimeSeriesScaler(mode='max_abs',
                                   static_features=self.static_features
@@ -136,6 +148,10 @@ def test_max_abs_scaler(self):
         transformed_test = np.concatenate([scaler.transform(raw_data) for raw_data in self.raw_data])
         self.assertTrue(np.allclose(transformed_test[:, [0, -1]], transformed_test[:, [0, -1]]))
 
+        scaler.dataset_is_small_preprocess = True
+        scaler.fit(self.small_data)
+        self.assertTrue(np.all(scaler.scale.values.flatten() == np.array([1e-10, 1., 1.])))
+
     def test_mean_abs_scaler(self):
         scaler = TimeSeriesScaler(mode='mean_abs',
                                   static_features=self.static_features
@@ -162,6 +178,10 @@ def test_mean_abs_scaler(self):
         transformed_test = np.concatenate([scaler.transform(raw_data) for raw_data in self.raw_data])
         self.assertTrue(np.allclose(transformed_test[:, [0, -1]], transformed_test[:, [0, -1]]))
 
+        scaler.dataset_is_small_preprocess = True
+        scaler.fit(self.small_data)
+        self.assertTrue(np.all(scaler.scale.values.flatten() == np.array([1e-10, 1., 1.])))
+
     def test_no_scaler(self):
         scaler = TimeSeriesScaler(mode='none',
                                   static_features=self.static_features

test/test_pipeline/components/setup/forecasting/test_forecasting_target_scaling.py

@@ -95,6 +95,11 @@ def test_target_mean_abs_scalar(self):
 
         self.assertIsNone(loc_full)
 
+        _, _, _, scale = scalar(
+            torch.Tensor([[1e-10, 1e-10, 1e-10], [1e-15, 1e-15, 1e-15]]).reshape([2, 3, 1])
+        )
+        self.assertTrue(torch.equal(scale.flatten(), torch.Tensor([1e-10, 1.])))
+
     def test_target_standard_scalar(self):
         X = {'dataset_properties': {}}
         scalar = BaseTargetScaler(scaling_mode='standard')
@@ -178,6 +183,11 @@ def test_target_standard_scalar(self):
         self.assertTrue(torch.equal(loc, loc_full))
         self.assertTrue(torch.equal(scale, scale_full))
 
+        _, _, _, scale = scalar(
+            torch.Tensor([[1e-10, -1e-10, 1e-10], [1e-15, -1e-15, 1e-15]]).reshape([2, 3, 1])
+        )
+        self.assertTrue(torch.all(torch.isclose(scale.flatten(), torch.Tensor([1.1547e-10, 1.]))))
+
     def test_target_min_max_scalar(self):
         X = {'dataset_properties': {}}
         scalar = BaseTargetScaler(scaling_mode='min_max')
@@ -245,6 +255,11 @@ def test_target_min_max_scalar(self):
         self.assertTrue(torch.equal(transformed_future_targets_full, transformed_future_targets_full))
         self.assertTrue(torch.equal(scale, scale_full))
 
+        _, _, _, scale = scalar(
+            torch.Tensor([[1e-10, 1e-10, 1e-10], [1e-15, 1e-15, 1e-15]]).reshape([2, 3, 1])
+        )
+        self.assertTrue(torch.equal(scale.flatten(), torch.Tensor([1e-10, 1.])))
+
     def test_target_max_abs_scalar(self):
         X = {'dataset_properties': {}}
         scalar = BaseTargetScaler(scaling_mode='max_abs')
@@ -309,3 +324,8 @@ def test_target_max_abs_scalar(self):
         self.assertTrue(torch.equal(transformed_future_targets_full, transformed_future_targets_full))
         self.assertIsNone(loc_full)
         self.assertTrue(torch.equal(scale, scale_full))
+
+        _, _, _, scale = scalar(
+            torch.Tensor([[1e-10, 1e-10, 1e-10], [1e-15, 1e-15, 1e-15]]).reshape([2, 3, 1])
+        )
+        self.assertTrue(torch.equal(scale.flatten(), torch.Tensor([1e-10, 1.])))