add warning to SparkBoostLGBM for low volume dataset under streaming …

…execution mode
sb-ai-lab · Aug 2, 2023 · 8beea57 · 8beea57
1 parent 16cdab4
commit 8beea57
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/sparklightautoml/ml_algo/boost_lgbm.py b/sparklightautoml/ml_algo/boost_lgbm.py
@@ -471,6 +471,12 @@ def fit_predict_single_fold(
  logger.info(f"Use single dataset mode: {lgbm.getUseSingleDatasetMode()}. NumThreads: {lgbm.getNumThreads()}")
  logger.info(f"All lgbm booster params: {run_params}")
 
+ if (run_params["executionMode"] == "streaming") and (full_data.count() <= 25_000):
+ warnings.warn(f"The fitting of lightgbm in streaming execution mode "
+ f"may fail with SEGSIGV / SIGBUS error (probably due to a bug in synapse ml) "
+ f"if too few data available per core. "
+ f"Consider switching to bulk execution mode if such crashes happen", RuntimeWarning)
+
  # fitting the model
  ml_model = lgbm.fit(self._assembler.transform(full_data))