raise exception sooner if not pre-processed before training

axolotl-ai-cloud · Jan 20, 2024 · 799779f · 799779f
1 parent 334f02c
commit 799779f
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/src/axolotl/utils/data.py b/src/axolotl/utils/data.py
@@ -160,6 +160,10 @@ def load_tokenized_prepared_datasets(
     else:
         LOG.info(f"Unable to find prepared dataset in {prepared_ds_path}")
         LOG.info("Loading raw datasets...")
+        if not cfg.is_preprocess:
+            raise RuntimeWarning(
+                "Processing datasets during training can lead to VRAM instability. Please pre-process your dataset"
+            )
 
         if cfg.seed:
             seed = cfg.seed

diff --git a/src/axolotl/utils/trainer.py b/src/axolotl/utils/trainer.py
@@ -107,10 +107,6 @@ def drop_long_seq(sample, sequence_len=2048):
 
 
 def process_datasets_for_packing(cfg, train_dataset, eval_dataset, tokenizer):
-    if cfg.is_preprocess:
-        LOG.warning(
-            "Processing datasets during training can lead to VRAM instability. Please pre-process your dataset"
-        )
     drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len)
     with zero_first(is_main_process()):
         if cfg.group_by_length: