fix

hjh0119 · Aug 7, 2024 · 47e08f3 · 47e08f3
1 parent ac2b2a7
commit 47e08f3
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/swift/llm/sft.py b/swift/llm/sft.py
@@ -339,7 +339,7 @@ def llm_sft(args: SftArguments) -> Dict[str, Any]:
             dataset_info['val_dataset'] = stat_dataset(val_dataset) if not streaming else None
     else:
         dataset_info = None
-        td0, tkwargs0 = template.encode(train_dataset[0]) if streaming else next(iter(train_dataset)), {}
+        td0, tkwargs0 = template.encode(train_dataset[0])
         print_example(td0, tokenizer, tkwargs0)
         train_dataset = LazyLLMDataset(train_dataset, template)
         if val_dataset is not None:

diff --git a/swift/llm/utils/argument.py b/swift/llm/utils/argument.py
@@ -1032,7 +1032,7 @@ def __post_init__(self) -> None:
             self.gradient_accumulation_steps = math.ceil(16 / self.batch_size / self.world_size)
         template_info = TEMPLATE_MAPPING[self.template_type]
         self._handle_streaming_args()
-        if self.lazy_tokenize is None:
+        if self.lazy_tokenize is None and not self.streaming:
             self.lazy_tokenize = template_info.get('lazy_tokenize', False)
             logger.info(f'Setting args.lazy_tokenize: {self.lazy_tokenize}')
         if self.dataloader_num_workers is None: