feat(merge): save tokenizer on merge (#362)

axolotl-ai-cloud · Aug 12, 2023 · 289d5c4 · 289d5c4
1 parent 35c8b90
commit 289d5c4
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/scripts/finetune.py b/scripts/finetune.py
@@ -265,6 +265,7 @@ def train(
  if cfg.local_rank == 0:
  LOG.info("saving merged model")
  model.save_pretrained(str(Path(cfg.output_dir) / "merged"))
+ tokenizer.save_pretrained(str(Path(cfg.output_dir) / "merged"))
  return
 
  if cfg.inference: