feat(merge): save tokenizer on merge (axolotl-ai-cloud#362)

mkeoliya · Aug 12, 2023 · 1631ead · 1631ead
1 parent 9804410
commit 1631ead
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/scripts/finetune.py b/scripts/finetune.py
@@ -265,6 +265,7 @@ def train(
  if cfg.local_rank == 0:
  LOG.info("saving merged model")
  model.save_pretrained(str(Path(cfg.output_dir) / "merged"))
+ tokenizer.save_pretrained(str(Path(cfg.output_dir) / "merged"))
  return
 
  if cfg.inference: