From 1631ead41ce4afeb4e4d174cf608f477a88aeb85 Mon Sep 17 00:00:00 2001
From: NanoCode012 <kevinvong@rocketmail.com>
Date: Sun, 13 Aug 2023 00:18:10 +0900
Subject: [PATCH] feat(merge): save tokenizer on merge (#362)

---
 scripts/finetune.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/finetune.py b/scripts/finetune.py
index 53d6d8557..635c06930 100644
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -265,6 +265,7 @@ def train(
         if cfg.local_rank == 0:
             LOG.info("saving merged model")
             model.save_pretrained(str(Path(cfg.output_dir) / "merged"))
+            tokenizer.save_pretrained(str(Path(cfg.output_dir) / "merged"))
         return
 
     if cfg.inference: