add support of codeqwen due to tokenizer

convert-hf-to-gguf.py

@@ -363,6 +363,16 @@ def _set_vocab_sentencepiece(self):
  scores.append(-1000.0)
  toktypes.append(SentencePieceTokenTypes.USER_DEFINED)
 
+ if vocab_size > len(tokens):
+ pad_count = vocab_size - len(tokens)
+ print(
+ f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]"
+ )
+ for i in range(1, pad_count + 1):
+ tokens.append(f"[PAD{i}]")
+ scores.append(-1000.0)
+ toktypes.append(SentencePieceTokenTypes.UNUSED)
+
  assert len(tokens) == vocab_size
 
  self.gguf_writer.add_tokenizer_model("llama")
@@ -1781,6 +1791,25 @@ def write_tensors(self):
 class Qwen2Model(Model):
  model_arch = gguf.MODEL_ARCH.QWEN2
 
+ def __init__(self, dir_model: Path, ftype: int, fname_out: Path, is_big_endian: bool, use_temp_file: bool):
+ super().__init__(dir_model, ftype, fname_out, is_big_endian, use_temp_file)
+ self.hparams = Qwen2Model.load_hparams(dir_model)
+
+ @staticmethod
+ def load_hparams(dir_model):
+ with open(dir_model / "config.json", "r", encoding="utf-8") as f1, \
+ open(dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f2:
+ hparams = json.load(f1)
+ hparams.update(json.load(f2))
+
+ return hparams
+
+ def set_vocab(self):
+ if self.hparams.get("tokenizer_class") == "PreTrainedTokenizerFast":
+ self._set_vocab_sentencepiece()
+ else:
+ self._set_vocab_gpt2()
+
 
 @Model.register("Qwen2MoeForCausalLM")
 class Qwen2MoeModel(Model):