gguf-py : revert ggml-org#2984 handling of negative token IDs

This reverts commit 6519e9c.
cebtenzzre · Oct 12, 2023 · 11a8e3e · 11a8e3e
1 parent 370359e
commit 11a8e3e
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/gguf-py/gguf/gguf.py b/gguf-py/gguf/gguf.py
@@ -1011,7 +1011,7 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool:
             else:
                 continue
             for maybe_token_id in (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content):
-                if isinstance(maybe_token_id, int) and maybe_token_id >= 0:
+                if isinstance(maybe_token_id, int):
                     self.special_token_ids[typ] = maybe_token_id
                 break
         return True
@@ -1024,7 +1024,7 @@ def _try_load_from_config_json(self, path: Path) -> bool:
             config = json.load(f)
         for typ in self.special_token_types:
             maybe_token_id = config.get(f'{typ}_token_id')
-            if isinstance(maybe_token_id, int) and maybe_token_id >= 0:
+            if isinstance(maybe_token_id, int):
                 self.special_token_ids[typ] = maybe_token_id
         return True
 

diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gguf"
-version = "0.4.4"
+version = "0.4.5"
 description = "Write ML models in GGUF for GGML"
 authors = ["GGML <ggml@ggml.ai>"]
 packages = [