switch tokenizer implementation with pure js and more compatible js-t…

…iktoken
transitive-bullshit · May 17, 2023 · 3ec62b8 · 3ec62b8
1 parent f39279c
commit 3ec62b8
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 12 deletions.
diff --git a/package.json b/package.json
@@ -37,10 +37,10 @@
     "test:prettier": "prettier '**/*.{js,jsx,ts,tsx}' --check"
   },
   "dependencies": {
-    "@dqbd/tiktoken": "^1.0.7",
     "cac": "^6.7.14",
     "conf": "^11.0.1",
     "eventsource-parser": "^1.0.0",
+    "js-tiktoken": "^1.0.5",
     "keyv": "^4.5.2",
     "p-timeout": "^6.1.1",
     "quick-lru": "^6.1.1",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/tokenizer.ts b/src/tokenizer.ts
@@ -1,8 +1,8 @@
-import { get_encoding } from '@dqbd/tiktoken'
+import { getEncoding } from 'js-tiktoken'
 
 // TODO: make this configurable
-const tokenizer = get_encoding('cl100k_base')
+const tokenizer = getEncoding('cl100k_base')
 
 export function encode(input: string): Uint32Array {
-  return tokenizer.encode(input)
+  return new Uint32Array(tokenizer.encode(input))
 }