fix(openai): switched to pure js tiktoken (#248)

traceloop · May 7, 2024 · 9d8805e · 9d8805e
1 parent 8ee046e
commit 9d8805e
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 12 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/instrumentation-openai/package.json b/packages/instrumentation-openai/package.json
@@ -40,7 +40,7 @@
     "@opentelemetry/instrumentation": "^0.49.0",
     "@opentelemetry/semantic-conventions": "^1.22.0",
     "@traceloop/ai-semantic-conventions": "^0.8.0",
-    "tiktoken": "^1.0.13"
+    "js-tiktoken": "^1.0.11"
   },
   "devDependencies": {
     "@pollyjs/adapter-node-http": "^6.0.6",

diff --git a/packages/instrumentation-openai/src/instrumentation.ts b/packages/instrumentation-openai/src/instrumentation.ts
@@ -45,7 +45,7 @@ import type {
 } from "openai/resources";
 import type { Stream } from "openai/streaming";
 import { version } from "../package.json";
-import { encoding_for_model, TiktokenModel, Tiktoken } from "tiktoken";
+import { encodingForModel, TiktokenModel, Tiktoken } from "js-tiktoken";
 
 export class OpenAIInstrumentation extends InstrumentationBase<any> {
   protected declare _config: OpenAIInstrumentationConfig;
@@ -726,7 +726,7 @@ export class OpenAIInstrumentation extends InstrumentationBase<any> {
 
     if (!encoding) {
       try {
-        encoding = encoding_for_model(model as TiktokenModel);
+        encoding = encodingForModel(model as TiktokenModel);
         this._encodingCache.set(model, encoding);
       } catch (e) {
         this._diag.warn(e);