feat(nitro): cl, vercel and netlify edge function

.gitignore

@@ -44,4 +44,6 @@ yarn-error.log*
 
 .netlify
 
+.nitro
+
 wrangler.toml

packages/core/package.json

@@ -116,23 +116,18 @@
   },
   "peerDependencies": {
     "@supabase/supabase-js": "^2.22.0",
-    "gpt3-tokenizer": "^1.1.5",
     "ofetch": "^1.0.1"
   },
   "peerDependenciesMeta": {
     "@supabase/supabase-js": {
       "optional": true
     },
-    "gpt3-tokenizer": {
-      "optional": true
-    },
     "ofetch": {
       "optional": false
     }
   },
   "devDependencies": {
     "@supabase/supabase-js": "^2.22.0",
-    "gpt3-tokenizer": "^1.1.5",
     "ofetch": "^1.0.1"
   }
 }

packages/core/src/openai/prompt.ts

@@ -1,6 +1,6 @@
 // https://github.com/sparticleinc/chatgpt-google-summary-extension/blob/main/src/content-script/prompt.ts
 
-import GPT3Tokenizer from 'gpt3-tokenizer'
+import { countWord } from '../utils'
 
 export enum ProviderType {
   ChatGPT = 'chatgpt',
@@ -21,8 +21,10 @@ export function getSummaryPrompt(transcript = '', providerConfigs?: ProviderType
 
 // Seems like 15,000 bytes is the limit for the prompt
 const textLimit = 14000
-const limit = 1100 // 1000 is a buffer
-const apiLimit = 2000
+// const limit = 1100 // 1000 is a buffer
+const countLimit = 800
+// const apiLimit = 2000
+const apiCountLimit = 1300
 
 function truncateTranscript(str: string, providerConfigs?: ProviderType) {
   let textStr = str
@@ -38,19 +40,39 @@ function truncateTranscript(str: string, providerConfigs?: ProviderType) {
 }
 
 function truncateTranscriptByToken(str: string, providerConfigs?: ProviderType) {
-  const tokenLimit = providerConfigs === ProviderType.GPT3 ? apiLimit : limit
+  // const tokenLimit = providerConfigs === ProviderType.GPT3 ? apiLimit : limit
+  const wordCountLimit = providerConfigs === ProviderType.GPT3 ? apiCountLimit : countLimit
+  let count = countWord(str)
 
-  const tokenizer = new GPT3Tokenizer({ type: 'gpt3' })
-  const encoded: { bpe: number[]; text: string[] } = tokenizer.encode(str)
-  const bytes = encoded.bpe.length
-
-  if (bytes > tokenLimit) {
-    const ratio = tokenLimit / bytes
-    const newStr = str.substring(0, str.length * ratio)
+  if (count > wordCountLimit) {
+    let newStr = str
+    while (count > wordCountLimit) {
+      const k = count - wordCountLimit
+      newStr = newStr.substring(0, newStr.length - (k / 2 < 10 ? 10 : k / 2))
+      count = countWord(newStr)
+      // console.log('in ', k, count)
+    }
+    // const encoded = encode(newStr)
+    // const bytes = encoded.length
+    // console.log(count, bytes)
 
     return newStr
   }
 
+  // const encoded = encode(str)
+  // const bytes = encoded.length
+  // console.log(count, bytes)
+
+  // if (bytes > tokenLimit) {
+  //   const ratio = tokenLimit / bytes
+  //   const newStr = str.substring(0, str.length * ratio)
+  //   const count = countWord(newStr)
+  //   const encoded = encode(newStr)
+  //   const _bytes = encoded.length
+  //   console.log(count, _bytes)
+  //   return newStr
+  // }
+
   return str
 }
 

packages/core/src/utils/index.ts

@@ -28,7 +28,7 @@ function countWord(data: string): number {
 
   for (let i = 0; i < m.length; i++) {
     if (m[i].charCodeAt(0) >= 0x4E00)
-      count += m[i].length
+      count += m[i].length * 2
     else
       count += 1
   }

packages/core/vite.config.ts

@@ -39,15 +39,13 @@ export default defineConfig({
       // into your library
       external: [
         '@supabase/supabase-js',
-        'gpt3-tokenizer',
         'ofetch',
       ],
       output: {
         // Provide global variables to use in the UMD build
         // for externalized deps
         globals: {
           '@supabase/supabase-js': '@supabase/supabase-js',
-          'gpt3-tokenizer': 'gpt3-tokenizer',
           'ofetch': 'ofetch',
         },
       },