Add a way to count tokens without encoding the whole text; improve performance; fix bugs

.github/workflows/node.js.yml

@@ -5,26 +5,24 @@ name: Node.js CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
         node-version: [12.x, 14.x]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: npm ci
-    - run: npm run build --if-present
-    - run: npm test
-
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: npm ci
+      - run: npm run build --if-present
+      - run: npm test

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-.npmrc
+*.d.ts
+.npmrc

.npmignore

@@ -0,0 +1,2 @@
+.npmrc
+tsconfig.json

.prettierignore

@@ -0,0 +1 @@
+data/*.json

README.md

@@ -1,7 +1,9 @@
 # GPT-3-Encoder
+
 Javascript BPE Encoder Decoder for GPT-2 / GPT-3
 
 ## About
+
 GPT-2 and GPT-3 use byte pair encoding to turn text into a series of integers to feed into the model. This is a javascript implementation of OpenAI's original python encoder/decoder which can be found [here](https://github.com/openai/gpt-2)
 
 ## Install with npm
@@ -15,20 +17,17 @@ npm install gpt-3-encoder
 Compatible with Node >= 12
 
 ```js
-const {encode, decode} = require('gpt-3-encoder')
+const { encode, decode } = require("gpt-3-encoder");
 
-const str = 'This is an example sentence to try encoding out on!'
-const encoded = encode(str)
-console.log('Encoded this string looks like: ', encoded)
+const str = "This is an example sentence to try encoding out on!";
+const encoded = encode(str);
+console.log("Encoded this string looks like: ", encoded);
 
-console.log('We can look at each token and what it represents')
-for(let token of encoded){
-  console.log({token, string: decode([token])})
+console.log("We can look at each token and what it represents");
+for (let token of encoded) {
+  console.log({ token, string: decode([token]) });
 }
 
-const decoded = decode(encoded)
-console.log('We can decode it back into:\n', decoded)
-
+const decoded = decode(encoded);
+console.log("We can decode it back into:\n", decoded);
 ```
-
-

data/getBpe.js

@@ -0,0 +1,20 @@
+const path = require("path");
+const fs = require("fs");
+const { dictZip, range } = require("../utils");
+
+const bpe_file = fs.readFileSync(path.join(__dirname, "./vocab.bpe"), "utf-8");
+const lines = bpe_file.split("\n");
+
+const bpe_merges = lines.slice(1, lines.length - 1).map((x) =>
+  x
+    .split(/(\s+)/)
+    .filter((e) => e.trim().length > 0)
+    .join(","),
+);
+
+const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length));
+
+fs.writeFileSync(
+  path.join(__dirname, "./bpe_ranks.json"),
+  JSON.stringify(bpe_ranks),
+);