tiktoken.js

var e={9:(e,t,n)=>{let r;function o(e){r=e}n.r(t),n.d(t,{Tiktoken:()=>A,__wbg_parse_161c68378e086ae1:()=>M,__wbg_set_wasm:()=>o,__wbg_stringify_f4f701bc34ceda61:()=>O,__wbindgen_error_new:()=>T,__wbindgen_is_undefined:()=>z,__wbindgen_object_drop_ref:()=>W,__wbindgen_string_get:()=>C,__wbindgen_throw:()=>P,encoding_for_model:()=>S,get_encoding:()=>E}),e=n.hmd(e);let i=new("undefined"==typeof TextDecoder?(0,e.require)("util").TextDecoder:TextDecoder)("utf-8",{ignoreBOM:!0,fatal:!0});i.decode();let _=null;function a(){return null!==_&&0!==_.byteLength||(_=new Uint8Array(r.memory.buffer)),_}function s(e,t){return e>>>=0,i.decode(a().subarray(e,e+t))}const d=new Array(128).fill(void 0);d.push(void 0,null,!0,!1);let c=d.length;function l(e){c===d.length&&d.push(d.length+1);const t=c;return c=d[t],d[t]=e,t}function p(e,t){try{return e.apply(this,t)}catch(e){r.__wbindgen_export_0(l(e))}}function u(e){return d[e]}function g(e){const t=u(e);return function(e){e<132||(d[e]=c,c=e)}(e),t}let b=0,f=new("undefined"==typeof TextEncoder?(0,e.require)("util").TextEncoder:TextEncoder)("utf-8");const w="function"==typeof f.encodeInto?function(e,t){return f.encodeInto(e,t)}:function(e,t){const n=f.encode(e);return t.set(n),{read:e.length,written:n.length}};function h(e,t,n){if(void 0===n){const n=f.encode(e),r=t(n.length,1)>>>0;return a().subarray(r,r+n.length).set(n),b=n.length,r}let r=e.length,o=t(r,1)>>>0;const i=a();let _=0;for(;_<r;_++){const t=e.charCodeAt(_);if(t>127)break;i[o+_]=t}if(_!==r){0!==_&&(e=e.slice(_)),o=n(o,r,r=_+3*e.length,1)>>>0;const t=a().subarray(o+_,o+r);_+=w(e,t).written,o=n(o,r,_,1)>>>0}return b=_,o}let y=null;function k(){return(null===y||!0===y.buffer.detached||void 0===y.buffer.detached&&y.buffer!==r.memory.buffer)&&(y=new DataView(r.memory.buffer)),y}let m=null;function x(){return null!==m&&0!==m.byteLength||(m=new Uint32Array(r.memory.buffer)),m}function v(e,t){return e>>>=0,x().subarray(e/4,e/4+t)}function I(e,t){return e>>>=0,a().subarray(e/1,e/1+t)}function E(e,t){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const i=r.__wbindgen_add_to_stack_pointer(-16),_=h(e,r.__wbindgen_export_1,r.__wbindgen_export_2),a=b;r.get_encoding(i,_,a,l(t));var n=k().getInt32(i+0,!0),o=k().getInt32(i+4,!0);if(k().getInt32(i+8,!0))throw g(o);return A.__wrap(n)}finally{r.__wbindgen_add_to_stack_pointer(16)}}function S(e,t){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const i=r.__wbindgen_add_to_stack_pointer(-16),_=h(e,r.__wbindgen_export_1,r.__wbindgen_export_2),a=b;r.encoding_for_model(i,_,a,l(t));var n=k().getInt32(i+0,!0),o=k().getInt32(i+4,!0);if(k().getInt32(i+8,!0))throw g(o);return A.__wrap(n)}finally{r.__wbindgen_add_to_stack_pointer(16)}}const j="undefined"==typeof FinalizationRegistry?{register:()=>{},unregister:()=>{}}:new FinalizationRegistry((e=>r.__wbg_tiktoken_free(e>>>0,1)));class A{static __wrap(e){e>>>=0;const t=Object.create(A.prototype);return t.__wbg_ptr=e,j.register(t,t.__wbg_ptr,t),t}__destroy_into_raw(){const e=this.__wbg_ptr;return this.__wbg_ptr=0,j.unregister(this),e}free(){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");const e=this.__destroy_into_raw();r.__wbg_tiktoken_free(e,0)}constructor(e,t,n){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");const o=h(e,r.__wbindgen_export_1,r.__wbindgen_export_2),i=b,_=h(n,r.__wbindgen_export_1,r.__wbindgen_export_2),a=b,s=r.tiktoken_new(o,i,l(t),_,a);return this.__wbg_ptr=s>>>0,j.register(this,this.__wbg_ptr,this),this}get name(){try{const n=r.__wbindgen_add_to_stack_pointer(-16);r.tiktoken_name(n,this.__wbg_ptr);var e=k().getInt32(n+0,!0),t=k().getInt32(n+4,!0);let o;return 0!==e&&(o=s(e,t).slice(),r.__wbindgen_export_3(e,1*t,1)),o}finally{r.__wbindgen_add_to_stack_pointer(16)}}encode(e,t,n){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const s=r.__wbindgen_add_to_stack_pointer(-16),d=h(e,r.__wbindgen_export_1,r.__wbindgen_export_2),c=b;r.tiktoken_encode(s,this.__wbg_ptr,d,c,l(t),l(n));var o=k().getInt32(s+0,!0),i=k().getInt32(s+4,!0),_=k().getInt32(s+8,!0);if(k().getInt32(s+12,!0))throw g(_);var a=v(o,i).slice();return r.__wbindgen_export_3(o,4*i,4),a}finally{r.__wbindgen_add_to_stack_pointer(16)}}encode_ordinary(e){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const i=r.__wbindgen_add_to_stack_pointer(-16),_=h(e,r.__wbindgen_export_1,r.__wbindgen_export_2),a=b;r.tiktoken_encode_ordinary(i,this.__wbg_ptr,_,a);var t=k().getInt32(i+0,!0),n=k().getInt32(i+4,!0),o=v(t,n).slice();return r.__wbindgen_export_3(t,4*n,4),o}finally{r.__wbindgen_add_to_stack_pointer(16)}}encode_with_unstable(e,t,n){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const _=r.__wbindgen_add_to_stack_pointer(-16),a=h(e,r.__wbindgen_export_1,r.__wbindgen_export_2),s=b;r.tiktoken_encode_with_unstable(_,this.__wbg_ptr,a,s,l(t),l(n));var o=k().getInt32(_+0,!0),i=k().getInt32(_+4,!0);if(k().getInt32(_+8,!0))throw g(i);return g(o)}finally{r.__wbindgen_add_to_stack_pointer(16)}}encode_single_token(e){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");const t=function(e,t){const n=t(1*e.length,1)>>>0;return a().set(e,n/1),b=e.length,n}(e,r.__wbindgen_export_1),n=b;return r.tiktoken_encode_single_token(this.__wbg_ptr,t,n)>>>0}decode(e){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const i=r.__wbindgen_add_to_stack_pointer(-16),_=function(e,t){const n=t(4*e.length,4)>>>0;return x().set(e,n/4),b=e.length,n}(e,r.__wbindgen_export_1),a=b;r.tiktoken_decode(i,this.__wbg_ptr,_,a);var t=k().getInt32(i+0,!0),n=k().getInt32(i+4,!0),o=I(t,n).slice();return r.__wbindgen_export_3(t,1*n,1),o}finally{r.__wbindgen_add_to_stack_pointer(16)}}decode_single_token_bytes(e){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");try{const i=r.__wbindgen_add_to_stack_pointer(-16);r.tiktoken_decode_single_token_bytes(i,this.__wbg_ptr,e);var t=k().getInt32(i+0,!0),n=k().getInt32(i+4,!0),o=I(t,n).slice();return r.__wbindgen_export_3(t,1*n,1),o}finally{r.__wbindgen_add_to_stack_pointer(16)}}token_byte_values(){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");return g(r.tiktoken_token_byte_values(this.__wbg_ptr))}}function M(){return p((function(e,t){return l(JSON.parse(s(e,t)))}),arguments)}function O(){return p((function(e){return l(JSON.stringify(u(e)))}),arguments)}function T(e,t){return l(new Error(s(e,t)))}function z(e){return void 0===u(e)}function W(e){g(e)}function C(e,t){if(null==r)throw new Error("tiktoken: WASM binary has not been propery initialized.");const n=u(t),o="string"==typeof n?n:void 0;var i=null==o?0:h(o,r.__wbindgen_export_1,r.__wbindgen_export_2),_=b;k().setInt32(e+4,_,!0),k().setInt32(e+0,i,!0)}function P(e,t){throw new Error(s(e,t))}}},t={};function n(r){var o=t[r];if(void 0!==o)return o.exports;var i=t[r]={id:r,loaded:!1,exports:{}};return e[r](i,i.exports,n),i.loaded=!0,i.exports}n.d=(e,t)=>{for(var r in t)n.o(t,r)&&!n.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},n.hmd=e=>((e=Object.create(e)).children||(e.children=[]),Object.defineProperty(e,"exports",{enumerable:!0,set:()=>{throw new Error("ES Modules may not assign module.exports or exports.*, Use ESM export syntax, instead: "+e.id)}}),e),n.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),n.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},(()=>{var e;if("string"==typeof import.meta.url&&(e=import.meta.url),!e)throw new Error("Automatic publicPath is not supported in this browser");e=e.replace(/#.*$/,"").replace(/\?.*$/,"").replace(/\/[^\/]+$/,"/"),n.p=e})();var r={};n.d(r,{sF:()=>o.Tiktoken,cI:()=>o.encoding_for_model,TR:()=>o.get_encoding,Ts:()=>_,Hh:()=>g}),n.p;var o=n(9);let i=!1;async function _(e){if(i)return o;const t=await e({"./tiktoken_bg.js":o}),n="instance"in t&&t.instance instanceof WebAssembly.Instance?t.instance:t instanceof WebAssembly.Instance?t:null;if(null==n)throw new Error("Missing instance");return o.__wbg_set_wasm(n.exports),i=!0,o}const a=[],s=[],d="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";for(var c=0;c<64;++c)a[c]=d[c],s[d.charCodeAt(c)]=c;function l(e,t,n){for(var r,o,i=[],_=t;_<n;_+=3)r=(e[_]<<16&16711680)+(e[_+1]<<8&65280)+(255&e[_+2]),i.push(a[(o=r)>>18&63]+a[o>>12&63]+a[o>>6&63]+a[63&o]);return i.join("")}function p(e){for(var t,n=e.length,r=n%3,o=[],i=16383,_=0,s=n-r;_<s;_+=i)o.push(l(e,_,_+i>s?s:_+i));return 1===r?(t=e[n-1],o.push(a[t>>2]+a[t<<4&63]+"==")):2===r&&(t=(e[n-2]<<8)+e[n-1],o.push(a[t>>10]+a[t>>4&63]+a[t<<2&63]+"=")),o.join("")}function u(e,t){const n=Array.from({length:256},((e,t)=>t)).filter((e=>{return!((t=e)<=31||t>=127&&t<=160||173==t||" "===String.fromCharCode(e));var t})),r=n.reduce(((e,t)=>(e[String.fromCharCode(t)]=t,e)),{});let o=0;for(let e=0;e<256;e++)n.includes(e)||(n.push(e),r[String.fromCharCode(256+o)]=e,o+=1);if(256!==n.length)throw new Error("rank_to_intbyte.length must be 2**8");const i=e.split("\n").slice(1,-1).map((e=>e.split(" ")));function _(e){return e.split("").map((e=>r[e]))}const a=Object.fromEntries(n.map(((e,t)=>[e,t])));o=n.length;for(const[e,t]of i)a[[..._(e),..._(t)].join(",")]=o,o+=1;const s=JSON.parse(t),d=Object.fromEntries(Object.entries(s).map((([e,t])=>[_(e).join(","),t])));function c(e){return JSON.stringify(Object.keys(e).sort().map((t=>[t,e[t]])))}if(delete d[_("<|endoftext|>").join(",")],delete d[_("<|startoftext|>").join(",")],c(a)!==c(d))throw new Error("bpe_ranks !== encoder_json_loaded");return a}async function g(e,t){const n=t||(e=>fetch(e).then((e=>e.text())));if("data_gym_to_mergeable_bpe_ranks"in e){const[t,o]=await Promise.all([n(e.data_gym_to_mergeable_bpe_ranks.vocab_bpe_file),n(e.data_gym_to_mergeable_bpe_ranks.encoder_json_file)]);return{explicit_n_vocab:e.explicit_n_vocab,pat_str:e.pat_str,special_tokens:e.special_tokens,bpe_ranks:(r=u(t,o),Object.entries(r).sort(((e,t)=>e[1]-t[1])).map((([e,t])=>[p(e.split(",").map((e=>Number.parseInt(e,10)))),t].join(" "))).join("\n")+"\n")}}return{explicit_n_vocab:e.explicit_n_vocab,pat_str:e.pat_str,special_tokens:e.special_tokens,bpe_ranks:await n(e.load_tiktoken_bpe)};var r}s["-".charCodeAt(0)]=62,s["_".charCodeAt(0)]=63;var b=r.sF,f=r.cI,w=r.TR,h=r.Ts,y=r.Hh;export{b as Tiktoken,f as encoding_for_model,w as get_encoding,h as init,y as load};