diff --git a/README.md b/README.md index db742297db3..ec1bdae4e1e 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ The following is the default `chatPromptTemplate`, although newlines and indenti #### Multi modal model -We currently only support IDEFICS as a multimodal model, hosted on TGI. You can enable it by using the following config (if you have a PRO HF Api token): +We currently support [IDEFICS](https://huggingface.co/blog/idefics) (hosted on TGI), OpenAI and Claude 3 as multimodal models. You can enable it by setting `multimodal: true` in your `MODELS` configuration. For IDEFICS, you must have a [PRO HF Api token](https://huggingface.co/settings/tokens). For OpenAI, see the [OpenAI section](#OpenAI). For Anthropic, see the [Anthropic section](#Anthropic). ```env { @@ -465,14 +465,34 @@ MODELS=`[ #### Anthropic -We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example. +We also support Anthropic models (including multimodal ones via `multmodal: true`) through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example. ``` MODELS=`[ + { + "name": "claude-3-haiku-20240307", + "displayName": "Claude 3 Haiku", + "description": "Fastest and most compact model for near-instant responsiveness", + "multimodal": true, + "parameters": { + "max_new_tokens": 4096, + }, + "endpoints": [ + { + "type": "anthropic", + // optionals + "apiKey": "sk-ant-...", + "baseURL": "https://api.anthropic.com", + "defaultHeaders": {}, + "defaultQuery": {} + } + ] + }, { "name": "claude-3-sonnet-20240229", "displayName": "Claude 3 Sonnet", "description": "Ideal balance of intelligence and speed", + "multimodal": true, "parameters": { "max_new_tokens": 4096, }, @@ -491,6 +511,7 @@ MODELS=`[ "name": "claude-3-opus-20240229", "displayName": "Claude 3 Opus", "description": "Most powerful model for highly complex tasks", + "multimodal": true, "parameters": { "max_new_tokens": 4096 }, @@ -516,6 +537,7 @@ MODELS=`[ "name": "claude-3-sonnet@20240229", "displayName": "Claude 3 Sonnet", "description": "Ideal balance of intelligence and speed", + "multimodal": true, "parameters": { "max_new_tokens": 4096, }, @@ -534,6 +556,7 @@ MODELS=`[ "name": "claude-3-haiku@20240307", "displayName": "Claude 3 Haiku", "description": "Fastest, most compact model for near-instant responsiveness", + "multimodal": true, "parameters": { "max_new_tokens": 4096 }, diff --git a/package-lock.json b/package-lock.json index 2c3e8aad09f..b82e2f623ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,13 +13,14 @@ "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", "@playwright/browser-chromium": "^1.43.1", - "@resvg/resvg-js": "^2.6.0", + "@resvg/resvg-js": "^2.6.2", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", "browser-image-resizer": "^2.4.1", "date-fns": "^2.29.3", "dotenv": "^16.0.3", "express": "^4.19.2", + "file-type": "^19.0.0", "handlebars": "^4.7.8", "highlight.js": "^11.7.0", "image-size": "^1.0.2", @@ -41,7 +42,7 @@ "satori-html": "^0.3.2", "sbd": "^1.0.19", "serpapi": "^1.1.1", - "sharp": "^0.33.2", + "sharp": "^0.33.3", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", @@ -88,7 +89,7 @@ "@google-cloud/vertexai": "^1.1.0", "aws4fetch": "^1.0.17", "cohere-ai": "^7.9.0", - "openai": "^4.14.2" + "openai": "^4.44.0" } }, "node_modules/@alloc/quick-lru": { @@ -237,9 +238,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "0.45.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-0.45.0.tgz", - "integrity": "sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz", + "integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==", "optional": true, "dependencies": { "tslib": "^2.4.0" @@ -811,9 +812,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.2.tgz", - "integrity": "sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz", + "integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==", "cpu": [ "arm64" ], @@ -832,13 +833,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.1" + "@img/sharp-libvips-darwin-arm64": "1.0.2" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.2.tgz", - "integrity": "sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz", + "integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==", "cpu": [ "x64" ], @@ -857,13 +858,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.1" + "@img/sharp-libvips-darwin-x64": "1.0.2" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.1.tgz", - "integrity": "sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz", + "integrity": "sha512-tcK/41Rq8IKlSaKRCCAuuY3lDJjQnYIW1UXU1kxcEKrfL8WR7N6+rzNoOxoQRJWTAECuKwgAHnPvqXGN8XfkHA==", "cpu": [ "arm64" ], @@ -882,9 +883,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.1.tgz", - "integrity": "sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz", + "integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==", "cpu": [ "x64" ], @@ -903,9 +904,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.1.tgz", - "integrity": "sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz", + "integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==", "cpu": [ "arm" ], @@ -924,9 +925,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.1.tgz", - "integrity": "sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz", + "integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==", "cpu": [ "arm64" ], @@ -945,9 +946,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.1.tgz", - "integrity": "sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz", + "integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==", "cpu": [ "s390x" ], @@ -966,9 +967,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.1.tgz", - "integrity": "sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz", + "integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==", "cpu": [ "x64" ], @@ -987,9 +988,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.1.tgz", - "integrity": "sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz", + "integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==", "cpu": [ "arm64" ], @@ -1008,9 +1009,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.1.tgz", - "integrity": "sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz", + "integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==", "cpu": [ "x64" ], @@ -1029,9 +1030,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.2.tgz", - "integrity": "sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz", + "integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==", "cpu": [ "arm" ], @@ -1050,13 +1051,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.1" + "@img/sharp-libvips-linux-arm": "1.0.2" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.2.tgz", - "integrity": "sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz", + "integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==", "cpu": [ "arm64" ], @@ -1075,13 +1076,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.1" + "@img/sharp-libvips-linux-arm64": "1.0.2" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.2.tgz", - "integrity": "sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz", + "integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==", "cpu": [ "s390x" ], @@ -1100,13 +1101,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.1" + "@img/sharp-libvips-linux-s390x": "1.0.2" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.2.tgz", - "integrity": "sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz", + "integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==", "cpu": [ "x64" ], @@ -1125,13 +1126,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.1" + "@img/sharp-libvips-linux-x64": "1.0.2" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.2.tgz", - "integrity": "sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz", + "integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==", "cpu": [ "arm64" ], @@ -1150,13 +1151,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.1" + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.2.tgz", - "integrity": "sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz", + "integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==", "cpu": [ "x64" ], @@ -1175,19 +1176,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.1" + "@img/sharp-libvips-linuxmusl-x64": "1.0.2" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.2.tgz", - "integrity": "sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz", + "integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==", "cpu": [ "wasm32" ], "optional": true, "dependencies": { - "@emnapi/runtime": "^0.45.0" + "@emnapi/runtime": "^1.1.0" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0", @@ -1200,9 +1201,9 @@ } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.2.tgz", - "integrity": "sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz", + "integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==", "cpu": [ "ia32" ], @@ -1221,9 +1222,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.2.tgz", - "integrity": "sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz", + "integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==", "cpu": [ "x64" ], @@ -1444,31 +1445,31 @@ "integrity": "sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q==" }, "node_modules/@resvg/resvg-js": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.0.tgz", - "integrity": "sha512-Tf3YpbBKcQn991KKcw/vg7vZf98v01seSv6CVxZBbRkL/xyjnoYB6KgrFL6zskT1A4dWC/vg77KyNOW+ePaNlA==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz", + "integrity": "sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q==", "engines": { "node": ">= 10" }, "optionalDependencies": { - "@resvg/resvg-js-android-arm-eabi": "2.6.0", - "@resvg/resvg-js-android-arm64": "2.6.0", - "@resvg/resvg-js-darwin-arm64": "2.6.0", - "@resvg/resvg-js-darwin-x64": "2.6.0", - "@resvg/resvg-js-linux-arm-gnueabihf": "2.6.0", - "@resvg/resvg-js-linux-arm64-gnu": "2.6.0", - "@resvg/resvg-js-linux-arm64-musl": "2.6.0", - "@resvg/resvg-js-linux-x64-gnu": "2.6.0", - "@resvg/resvg-js-linux-x64-musl": "2.6.0", - "@resvg/resvg-js-win32-arm64-msvc": "2.6.0", - "@resvg/resvg-js-win32-ia32-msvc": "2.6.0", - "@resvg/resvg-js-win32-x64-msvc": "2.6.0" + "@resvg/resvg-js-android-arm-eabi": "2.6.2", + "@resvg/resvg-js-android-arm64": "2.6.2", + "@resvg/resvg-js-darwin-arm64": "2.6.2", + "@resvg/resvg-js-darwin-x64": "2.6.2", + "@resvg/resvg-js-linux-arm-gnueabihf": "2.6.2", + "@resvg/resvg-js-linux-arm64-gnu": "2.6.2", + "@resvg/resvg-js-linux-arm64-musl": "2.6.2", + "@resvg/resvg-js-linux-x64-gnu": "2.6.2", + "@resvg/resvg-js-linux-x64-musl": "2.6.2", + "@resvg/resvg-js-win32-arm64-msvc": "2.6.2", + "@resvg/resvg-js-win32-ia32-msvc": "2.6.2", + "@resvg/resvg-js-win32-x64-msvc": "2.6.2" } }, "node_modules/@resvg/resvg-js-android-arm-eabi": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.0.tgz", - "integrity": "sha512-lJnZ/2P5aMocrFMW7HWhVne5gH82I8xH6zsfH75MYr4+/JOaVcGCTEQ06XFohGMdYRP3v05SSPLPvTM/RHjxfA==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.2.tgz", + "integrity": "sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==", "cpu": [ "arm" ], @@ -1481,9 +1482,9 @@ } }, "node_modules/@resvg/resvg-js-android-arm64": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.0.tgz", - "integrity": "sha512-N527f529bjMwYWShZYfBD60dXA4Fux+D695QsHQ93BDYZSHUoOh1CUGUyICevnTxs7VgEl98XpArmUWBZQVMfQ==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.2.tgz", + "integrity": "sha512-VcOKezEhm2VqzXpcIJoITuvUS/fcjIw5NA/w3tjzWyzmvoCdd+QXIqy3FBGulWdClvp4g+IfUemigrkLThSjAQ==", "cpu": [ "arm64" ], @@ -1496,9 +1497,9 @@ } }, "node_modules/@resvg/resvg-js-darwin-arm64": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.0.tgz", - "integrity": "sha512-MabUKLVayEwlPo0mIqAmMt+qESN8LltCvv5+GLgVga1avpUrkxj/fkU1TKm8kQegutUjbP/B0QuMuUr0uhF8ew==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.2.tgz", + "integrity": "sha512-nmok2LnAd6nLUKI16aEB9ydMC6Lidiiq2m1nEBDR1LaaP7FGs4AJ90qDraxX+CWlVuRlvNjyYJTNv8qFjtL9+A==", "cpu": [ "arm64" ], @@ -1511,9 +1512,9 @@ } }, "node_modules/@resvg/resvg-js-darwin-x64": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.0.tgz", - "integrity": "sha512-zrFetdnSw/suXjmyxSjfDV7i61hahv6DDG6kM7BYN2yJ3Es5+BZtqYZTcIWogPJedYKmzN1YTMWGd/3f0ubFiA==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.2.tgz", + "integrity": "sha512-GInyZLjgWDfsVT6+SHxQVRwNzV0AuA1uqGsOAW+0th56J7Nh6bHHKXHBWzUrihxMetcFDmQMAX1tZ1fZDYSRsw==", "cpu": [ "x64" ], @@ -1526,9 +1527,9 @@ } }, "node_modules/@resvg/resvg-js-linux-arm-gnueabihf": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.0.tgz", - "integrity": "sha512-sH4gxXt7v7dGwjGyzLwn7SFGvwZG6DQqLaZ11MmzbCwd9Zosy1TnmrMJfn6TJ7RHezmQMgBPi18bl55FZ1AT4A==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.2.tgz", + "integrity": "sha512-YIV3u/R9zJbpqTTNwTZM5/ocWetDKGsro0SWp70eGEM9eV2MerWyBRZnQIgzU3YBnSBQ1RcxRZvY/UxwESfZIw==", "cpu": [ "arm" ], @@ -1541,9 +1542,9 @@ } }, "node_modules/@resvg/resvg-js-linux-arm64-gnu": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.0.tgz", - "integrity": "sha512-fCyMncqCJtrlANADIduYF4IfnWQ295UKib7DAxFXQhBsM9PLDTpizr0qemZcCNadcwSVHnAIzL4tliZhCM8P6A==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.2.tgz", + "integrity": "sha512-zc2BlJSim7YR4FZDQ8OUoJg5holYzdiYMeobb9pJuGDidGL9KZUv7SbiD4E8oZogtYY42UZEap7dqkkYuA91pg==", "cpu": [ "arm64" ], @@ -1556,9 +1557,9 @@ } }, "node_modules/@resvg/resvg-js-linux-arm64-musl": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.0.tgz", - "integrity": "sha512-ouLjTgBQHQyxLht4FdMPTvuY8xzJigM9EM2Tlu0llWkN1mKyTQrvYWi6TA6XnKdzDJHy7ZLpWpjZi7F5+Pg+Vg==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.2.tgz", + "integrity": "sha512-3h3dLPWNgSsD4lQBJPb4f+kvdOSJHa5PjTYVsWHxLUzH4IFTJUAnmuWpw4KqyQ3NA5QCyhw4TWgxk3jRkQxEKg==", "cpu": [ "arm64" ], @@ -1571,9 +1572,9 @@ } }, "node_modules/@resvg/resvg-js-linux-x64-gnu": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.0.tgz", - "integrity": "sha512-n3zC8DWsvxC1AwxpKFclIPapDFibs5XdIRoV/mcIlxlh0vseW1F49b97F33BtJQRmlntsqqN6GMMqx8byB7B+Q==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.2.tgz", + "integrity": "sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==", "cpu": [ "x64" ], @@ -1586,9 +1587,9 @@ } }, "node_modules/@resvg/resvg-js-linux-x64-musl": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.0.tgz", - "integrity": "sha512-n4tasK1HOlAxdTEROgYA1aCfsEKk0UOFDNd/AQTTZlTmCbHKXPq+O8npaaKlwXquxlVK8vrkcWbksbiGqbCAcw==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.2.tgz", + "integrity": "sha512-UOf83vqTzoYQO9SZ0fPl2ZIFtNIz/Rr/y+7X8XRX1ZnBYsQ/tTb+cj9TE+KHOdmlTFBxhYzVkP2lRByCzqi4jQ==", "cpu": [ "x64" ], @@ -1601,9 +1602,9 @@ } }, "node_modules/@resvg/resvg-js-win32-arm64-msvc": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.0.tgz", - "integrity": "sha512-X2+EoBJFwDI5LDVb51Sk7ldnVLitMGr9WwU/i21i3fAeAXZb3hM16k67DeTy16OYkT2dk/RfU1tP1wG+rWbz2Q==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.2.tgz", + "integrity": "sha512-7C/RSgCa+7vqZ7qAbItfiaAWhyRSoD4l4BQAbVDqRRsRgY+S+hgS3in0Rxr7IorKUpGE69X48q6/nOAuTJQxeQ==", "cpu": [ "arm64" ], @@ -1616,9 +1617,9 @@ } }, "node_modules/@resvg/resvg-js-win32-ia32-msvc": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.0.tgz", - "integrity": "sha512-L7oevWjQoUgK5W1fCKn0euSVemhDXVhrjtwqpc7MwBKKimYeiOshO1Li1pa8bBt5PESahenhWgdB6lav9O0fEg==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.2.tgz", + "integrity": "sha512-har4aPAlvjnLcil40AC77YDIk6loMawuJwFINEM7n0pZviwMkMvjb2W5ZirsNOZY4aDbo5tLx0wNMREp5Brk+w==", "cpu": [ "ia32" ], @@ -1631,9 +1632,9 @@ } }, "node_modules/@resvg/resvg-js-win32-x64-msvc": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.0.tgz", - "integrity": "sha512-8lJlghb+Unki5AyKgsnFbRJwkEj9r1NpwyuBG8yEJiG1W9eEGl03R3I7bsVa3haof/3J1NlWf0rzSa1G++A2iw==", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.2.tgz", + "integrity": "sha512-ZXtYhtUr5SSaBrUDq7DiyjOFJqBVL/dOBN7N/qmi/pO0IgiWW/f/ue3nbvu9joWE5aAKDoIzy/CxsY0suwGosQ==", "cpu": [ "x64" ], @@ -2075,6 +2076,11 @@ "node": ">=4" } }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" + }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -3831,9 +3837,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", - "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", "engines": { "node": ">=8" } @@ -4556,6 +4562,22 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-type": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.0.0.tgz", + "integrity": "sha512-s7cxa7/leUWLiXO78DVVfBVse+milos9FitauDLG1pI7lNaJ2+5lzPnr2N24ym+84HVwJL6hVuGfgVE+ALvU8Q==", + "dependencies": { + "readable-web-to-node-stream": "^3.0.2", + "strtok3": "^7.0.0", + "token-types": "^5.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -6385,16 +6407,15 @@ } }, "node_modules/openai": { - "version": "4.14.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.14.2.tgz", - "integrity": "sha512-JGlm7mMC7J+cyQZnQMOH7daD9cBqqWqLtlBsejElEkgoehPrYfdyxSxIGICz5xk4YimbwI5FlLATSVojLtCKXQ==", + "version": "4.47.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.47.1.tgz", + "integrity": "sha512-WWSxhC/69ZhYWxH/OBsLEirIjUcfpQ5+ihkXKp06hmeYXgBBIUCa9IptMzYx6NdkiOCsSGYCnTIsxaic3AjRCQ==", "optional": true, "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7", @@ -6620,6 +6641,18 @@ "node": "*" } }, + "node_modules/peek-readable": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", + "integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/periscopic": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz", @@ -7475,6 +7508,21 @@ "node": ">= 6" } }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -7736,12 +7784,9 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", "bin": { "semver": "bin/semver.js" }, @@ -7840,42 +7885,42 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/sharp": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz", - "integrity": "sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ==", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz", + "integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==", "hasInstallScript": true, "dependencies": { "color": "^4.2.3", - "detect-libc": "^2.0.2", - "semver": "^7.5.4" + "detect-libc": "^2.0.3", + "semver": "^7.6.0" }, "engines": { - "libvips": ">=8.15.1", + "libvips": ">=8.15.2", "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.2", - "@img/sharp-darwin-x64": "0.33.2", - "@img/sharp-libvips-darwin-arm64": "1.0.1", - "@img/sharp-libvips-darwin-x64": "1.0.1", - "@img/sharp-libvips-linux-arm": "1.0.1", - "@img/sharp-libvips-linux-arm64": "1.0.1", - "@img/sharp-libvips-linux-s390x": "1.0.1", - "@img/sharp-libvips-linux-x64": "1.0.1", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.1", - "@img/sharp-libvips-linuxmusl-x64": "1.0.1", - "@img/sharp-linux-arm": "0.33.2", - "@img/sharp-linux-arm64": "0.33.2", - "@img/sharp-linux-s390x": "0.33.2", - "@img/sharp-linux-x64": "0.33.2", - "@img/sharp-linuxmusl-arm64": "0.33.2", - "@img/sharp-linuxmusl-x64": "0.33.2", - "@img/sharp-wasm32": "0.33.2", - "@img/sharp-win32-ia32": "0.33.2", - "@img/sharp-win32-x64": "0.33.2" + "@img/sharp-darwin-arm64": "0.33.3", + "@img/sharp-darwin-x64": "0.33.3", + "@img/sharp-libvips-darwin-arm64": "1.0.2", + "@img/sharp-libvips-darwin-x64": "1.0.2", + "@img/sharp-libvips-linux-arm": "1.0.2", + "@img/sharp-libvips-linux-arm64": "1.0.2", + "@img/sharp-libvips-linux-s390x": "1.0.2", + "@img/sharp-libvips-linux-x64": "1.0.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2", + "@img/sharp-libvips-linuxmusl-x64": "1.0.2", + "@img/sharp-linux-arm": "0.33.3", + "@img/sharp-linux-arm64": "0.33.3", + "@img/sharp-linux-s390x": "0.33.3", + "@img/sharp-linux-x64": "0.33.3", + "@img/sharp-linuxmusl-arm64": "0.33.3", + "@img/sharp-linuxmusl-x64": "0.33.3", + "@img/sharp-wasm32": "0.33.3", + "@img/sharp-win32-ia32": "0.33.3", + "@img/sharp-win32-x64": "0.33.3" } }, "node_modules/shebang-command": { @@ -8187,6 +8232,22 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/strtok3": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz", + "integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^5.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/sucrase": { "version": "3.32.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz", @@ -8709,6 +8770,22 @@ "node": ">=0.6" } }, + "node_modules/token-types": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", + "integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/totalist": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz", diff --git a/package.json b/package.json index c150794d57b..5952e57c9b8 100644 --- a/package.json +++ b/package.json @@ -57,14 +57,15 @@ "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", + "@resvg/resvg-js": "^2.6.2", "@playwright/browser-chromium": "^1.43.1", - "@resvg/resvg-js": "^2.6.0", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", "browser-image-resizer": "^2.4.1", "date-fns": "^2.29.3", "dotenv": "^16.0.3", "express": "^4.19.2", + "file-type": "^19.0.0", "handlebars": "^4.7.8", "highlight.js": "^11.7.0", "image-size": "^1.0.2", @@ -86,7 +87,7 @@ "satori-html": "^0.3.2", "sbd": "^1.0.19", "serpapi": "^1.1.1", - "sharp": "^0.33.2", + "sharp": "^0.33.3", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", @@ -98,6 +99,6 @@ "@google-cloud/vertexai": "^1.1.0", "aws4fetch": "^1.0.17", "cohere-ai": "^7.9.0", - "openai": "^4.14.2" + "openai": "^4.44.0" } } diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index c1ddb5a23b6..f76cd12a358 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -308,17 +308,17 @@ {#if message.files && message.files.length > 0}
{#each message.files as file} - - {#if file.length === 64} + + {#if file.type === "hash"} input from user {:else} input from user diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index 5805c98aeb3..a00133d94e1 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -92,7 +92,9 @@ (lastMessage.from === "user" || lastMessage.updates?.findIndex((u) => u.type === "status" && u.status === "error") !== -1); - $: sources = files.map((file) => file2base64(file)); + $: sources = files?.map((file) => + file2base64(file).then((value) => ({ type: "base64", value, mime: file.type })) + ); function onShare() { dispatch("share"); @@ -229,13 +231,13 @@
- {#if sources.length} + {#if sources?.length}
{#each sources as source, index} {#await source then src}
input content diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts index 4353c6b11a5..889bceeff0c 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropic.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropic.ts @@ -1,7 +1,9 @@ import { z } from "zod"; -import { env } from "$env/dynamic/private"; import type { Endpoint } from "../endpoints"; +import { env } from "$env/dynamic/private"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; +import { createImageProcessorOptionsValidator } from "../images"; +import { endpointMessagesToAnthropicMessages } from "./utils"; export const endpointAnthropicParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -11,12 +13,24 @@ export const endpointAnthropicParametersSchema = z.object({ apiKey: z.string().default(env.ANTHROPIC_API_KEY ?? "sk-"), defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), + multimodal: z + .object({ + image: createImageProcessorOptionsValidator({ + supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"], + preferredMimeType: "image/webp", + // The 4 / 3 compensates for the 33% increase in size when converting to base64 + maxSizeInMB: (5 / 4) * 3, + maxWidth: 4096, + maxHeight: 4096, + }), + }) + .default({}), }); export async function endpointAnthropic( input: z.input ): Promise { - const { baseURL, apiKey, model, defaultHeaders, defaultQuery } = + const { baseURL, apiKey, model, defaultHeaders, defaultQuery, multimodal } = endpointAnthropicParametersSchema.parse(input); let Anthropic; try { @@ -38,16 +52,6 @@ export async function endpointAnthropic( system = messages[0].content; } - const messagesFormatted = messages - .filter((message) => message.from !== "system") - .map((message) => ({ - role: message.from, - content: message.content, - })) as unknown as { - role: "user" | "assistant"; - content: string; - }[]; - let tokenId = 0; const parameters = { ...model.parameters, ...generateSettings }; @@ -55,7 +59,7 @@ export async function endpointAnthropic( return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: messagesFormatted, + messages: await endpointMessagesToAnthropicMessages(messages, multimodal), max_tokens: parameters?.max_new_tokens, temperature: parameters?.temperature, top_p: parameters?.top_p, diff --git a/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts b/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts index 620b6382128..a90dd627b21 100644 --- a/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts +++ b/src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts @@ -1,6 +1,8 @@ import { z } from "zod"; import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; +import { createImageProcessorOptionsValidator } from "../images"; +import { endpointMessagesToAnthropicMessages } from "./utils"; export const endpointAnthropicVertexParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -10,12 +12,24 @@ export const endpointAnthropicVertexParametersSchema = z.object({ projectId: z.string(), defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), + multimodal: z + .object({ + image: createImageProcessorOptionsValidator({ + supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"], + preferredMimeType: "image/webp", + // The 4 / 3 compensates for the 33% increase in size when converting to base64 + maxSizeInMB: (5 / 4) * 3, + maxWidth: 4096, + maxHeight: 4096, + }), + }) + .default({}), }); export async function endpointAnthropicVertex( input: z.input ): Promise { - const { region, projectId, model, defaultHeaders, defaultQuery } = + const { region, projectId, model, defaultHeaders, defaultQuery, multimodal } = endpointAnthropicVertexParametersSchema.parse(input); let AnthropicVertex; try { @@ -38,21 +52,11 @@ export async function endpointAnthropicVertex( system = messages[0].content; } - const messagesFormatted = messages - .filter((message) => message.from !== "system") - .map((message) => ({ - role: message.from, - content: message.content, - })) as unknown as { - role: "user" | "assistant"; - content: string; - }[]; - let tokenId = 0; return (async function* () { const stream = anthropic.messages.stream({ model: model.id ?? model.name, - messages: messagesFormatted, + messages: await endpointMessagesToAnthropicMessages(messages, multimodal), max_tokens: model.parameters?.max_new_tokens, temperature: model.parameters?.temperature, top_p: model.parameters?.top_p, diff --git a/src/lib/server/endpoints/anthropic/utils.ts b/src/lib/server/endpoints/anthropic/utils.ts new file mode 100644 index 00000000000..93e15d15bf9 --- /dev/null +++ b/src/lib/server/endpoints/anthropic/utils.ts @@ -0,0 +1,44 @@ +import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources"; +import { makeImageProcessor, type ImageProcessorOptions } from "../images"; +import type { EndpointMessage } from "../endpoints"; +import type { MessageFile } from "$lib/types/Message"; + +export async function fileToImageBlock( + file: MessageFile, + opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> +): Promise { + const processor = makeImageProcessor(opts); + const { image, mime } = await processor(file); + + return { + type: "image", + source: { + type: "base64", + media_type: mime, + data: image.toString("base64"), + }, + }; +} + +type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" }; + +export async function endpointMessagesToAnthropicMessages( + messages: EndpointMessage[], + multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> } +): Promise { + return await Promise.all( + messages + .filter((message): message is NonSystemMessage => message.from !== "system") + .map>(async (message) => { + return { + role: message.from, + content: [ + ...(await Promise.all( + (message.files ?? []).map((file) => fileToImageBlock(file, multimodal.image)) + )), + { type: "text", text: message.content }, + ], + }; + }) + ); +} diff --git a/src/lib/server/endpoints/endpoints.ts b/src/lib/server/endpoints/endpoints.ts index dddc1325bc5..09a78c345fb 100644 --- a/src/lib/server/endpoints/endpoints.ts +++ b/src/lib/server/endpoints/endpoints.ts @@ -1,4 +1,5 @@ import type { Conversation } from "$lib/types/Conversation"; +import type { Message } from "$lib/types/Message"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import { endpointTgi, endpointTgiParametersSchema } from "./tgi/endpointTgi"; import { z } from "zod"; @@ -25,9 +26,10 @@ import endpointLangserve, { endpointLangserveParametersSchema, } from "./langserve/endpointLangserve"; +export type EndpointMessage = Omit; // parameters passed when generating text export interface EndpointParameters { - messages: Omit[]; + messages: EndpointMessage[]; preprompt?: Conversation["preprompt"]; continueMessage?: boolean; // used to signal that the last message will be extended generateSettings?: Partial; diff --git a/src/lib/server/endpoints/images.ts b/src/lib/server/endpoints/images.ts new file mode 100644 index 00000000000..7d408814cf2 --- /dev/null +++ b/src/lib/server/endpoints/images.ts @@ -0,0 +1,211 @@ +import type { Sharp } from "sharp"; +import sharp from "sharp"; +import type { MessageFile } from "$lib/types/Message"; +import { z, type util } from "zod"; + +export interface ImageProcessorOptions { + supportedMimeTypes: TMimeType[]; + preferredMimeType: TMimeType; + maxSizeInMB: number; + maxWidth: number; + maxHeight: number; +} +export type ImageProcessor = (file: MessageFile) => Promise<{ + image: Buffer; + mime: TMimeType; +}>; + +export function createImageProcessorOptionsValidator( + defaults: ImageProcessorOptions +) { + return z + .object({ + supportedMimeTypes: z + .array( + z.enum([ + defaults.supportedMimeTypes[0], + ...defaults.supportedMimeTypes.slice(1), + ]) + ) + .default(defaults.supportedMimeTypes), + preferredMimeType: z + .enum([defaults.supportedMimeTypes[0], ...defaults.supportedMimeTypes.slice(1)]) + .default(defaults.preferredMimeType as util.noUndefined), + maxSizeInMB: z.number().positive().default(defaults.maxSizeInMB), + maxWidth: z.number().int().positive().default(defaults.maxWidth), + maxHeight: z.number().int().positive().default(defaults.maxHeight), + }) + .default(defaults); +} + +export function makeImageProcessor( + options: ImageProcessorOptions +): ImageProcessor { + return async (file) => { + const { supportedMimeTypes, preferredMimeType, maxSizeInMB, maxWidth, maxHeight } = options; + const { mime, value } = file; + + const buffer = Buffer.from(value, "base64"); + let sharpInst = sharp(buffer); + + const metadata = await sharpInst.metadata(); + if (!metadata) throw Error("Failed to read image metadata"); + const { width, height } = metadata; + if (width === undefined || height === undefined) throw Error("Failed to read image size"); + + const tooLargeInSize = width > maxWidth || height > maxHeight; + const tooLargeInBytes = buffer.byteLength > maxSizeInMB * 1000 * 1000; + + const outputMime = chooseMimeType(supportedMimeTypes, preferredMimeType, mime, { + preferSizeReduction: tooLargeInBytes, + }); + + // Resize if necessary + if (tooLargeInSize || tooLargeInBytes) { + const size = chooseImageSize({ + mime: outputMime, + width, + height, + maxWidth, + maxHeight, + maxSizeInMB, + }); + if (size.width !== width || size.height !== height) { + sharpInst = resizeImage(sharpInst, size.width, size.height); + } + } + + // Convert format if necessary + // We always want to convert the image when the file was too large in bytes + // so we can guarantee that ideal options are used, which are expected when + // choosing the image size + if (outputMime !== mime || tooLargeInBytes) { + sharpInst = convertImage(sharpInst, outputMime); + } + + const processedImage = await sharpInst.toBuffer(); + return { image: processedImage, mime: outputMime }; + }; +} + +const outputFormats = ["png", "jpeg", "webp", "avif", "tiff", "gif"] as const; +type OutputImgFormat = (typeof outputFormats)[number]; +const isOutputFormat = (format: string): format is (typeof outputFormats)[number] => + outputFormats.includes(format as OutputImgFormat); + +export function convertImage(sharpInst: Sharp, outputMime: string): Sharp { + const [type, format] = outputMime.split("/"); + if (type !== "image") throw Error(`Requested non-image mime type: ${outputMime}`); + if (!isOutputFormat(format)) { + throw Error(`Requested to convert to an unsupported format: ${format}`); + } + + return sharpInst[format](); +} + +// heic/heif requires proprietary license +// TODO: blocking heif may be incorrect considering it also supports av1, so we should instead +// detect the compression method used via sharp().metadata().compression +// TODO: consider what to do about animated formats: apng, gif, animated webp, ... +const blocklistedMimes = ["image/heic", "image/heif"]; + +/** Sorted from largest to smallest */ +const mimesBySizeDesc = [ + "image/png", + "image/tiff", + "image/gif", + "image/jpeg", + "image/webp", + "image/avif", +]; + +/** + * Defaults to preferred format or uses existing mime if supported + * When preferSizeReduction is true, it will choose the smallest format that is supported + **/ +function chooseMimeType( + supportedMimes: T, + preferredMime: string, + mime: string, + { preferSizeReduction }: { preferSizeReduction: boolean } +): T[number] { + if (!supportedMimes.includes(preferredMime)) { + const supportedMimesStr = supportedMimes.join(", "); + throw Error( + `Preferred format "${preferredMime}" not found in supported mimes: ${supportedMimesStr}` + ); + } + + const [type] = mime.split("/"); + if (type !== "image") throw Error(`Received non-image mime type: ${mime}`); + + if (supportedMimes.includes(mime) && !preferSizeReduction) return mime; + + if (blocklistedMimes.includes(mime)) throw Error(`Received blocklisted mime type: ${mime}`); + + const smallestMime = mimesBySizeDesc.findLast((m) => supportedMimes.includes(m)); + return smallestMime ?? preferredMime; +} + +interface ImageSizeOptions { + mime: string; + width: number; + height: number; + maxWidth: number; + maxHeight: number; + maxSizeInMB: number; +} + +/** Resizes the image to fit within the specified size in MB by guessing the output size */ +export function chooseImageSize({ + mime, + width, + height, + maxWidth, + maxHeight, + maxSizeInMB, +}: ImageSizeOptions): { width: number; height: number } { + const biggestDiscrepency = Math.max(1, width / maxWidth, height / maxHeight); + + let selectedWidth = Math.ceil(width / biggestDiscrepency); + let selectedHeight = Math.ceil(height / biggestDiscrepency); + + do { + const estimatedSize = estimateImageSizeInBytes(mime, selectedWidth, selectedHeight); + if (estimatedSize < maxSizeInMB * 1024 * 1024) { + return { width: selectedWidth, height: selectedHeight }; + } + selectedWidth = Math.floor(selectedWidth / 1.1); + selectedHeight = Math.floor(selectedHeight / 1.1); + } while (selectedWidth > 1 && selectedHeight > 1); + + throw Error(`Failed to resize image to fit within ${maxSizeInMB}MB`); +} + +const mimeToCompressionRatio: Record = { + "image/png": 1 / 2, + "image/jpeg": 1 / 10, + "image/webp": 1 / 4, + "image/avif": 1 / 5, + "image/tiff": 1, + "image/gif": 1 / 5, +}; + +/** + * Guesses the side of an image in MB based on its format and dimensions + * Should guess the worst case + **/ +function estimateImageSizeInBytes(mime: string, width: number, height: number): number { + const compressionRatio = mimeToCompressionRatio[mime]; + if (!compressionRatio) throw Error(`Unsupported image format: ${mime}`); + + const bitsPerPixel = 32; // Assuming 32-bit color depth for 8-bit R G B A + const bytesPerPixel = bitsPerPixel / 8; + const uncompressedSize = width * height * bytesPerPixel; + + return uncompressedSize * compressionRatio; +} + +export function resizeImage(sharpInst: Sharp, maxWidth: number, maxHeight: number): Sharp { + return sharpInst.resize({ width: maxWidth, height: maxHeight, fit: "inside" }); +} diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index 84a221bdfc0..7611ed246f0 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -6,6 +6,10 @@ import type { ChatCompletionCreateParamsStreaming } from "openai/resources/chat/ import { buildPrompt } from "$lib/buildPrompt"; import { env } from "$env/dynamic/private"; import type { Endpoint } from "../endpoints"; +import type OpenAI from "openai"; +import { createImageProcessorOptionsValidator, makeImageProcessor } from "../images"; +import type { MessageFile } from "$lib/types/Message"; +import type { EndpointMessage } from "../endpoints"; export const endpointOAIParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -19,13 +23,41 @@ export const endpointOAIParametersSchema = z.object({ defaultHeaders: z.record(z.string()).optional(), defaultQuery: z.record(z.string()).optional(), extraBody: z.record(z.any()).optional(), + multimodal: z + .object({ + image: createImageProcessorOptionsValidator({ + supportedMimeTypes: [ + "image/png", + "image/jpeg", + "image/webp", + "image/avif", + "image/tiff", + "image/gif", + ], + preferredMimeType: "image/webp", + maxSizeInMB: Infinity, + maxWidth: 4096, + maxHeight: 4096, + }), + }) + .default({}), }); export async function endpointOai( input: z.input ): Promise { - const { baseURL, apiKey, completion, model, defaultHeaders, defaultQuery, extraBody } = - endpointOAIParametersSchema.parse(input); + const { + baseURL, + apiKey, + completion, + model, + defaultHeaders, + defaultQuery, + multimodal, + extraBody, + } = endpointOAIParametersSchema.parse(input); + + /* eslint-disable-next-line no-shadow */ let OpenAI; try { OpenAI = (await import("openai")).OpenAI; @@ -40,6 +72,8 @@ export async function endpointOai( defaultQuery, }); + const imageProcessor = makeImageProcessor(multimodal.image); + if (completion === "completions") { return async ({ messages, preprompt, continueMessage, generateSettings }) => { const prompt = await buildPrompt({ @@ -69,10 +103,8 @@ export async function endpointOai( }; } else if (completion === "chat_completions") { return async ({ messages, preprompt, generateSettings }) => { - let messagesOpenAI = messages.map((message) => ({ - role: message.from, - content: message.content, - })); + let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = + await prepareMessages(messages, imageProcessor); if (messagesOpenAI?.[0]?.role !== "system") { messagesOpenAI = [{ role: "system", content: "" }, ...messagesOpenAI]; @@ -104,3 +136,39 @@ export async function endpointOai( throw new Error("Invalid completion type"); } } + +async function prepareMessages( + messages: EndpointMessage[], + imageProcessor: ReturnType +): Promise { + return Promise.all( + messages.map(async (message) => { + if (message.from === "user") { + return { + role: message.from, + content: [ + ...(await prepareFiles(imageProcessor, message.files ?? [])), + { type: "text", text: message.content }, + ], + }; + } + return { + role: message.from, + content: message.content, + }; + }) + ); +} + +async function prepareFiles( + imageProcessor: ReturnType, + files: MessageFile[] +): Promise { + const processedFiles = await Promise.all(files.map(imageProcessor)); + return processedFiles.map((file) => ({ + type: "image_url" as const, + image_url: { + url: `data:${file.mime};base64,${file.image.toString("base64")}`, + }, + })); +} diff --git a/src/lib/server/endpoints/preprocessMessages.ts b/src/lib/server/endpoints/preprocessMessages.ts new file mode 100644 index 00000000000..e38dd9abc26 --- /dev/null +++ b/src/lib/server/endpoints/preprocessMessages.ts @@ -0,0 +1,56 @@ +import type { Message } from "$lib/types/Message"; +import { format } from "date-fns"; +import type { EndpointMessage } from "./endpoints"; +import { downloadFile } from "../files/downloadFile"; +import type { ObjectId } from "mongodb"; + +export async function preprocessMessages( + messages: Message[], + webSearch: Message["webSearch"], + convId: ObjectId +): Promise { + return Promise.resolve(messages) + .then((msgs) => addWebSearchContext(msgs, webSearch)) + .then((msgs) => downloadFiles(msgs, convId)); +} + +function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) { + const webSearchContext = webSearch?.contextSources + .map(({ context }) => context.trim()) + .join("\n\n----------\n\n"); + + // No web search context available, skip + if (!webSearch || !webSearchContext?.trim()) return messages; + // No messages available, skip + if (messages.length === 0) return messages; + + const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? ""; + const previousQuestions = messages + .filter((el) => el.from === "user") + .slice(0, -1) + .map((el) => el.content); + const currentDate = format(new Date(), "MMMM d, yyyy"); + + const finalMessage = { + ...messages[messages.length - 1], + content: `I searched the web using the query: ${webSearch.searchQuery}. +Today is ${currentDate} and here are the results: +===================== +${webSearchContext} +===================== +${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""} +Answer the question: ${lastQuestion}`, + }; + + return [...messages.slice(0, -1), finalMessage]; +} + +async function downloadFiles(messages: Message[], convId: ObjectId): Promise { + return Promise.all( + messages.map>((message) => + Promise.all((message.files ?? []).map((file) => downloadFile(file.value, convId))).then( + (files) => ({ ...message, files }) + ) + ) + ); +} diff --git a/src/lib/server/endpoints/tgi/endpointTgi.ts b/src/lib/server/endpoints/tgi/endpointTgi.ts index aed06739722..53f69ca1e4b 100644 --- a/src/lib/server/endpoints/tgi/endpointTgi.ts +++ b/src/lib/server/endpoints/tgi/endpointTgi.ts @@ -1,8 +1,13 @@ import { env } from "$env/dynamic/private"; import { buildPrompt } from "$lib/buildPrompt"; import { textGenerationStream } from "@huggingface/inference"; -import type { Endpoint } from "../endpoints"; +import type { Endpoint, EndpointMessage } from "../endpoints"; import { z } from "zod"; +import { + createImageProcessorOptionsValidator, + makeImageProcessor, + type ImageProcessor, +} from "../images"; export const endpointTgiParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -11,14 +16,32 @@ export const endpointTgiParametersSchema = z.object({ url: z.string().url(), accessToken: z.string().default(env.HF_TOKEN ?? env.HF_ACCESS_TOKEN), authorization: z.string().optional(), + multimodal: z + .object({ + // Assumes IDEFICS + image: createImageProcessorOptionsValidator({ + supportedMimeTypes: ["image/jpeg", "image/webp"], + preferredMimeType: "image/webp", + maxSizeInMB: 5, + maxWidth: 224, + maxHeight: 224, + }), + }) + .default({}), }); export function endpointTgi(input: z.input): Endpoint { - const { url, accessToken, model, authorization } = endpointTgiParametersSchema.parse(input); + const { url, accessToken, model, authorization, multimodal } = + endpointTgiParametersSchema.parse(input); + const imageProcessor = makeImageProcessor(multimodal.image); return async ({ messages, preprompt, continueMessage, generateSettings }) => { + const messagesWithResizedFiles = await Promise.all( + messages.map((message) => prepareMessage(message, imageProcessor)) + ); + const prompt = await buildPrompt({ - messages, + messages: messagesWithResizedFiles, preprompt, model, continueMessage, @@ -48,4 +71,23 @@ export function endpointTgi(input: z.input): }; } -export default endpointTgi; +const whiteImage = { + mime: "image/png", + image: Buffer.from( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==", + "base64" + ), +}; + +async function prepareMessage( + message: EndpointMessage, + imageProcessor: ImageProcessor +): Promise { + const files = await Promise.all(message.files?.map(imageProcessor) ?? [whiteImage]); + const markdowns = files.map( + (file) => `![](data:${file.mime};base64,${file.image.toString("base64")})` + ); + const content = message.content + "\n" + markdowns.join("\n "); + + return { ...message, content }; +} diff --git a/src/lib/server/files/downloadFile.ts b/src/lib/server/files/downloadFile.ts index 91b430fc5d8..dac5bbda848 100644 --- a/src/lib/server/files/downloadFile.ts +++ b/src/lib/server/files/downloadFile.ts @@ -2,15 +2,16 @@ import { error } from "@sveltejs/kit"; import { collections } from "$lib/server/database"; import type { Conversation } from "$lib/types/Conversation"; import type { SharedConversation } from "$lib/types/SharedConversation"; +import type { MessageFile } from "$lib/types/Message"; export async function downloadFile( sha256: string, convId: Conversation["_id"] | SharedConversation["_id"] -) { +): Promise { const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` }); let mime = ""; - const content = await fileId.next().then(async (file) => { + const buffer = await fileId.next().then(async (file) => { if (!file) { throw error(404, "File not found"); } @@ -32,5 +33,5 @@ export async function downloadFile( return fileBuffer; }); - return { content, mime }; + return { type: "base64", value: buffer.toString("base64"), mime }; } diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts index 34452245741..339a4b4ea85 100644 --- a/src/lib/server/files/uploadFile.ts +++ b/src/lib/server/files/uploadFile.ts @@ -1,21 +1,27 @@ import type { Conversation } from "$lib/types/Conversation"; +import type { MessageFile } from "$lib/types/Message"; import { sha256 } from "$lib/utils/sha256"; +import { fileTypeFromBuffer } from "file-type"; import { collections } from "$lib/server/database"; -export async function uploadFile(file: Blob, conv: Conversation): Promise { +export async function uploadFile(file: File, conv: Conversation): Promise { const sha = await sha256(await file.text()); + const buffer = await file.arrayBuffer(); + + // Attempt to detect the mime type of the file, fallback to the uploaded mime + const mime = await fileTypeFromBuffer(buffer).then((fileType) => fileType?.mime ?? file.type); const upload = collections.bucket.openUploadStream(`${conv._id}-${sha}`, { - metadata: { conversation: conv._id.toString(), mime: "image/jpeg" }, + metadata: { conversation: conv._id.toString(), mime }, }); upload.write((await file.arrayBuffer()) as unknown as Buffer); upload.end(); - // only return the filename when upload throws a finish event or a 10s time out occurs + // only return the filename when upload throws a finish event or a 20s time out occurs return new Promise((resolve, reject) => { - upload.once("finish", () => resolve(sha)); + upload.once("finish", () => resolve({ type: "hash", value: sha, mime: file.type })); upload.once("error", reject); - setTimeout(() => reject(new Error("Upload timed out")), 10000); + setTimeout(() => reject(new Error("Upload timed out")), 20_000); }); } diff --git a/src/lib/server/generateFromDefaultEndpoint.ts b/src/lib/server/generateFromDefaultEndpoint.ts index 428e94a06f9..4f798f90f51 100644 --- a/src/lib/server/generateFromDefaultEndpoint.ts +++ b/src/lib/server/generateFromDefaultEndpoint.ts @@ -1,12 +1,12 @@ import { smallModel } from "$lib/server/models"; -import type { Conversation } from "$lib/types/Conversation"; +import type { EndpointMessage } from "./endpoints/endpoints"; export async function generateFromDefaultEndpoint({ messages, preprompt, generateSettings, }: { - messages: Omit[]; + messages: EndpointMessage[]; preprompt?: string; generateSettings?: Record; }): Promise { diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index fb9604b625a..78f0383331a 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -3,7 +3,7 @@ import type { ChatTemplateInput } from "$lib/types/Template"; import { compileTemplate } from "$lib/utils/template"; import { z } from "zod"; import endpoints, { endpointSchema, type Endpoint } from "./endpoints/endpoints"; -import endpointTgi from "./endpoints/tgi/endpointTgi"; +import { endpointTgi } from "./endpoints/tgi/endpointTgi"; import { sum } from "$lib/utils/sum"; import { embeddingModels, validateEmbeddingModelByName } from "./embeddingModels"; diff --git a/src/lib/server/preprocessMessages.ts b/src/lib/server/preprocessMessages.ts deleted file mode 100644 index c5ff2c585c0..00000000000 --- a/src/lib/server/preprocessMessages.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { Conversation } from "$lib/types/Conversation"; -import type { Message } from "$lib/types/Message"; -import { format } from "date-fns"; -import { downloadFile } from "./files/downloadFile"; -import { logger } from "$lib/server/logger"; - -export async function preprocessMessages( - messages: Message[], - webSearch: Message["webSearch"], - multimodal: boolean, - id: Conversation["_id"] -): Promise { - return await Promise.all( - structuredClone(messages).map(async (message, idx) => { - const webSearchContext = webSearch?.contextSources - .map(({ context }) => context.trim()) - .join("\n\n----------\n\n"); - - // start by adding websearch to the last message - if (idx === messages.length - 1 && webSearch && webSearchContext?.trim()) { - const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? ""; - const previousQuestions = messages - .filter((el) => el.from === "user") - .slice(0, -1) - .map((el) => el.content); - const currentDate = format(new Date(), "MMMM d, yyyy"); - - message.content = `I searched the web using the query: ${webSearch.searchQuery}. -Today is ${currentDate} and here are the results: -===================== -${webSearchContext} -===================== -${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""} -Answer the question: ${lastQuestion}`; - } - // handle files if model is multimodal - if (multimodal) { - if (message.files && message.files.length > 0) { - const markdowns = await Promise.all( - message.files.map(async (hash) => { - try { - const { content: image, mime } = await downloadFile(hash, id); - const b64 = image.toString("base64"); - return `![](data:${mime};base64,${b64})})`; - } catch (e) { - logger.error(e); - } - }) - ); - message.content += markdowns.join("\n "); - } else { - // if no image, append an empty white image - message.content += - "\n![](data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==)"; - } - } - - return message; - }) - ); -} diff --git a/src/lib/server/summarize.ts b/src/lib/server/summarize.ts index 4cef6174dc9..033e1ae7033 100644 --- a/src/lib/server/summarize.ts +++ b/src/lib/server/summarize.ts @@ -1,6 +1,6 @@ import { env } from "$env/dynamic/private"; import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint"; -import type { Message } from "$lib/types/Message"; +import type { EndpointMessage } from "./endpoints/endpoints"; import { logger } from "$lib/server/logger"; export async function summarize(prompt: string) { @@ -8,7 +8,7 @@ export async function summarize(prompt: string) { return prompt.split(/\s+/g).slice(0, 5).join(" "); } - const messages: Array> = [ + const messages: Array = [ { from: "user", content: "Who is the president of Gabon?" }, { from: "assistant", content: "🇬🇦 President of Gabon" }, { from: "user", content: "Who is Julien Chaumond?" }, diff --git a/src/lib/server/websearch/search/generateQuery.ts b/src/lib/server/websearch/search/generateQuery.ts index b08a3df6717..c71841a8c17 100644 --- a/src/lib/server/websearch/search/generateQuery.ts +++ b/src/lib/server/websearch/search/generateQuery.ts @@ -1,5 +1,6 @@ import type { Message } from "$lib/types/Message"; import { format } from "date-fns"; +import type { EndpointMessage } from "../../endpoints/endpoints"; import { generateFromDefaultEndpoint } from "../../generateFromDefaultEndpoint"; export async function generateQuery(messages: Message[]) { @@ -9,7 +10,7 @@ export async function generateQuery(messages: Message[]) { const lastMessage = userMessages.slice(-1)[0]; - const convQuery: Array> = [ + const convQuery: Array = [ { from: "user", content: `Previous Questions: diff --git a/src/lib/types/Message.ts b/src/lib/types/Message.ts index 68f9d6b271c..6791164febf 100644 --- a/src/lib/types/Message.ts +++ b/src/lib/types/Message.ts @@ -11,7 +11,11 @@ export type Message = Partial & { webSearchId?: WebSearch["_id"]; // legacy version webSearch?: WebSearch; score?: -1 | 0 | 1; - files?: string[]; // can contain either the hash of the file or the b64 encoded image data on the client side when uploading + /** + * Either contains the base64 encoded image data + * or the hash of the file stored on the server + **/ + files?: MessageFile[]; interrupted?: boolean; // needed for conversation trees @@ -20,3 +24,9 @@ export type Message = Partial & { // goes one level deep children?: Message["id"][]; }; + +export type MessageFile = { + type: "hash" | "base64"; + value: string; + mime: string; +}; diff --git a/src/lib/utils/messageUpdates.ts b/src/lib/utils/messageUpdates.ts index 82b8bb9a2ca..83929255e01 100644 --- a/src/lib/utils/messageUpdates.ts +++ b/src/lib/utils/messageUpdates.ts @@ -1,3 +1,4 @@ +import type { MessageFile } from "$lib/types/Message"; import type { MessageUpdate, TextStreamUpdate } from "$lib/types/MessageUpdate"; type MessageUpdateRequestOptions = { @@ -7,7 +8,7 @@ type MessageUpdateRequestOptions = { isRetry: boolean; isContinue: boolean; webSearch: boolean; - files?: string[]; + files?: MessageFile[]; }; export async function fetchMessageUpdates( conversationId: string, diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index 04888829a06..806b30bd772 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -75,20 +75,10 @@ loading = true; pending = true; - const module = await import("browser-image-resizer"); - // currently, only IDEFICS is supported by TGI - // the size of images is hardcoded to 224x224 in TGI - // this will need to be configurable when support for more models is added - const resizedImages = await Promise.all( - files.map(async (file) => { - return await module - .readAndCompressImage(file, { - maxHeight: 224, - maxWidth: 224, - quality: 1, - }) - .then(async (el) => await file2base64(el as File)); - }) + const base64Files = await Promise.all( + (files ?? []).map((file) => + file2base64(file).then((value) => ({ type: "base64" as const, value, mime: file.type })) + ) ); let messageToWriteToId: Message["id"] | undefined = undefined; @@ -120,7 +110,11 @@ messages, rootMessageId: data.rootMessageId, }, - { from: "user", content: prompt }, + { + from: "user", + content: prompt, + files: messageToRetry.files, + }, messageId ); messageToWriteToId = addChildren( @@ -128,7 +122,7 @@ messages, rootMessageId: data.rootMessageId, }, - { from: "assistant", content: "", files: resizedImages }, + { from: "assistant", content: "" }, newUserMessageId ); } else if (messageToRetry?.from === "assistant") { @@ -154,7 +148,7 @@ { from: "user", content: prompt ?? "", - files: resizedImages, + files: base64Files, createdAt: new Date(), updatedAt: new Date(), }, @@ -181,6 +175,7 @@ } messages = [...messages]; + const userMessage = messages.find((message) => message.id === messageId); const messageToWriteTo = messages.find((message) => message.id === messageToWriteToId); if (!messageToWriteTo) { throw new Error("Message to write to not found"); @@ -198,7 +193,7 @@ isRetry, isContinue, webSearch: !hasAssistant && $webSearchParameters.useSearch, - files: isRetry ? undefined : resizedImages, + files: isRetry ? userMessage?.files : base64Files, }, messageUpdatesAbortController.signal ).catch((err) => { diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index ebbedb945f9..0dc4913fe57 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -13,14 +13,13 @@ import { runWebSearch } from "$lib/server/websearch/runWebSearch"; import { AbortedGenerations } from "$lib/server/abortedGenerations"; import { summarize } from "$lib/server/summarize"; import { uploadFile } from "$lib/server/files/uploadFile"; -import sizeof from "image-size"; import type { Assistant } from "$lib/types/Assistant"; import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation"; import { isMessageId } from "$lib/utils/tree/isMessageId"; import { buildSubtree } from "$lib/utils/tree/buildSubtree.js"; import { addChildren } from "$lib/utils/tree/addChildren.js"; import { addSibling } from "$lib/utils/tree/addSibling.js"; -import { preprocessMessages } from "$lib/server/preprocessMessages.js"; +import { preprocessMessages } from "$lib/server/endpoints/preprocessMessages.js"; import { usageLimits } from "$lib/server/usageLimits"; import { isURLLocal } from "$lib/server/isURLLocal.js"; import { logger } from "$lib/server/logger.js"; @@ -134,7 +133,7 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: isRetry, is_continue: isContinue, web_search: webSearch, - files: b64files, + files: inputFiles, } = z .object({ id: z.string().uuid().refine(isMessageId).optional(), // parent message id to append to for a normal message, or the message id for a retry/continue @@ -147,44 +146,43 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: z.optional(z.boolean()), is_continue: z.optional(z.boolean()), web_search: z.optional(z.boolean()), - files: z.optional(z.array(z.string())), + files: z.optional( + z.array( + z.object({ + type: z.literal("base64").or(z.literal("hash")), + value: z.string(), + mime: z.string(), + }) + ) + ), }) .parse(json); if (usageLimits?.messageLength && (newPrompt?.length ?? 0) > usageLimits.messageLength) { throw error(400, "Message too long."); } - // files is an array of base64 strings encoding Blob objects - // we need to convert this array to an array of File objects - const files = b64files?.map((file) => { - const blob = Buffer.from(file, "base64"); - return new File([blob], "image.png"); - }); + // each file is either: + // base64 string requiring upload to the server + // hash pointing to an existing file + const hashFiles = inputFiles?.filter((file) => file.type === "hash") ?? []; + const b64Files = + inputFiles + ?.filter((file) => file.type !== "hash") + .map((file) => { + const blob = Buffer.from(file.value, "base64"); + return new File([blob], "file", { type: file.mime }); + }) ?? []; // check sizes - if (files) { - const filechecks = await Promise.all( - files.map(async (file) => { - const dimensions = sizeof(Buffer.from(await file.arrayBuffer())); - return ( - file.size > 2 * 1024 * 1024 || - (dimensions.width ?? 0) > 224 || - (dimensions.height ?? 0) > 224 - ); - }) - ); - - if (filechecks.some((check) => check)) { - throw error(413, "File too large, should be <2MB and 224x224 max."); - } + // todo: make configurable + if (b64Files.some((file) => file.size > 10 * 1024 * 1024)) { + throw error(413, "File too large, should be <10MB"); } - let hashes: undefined | string[]; - - if (files) { - hashes = await Promise.all(files.map(async (file) => await uploadFile(file, conv))); - } + const uploadedFiles = await Promise.all(b64Files.map((file) => uploadFile(file, conv))).then( + (files) => [...files, ...hashFiles] + ); // we will append tokens to the content of this message let messageToWriteToId: Message["id"] | undefined = undefined; @@ -216,7 +214,13 @@ export async function POST({ request, locals, params, getClientAddress }) { // add a children to that sibling, where we can write to const newUserMessageId = addSibling( conv, - { from: "user", content: newPrompt, createdAt: new Date(), updatedAt: new Date() }, + { + from: "user", + content: newPrompt, + files: uploadedFiles, + createdAt: new Date(), + updatedAt: new Date(), + }, messageId ); messageToWriteToId = addChildren( @@ -224,7 +228,6 @@ export async function POST({ request, locals, params, getClientAddress }) { { from: "assistant", content: "", - files: hashes, createdAt: new Date(), updatedAt: new Date(), }, @@ -250,7 +253,7 @@ export async function POST({ request, locals, params, getClientAddress }) { { from: "user", content: newPrompt ?? "", - files: hashes, + files: uploadedFiles, createdAt: new Date(), updatedAt: new Date(), }, @@ -411,10 +414,9 @@ export async function POST({ request, locals, params, getClientAddress }) { } // inject websearch result & optionally images into the messages - const processedMessages = await preprocessMessages( + const processedMessages = preprocessMessages( messagesForPrompt, messageToWriteTo.webSearch, - model.multimodal, convId ); @@ -429,7 +431,7 @@ export async function POST({ request, locals, params, getClientAddress }) { try { const endpoint = await model.getEndpoint(); for await (const output of await endpoint({ - messages: processedMessages, + messages: await processedMessages, preprompt, continueMessage: isContinue, generateSettings: assistant?.generateSettings, diff --git a/src/routes/conversation/[id]/output/[sha256]/+server.ts b/src/routes/conversation/[id]/output/[sha256]/+server.ts index 79ae37b7585..5b3ae84dcf8 100644 --- a/src/routes/conversation/[id]/output/[sha256]/+server.ts +++ b/src/routes/conversation/[id]/output/[sha256]/+server.ts @@ -39,9 +39,9 @@ export const GET: RequestHandler = async ({ locals, params }) => { } } - const { content, mime } = await downloadFile(sha256, params.id); + const { value, mime } = await downloadFile(sha256, params.id); - return new Response(content, { + return new Response(Buffer.from(value, "base64"), { headers: { "Content-Type": mime ?? "application/octet-stream", },