Scale and decimate images as needed in browser

When the user tries to embed an image file that's very large, or is in a
file format that isn't supported by the backend (e.g. webp), the js code
will now convert it to jpeg and lower the quality as needed.

llama.cpp/main/main.1.asc

@@ -976,7 +976,7 @@
                    -n 500 \
                    --no-display-prompt 2>/dev/null
 
-       Here's how you can use llamafile to describe a jpg/png/gif/bmp image:
+       Here's how you can use llamafile to describe a jpg/png/gif image:
 
              llamafile --temp 0 \
                --image lemurs.jpg \

llamafile/chatbot_eval.cpp

@@ -128,7 +128,7 @@ bool eval_string(std::string_view s, bool add_special, bool parse_special) {
         size_t end = uri.parse(s.substr(pos + 5));
         if (end == std::string_view::npos)
             continue;
-        if (!uri.mime.starts_with("image/"))
+        if (!lf::startscasewith(uri.mime, "image/"))
             continue;
         std::string image;
         try {

llamafile/datauri.cpp

@@ -16,8 +16,9 @@
 // limitations under the License.
 
 #include "datauri.h"
-
 #include "llama.cpp/base64.h"
+#include "llamafile/string.h"
+#include <cctype>
 
 // See RFC2045 (MIME)
 static const char kMimeToken[] = {
@@ -273,14 +274,14 @@ std::string DataUri::decode() {
 
 bool DataUri::has_param(std::string_view attribute) {
     for (const auto &param : params)
-        if (param.first == attribute)
+        if (!lf::strcasecmp(param.first, attribute))
             return true;
     return false;
 }
 
 std::string_view DataUri::get_param(std::string_view attribute) {
     for (const auto &param : params)
-        if (param.first == attribute)
+        if (!lf::strcasecmp(param.first, attribute))
             return param.second;
     return "";
 }

llamafile/datauri_test.cpp

@@ -17,7 +17,7 @@
 
 #include "datauri.h"
 #include "image.h"
-
+#include "string.h"
 #include <string>
 #include <vector>
 
@@ -30,7 +30,7 @@ void rfc2397_example1() {
         exit(1);
     if (uri.mime != "text/plain")
         exit(2);
-    if (uri.get_param("charset") != "US-ASCII")
+    if (lf::strcasecmp(uri.get_param("charset"), "US-ASCII"))
         exit(3);
     if (uri.decode() != "A brief note")
         exit(4);

llamafile/server/atomize.cpp

@@ -22,6 +22,7 @@
 #include "llamafile/image.h"
 #include "llamafile/llama.h"
 #include "llamafile/server/image.h"
+#include "llamafile/string.h"
 #include <string>
 #include <vector>
 
@@ -58,7 +59,7 @@ atomize(const llama_model* model,
         size_t end = uri.parse(s.substr(i));
         if (end == std::string_view::npos)
             continue;
-        if (!uri.mime.starts_with("image/"))
+        if (!startscasewith(uri.mime, "image/"))
             continue;
         std::string image;
         try {

llamafile/server/www/chatbot.js

@@ -217,19 +217,69 @@ function onPaste(e) {
   }
 }
 
-function onFile(file) {
-  if (!file.type.startsWith('image/')) {
-    console.warn('Only image files are supported');
-    return;
+// fixes image data uri
+// - convert to jpg if it's not jpg/png/gif
+// - reduce quality and/or downscale if too big
+async function fixImageDataUri(dataUri, maxLength = 1024 * 1024) {
+  const mimeMatch = dataUri.match(/^data:([^;,]+)/);
+  if (!mimeMatch)
+    throw new Error('bad image data uri');
+  const mimeType = mimeMatch[1].toLowerCase();
+  const supported = ['image/jpeg', 'image/jpg', 'image/png', 'image/gif'];
+  if (supported.includes(mimeType))
+    if (dataUri.length <= maxLength)
+      return dataUri;
+  const lossless = ['image/png', 'image/gif'];
+  const quality = lossless.includes(mimeType) ? 0.92 : 0.8;
+  function createScaledCanvas(img, scale) {
+    const canvas = document.createElement('canvas');
+    canvas.width = Math.floor(img.width * scale);
+    canvas.height = Math.floor(img.height * scale);
+    const ctx = canvas.getContext('2d');
+    ctx.imageSmoothingEnabled = true;
+    ctx.imageSmoothingQuality = 'high';
+    ctx.fillStyle = 'white';  // in case of transparency
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
+    ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+    return canvas;
   }
-  if (file.size > 1 * 1024 * 1024) {
-    console.warn('Image is larger than 1mb');
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = async () => {
+      let scale = 1.0;
+      let attempts = 0;
+      const maxAttempts = 5;
+      const initialCanvas = createScaledCanvas(img, scale);
+      let result = initialCanvas.toDataURL('image/jpeg', quality);
+      while (result.length > maxLength && attempts < maxAttempts) {
+        attempts++;
+        scale *= 0.7071;
+        const scaledCanvas = createScaledCanvas(img, scale);
+        result = scaledCanvas.toDataURL('image/jpeg', quality);
+        result.length = result.length;
+      }
+      if (result.length <= maxLength) {
+        resolve(result);
+      } else {
+        reject(new Error(`Could not reduce image to ${(maxLength/1024).toFixed(2)}kb after ${maxAttempts} attempts`));
+      }
+    };
+    img.onerror = () => {
+      reject(new Error('Failed to load image from data URI'));
+    };
+    img.src = dataUri;
+  });
+}
+
+async function onFile(file) {
+  if (!file.type.toLowerCase().startsWith('image/')) {
+    console.warn('Only image files are supported');
     return;
   }
   const reader = new FileReader();
-  reader.onloadend = function() {
+  reader.onloadend = async function() {
     const description = file.name;
-    const realDataUri = reader.result;
+    const realDataUri = await fixImageDataUri(reader.result);
     const fakeDataUri = 'data:,placeholder/' + generateId();
     uploadedFiles.push([fakeDataUri, realDataUri]);
     insertText(chatInput, `![${description}](${fakeDataUri})`);

llamafile/string.cpp

@@ -16,22 +16,42 @@
 // limitations under the License.
 
 #include "string.h"
-
 #include <cctype>
 #include <cosmo.h>
 #include <cstdio>
 #include <string>
+#include <utility>
 #include <vector>
 
 namespace lf {
 
-std::string tolower(const std::string_view s) {
+std::string tolower(const std::string_view &s) {
     std::string b;
     for (char c : s)
         b += std::tolower(c);
     return b;
 }
 
+int strcasecmp(const std::string_view &a, const std::string_view &b) {
+    size_t n = std::min(a.size(), b.size());
+    for (size_t i = 0; i < n; ++i) {
+        unsigned char al = std::tolower(a[i] & 255);
+        unsigned char bl = std::tolower(b[i] & 255);
+        if (al != bl)
+            return (al > bl) - (al < bl);
+    }
+    return (a.size() > b.size()) - (a.size() < b.size());
+}
+
+bool startscasewith(const std::string_view &str, const std::string_view &prefix) {
+    if (prefix.size() > str.size())
+        return false;
+    for (size_t i = 0; i < prefix.size(); ++i)
+        if (std::tolower(str[i] & 255) != std::tolower(prefix[i] & 255))
+            return false;
+    return true;
+}
+
 void append_wchar(std::string *r, wchar_t c) {
     if (isascii(c)) {
         *r += c;
@@ -68,7 +88,7 @@ std::string join(const std::vector<std::string> &vec, const std::string_view &de
     return result;
 }
 
-std::string basename(const std::string_view path) {
+std::string basename(const std::string_view &path) {
     size_t i, e;
     if ((e = path.size())) {
         while (e > 1 && path[e - 1] == '/')
@@ -90,7 +110,7 @@ std::string_view extname(const std::string_view &path) {
     return path;
 }
 
-std::string dirname(const std::string_view path) {
+std::string dirname(const std::string_view &path) {
     size_t e = path.size();
     if (e--) {
         for (; path[e] == '/'; e--)
@@ -107,7 +127,7 @@ std::string dirname(const std::string_view path) {
     return ".";
 }
 
-std::string resolve(const std::string_view lhs, const std::string_view rhs) {
+std::string resolve(const std::string_view &lhs, const std::string_view &rhs) {
     if (lhs.empty())
         return std::string(rhs);
     if (!rhs.empty() && rhs[0] == '/')
@@ -126,7 +146,7 @@ std::string resolve(const std::string_view lhs, const std::string_view rhs) {
 }
 
 // replaces multiple isspace() with one space and trims result
-std::string collapse(const std::string_view input) {
+std::string collapse(const std::string_view &input) {
     size_t start = 0;
     while (start < input.length() && std::isspace(input[start]))
         ++start;

llamafile/string.h

@@ -19,19 +19,21 @@
 #include <__fwd/string.h>
 #include <__fwd/string_view.h>
 #include <__fwd/vector.h>
-#include <time.h>
+#include <ctime>
 
 namespace lf {
 
+bool startscasewith(const std::string_view &, const std::string_view &);
+int strcasecmp(const std::string_view &, const std::string_view &);
 ssize_t slurp(std::string *, const char *);
-std::string basename(const std::string_view);
-std::string collapse(const std::string_view);
-std::string dirname(const std::string_view);
+std::string basename(const std::string_view &);
+std::string collapse(const std::string_view &);
+std::string dirname(const std::string_view &);
 std::string format(const char *, ...) __attribute__((format(printf, 1, 2)));
-std::string join(const std::vector<std::string> &, const std::string_view &);
-std::string resolve(const std::string_view, const std::string_view);
-std::string tolower(const std::string_view);
 std::string iso8601(struct timespec);
+std::string join(const std::vector<std::string> &, const std::string_view &);
+std::string resolve(const std::string_view &, const std::string_view &);
+std::string tolower(const std::string_view &);
 std::string_view extname(const std::string_view &);
 void append_wchar(std::string *, wchar_t);