Allow uploading huge files to WebServer (#2180)

From espressif/arduino-esp32#9440

Author: earlephilhower

libraries/WebServer/examples/UploadHugeFile/README.md

@@ -0,0 +1,13 @@
+# Upload Huge File To Filesystem Over HTTP
+
+This project is an example of an HTTP server designed to facilitate the transfer of large files using the PUT method, in accordance with RFC specifications.
+
+### Example cURL Command
+
+```bash
+curl -X PUT -T ./my-file.mp3 http://pico-ip/upload/my-file.mp3
+```
+
+## Resources
+
+- RFC HTTP/1.0 - Additional Request Methods - PUT : [Link](https://datatracker.ietf.org/doc/html/rfc1945#appendix-D.1.1)

libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino

@@ -0,0 +1,92 @@
+#include <WiFi.h>
+#include <WiFiClient.h>
+#include <WebServer.h>
+#include <uri/UriRegex.h>
+#include <LittleFS.h>
+
+#ifndef STASSID
+#define STASSID "your-ssid"
+#define STAPSK "your-password"
+#endif
+
+const char *ssid = STASSID;
+const char *password = STAPSK;
+
+WebServer server(80);
+
+File rawFile;
+void handleCreate() {
+  server.send(200, "text/plain", "");
+}
+void handleCreateProcess() {
+  String path = "/" + server.pathArg(0);
+  HTTPRaw& raw = server.raw();
+  if (raw.status == RAW_START) {
+    if (LittleFS.exists((char *)path.c_str())) {
+      LittleFS.remove((char *)path.c_str());
+    }
+    rawFile = LittleFS.open(path.c_str(), "w");
+    Serial.print("Upload: START, filename: ");
+    Serial.println(path);
+  } else if (raw.status == RAW_WRITE) {
+    if (rawFile) {
+      rawFile.write(raw.buf, raw.currentSize);
+    }
+    Serial.print("Upload: WRITE, Bytes: ");
+    Serial.println(raw.currentSize);
+  } else if (raw.status == RAW_END) {
+    if (rawFile) {
+      rawFile.close();
+    }
+    Serial.print("Upload: END, Size: ");
+    Serial.println(raw.totalSize);
+  }
+}
+
+void returnFail(String msg) {
+  server.send(500, "text/plain", msg + "\r\n");
+}
+
+void handleNotFound() {
+  String message = "File Not Found\n\n";
+  message += "URI: ";
+  message += server.uri();
+  message += "\nMethod: ";
+  message += (server.method() == HTTP_GET) ? "GET" : "POST";
+  message += "\nArguments: ";
+  message += server.args();
+  message += "\n";
+  for (uint8_t i = 0; i < server.args(); i++) {
+    message += " " + server.argName(i) + ": " + server.arg(i) + "\n";
+  }
+  server.send(404, "text/plain", message);
+}
+
+void setup(void) {
+  Serial.begin(115200);
+
+  LittleFS.begin();
+
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(ssid, password);
+
+  while (WiFi.status() != WL_CONNECTED) {
+    delay(500);
+    Serial.print(".");
+  }
+  Serial.print("Connected to ");
+  Serial.println(ssid);
+  Serial.print("IP address: ");
+  Serial.println(WiFi.localIP());
+
+  server.on(UriRegex("/upload/(.*)"), HTTP_PUT, handleCreate, handleCreateProcess);
+  server.onNotFound(handleNotFound);
+  server.begin();
+  Serial.println("HTTP server started");
+
+}
+
+void loop(void) {
+  server.handleClient();
+  delay(2);//allow the cpu to switch to other tasks
+}

libraries/WebServer/src/HTTPServer.h

@@ -31,6 +31,7 @@
 enum HTTPUploadStatus { UPLOAD_FILE_START, UPLOAD_FILE_WRITE, UPLOAD_FILE_END,
                         UPLOAD_FILE_ABORTED
                       };
+enum HTTPRawStatus { RAW_START, RAW_WRITE, RAW_END, RAW_ABORTED };
 enum HTTPClientStatus { HC_NONE, HC_WAIT_READ, HC_WAIT_CLOSE };
 enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH };
 
@@ -40,6 +41,10 @@ enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH };
 #define HTTP_UPLOAD_BUFLEN 1436
 #endif
 
+#ifndef HTTP_RAW_BUFLEN
+#define HTTP_RAW_BUFLEN 1436
+#endif
+
 #define HTTP_MAX_DATA_WAIT 5000 //ms to wait for the client to send the request
 #define HTTP_MAX_DATA_AVAILABLE_WAIT 30 //ms to wait for the client to send the request when there is another client with data available
 #define HTTP_MAX_POST_WAIT 5000 //ms to wait for POST data to arrive
@@ -63,6 +68,16 @@ typedef struct {
     uint8_t buf[HTTP_UPLOAD_BUFLEN];
 } HTTPUpload;
 
+
+typedef struct {
+    HTTPRawStatus status;
+    size_t  totalSize;   // content size
+    size_t  currentSize; // size of data currently in buf
+    uint8_t buf[HTTP_UPLOAD_BUFLEN];
+    void    *data;       // additional data
+} HTTPRaw;
+
+
 #include "detail/RequestHandler.h"
 
 namespace fs {
@@ -97,6 +112,9 @@ class HTTPServer {
     HTTPUpload& upload() {
         return *_currentUpload;
     }
+    HTTPRaw& raw() {
+        return *_currentRaw;
+    }
 
     String pathArg(unsigned int i); // get request path argument by number
     String arg(String name);        // get request argument value by name
@@ -257,6 +275,7 @@ class HTTPServer {
     RequestArgument* _postArgs;
 
     std::unique_ptr<HTTPUpload> _currentUpload;
+    std::unique_ptr<HTTPRaw>    _currentRaw;
 
     int              _headerKeysCount;
     RequestArgument* _currentHeaders;

libraries/WebServer/src/Parsing.cpp

@@ -188,7 +188,30 @@ HTTPServer::ClientFuture HTTPServer::_parseRequest(WiFiClient* client) {
             }
         }
 
-        if (!isForm) {
+        if (!isForm && _currentHandler && _currentHandler->canRaw(_currentUri)) {
+            log_v("Parse raw");
+            _currentRaw.reset(new HTTPRaw());
+            _currentRaw->status = RAW_START;
+            _currentRaw->totalSize = 0;
+            _currentRaw->currentSize = 0;
+            log_v("Start Raw");
+            _currentHandler->raw(*this, _currentUri, *_currentRaw);
+            _currentRaw->status = RAW_WRITE;
+
+            while (_currentRaw->totalSize < (size_t)_clientContentLength) {
+                _currentRaw->currentSize = client->readBytes(_currentRaw->buf, HTTP_RAW_BUFLEN);
+                _currentRaw->totalSize += _currentRaw->currentSize;
+                if (_currentRaw->currentSize == 0) {
+                    _currentRaw->status = RAW_ABORTED;
+                    _currentHandler->raw(*this, _currentUri, *_currentRaw);
+                    return CLIENT_MUST_STOP;
+                }
+                _currentHandler->raw(*this, _currentUri, *_currentRaw);
+            }
+            _currentRaw->status = RAW_END;
+            _currentHandler->raw(*this, _currentUri, *_currentRaw);
+            log_v("Finish Raw");
+        } else if (!isForm) {
             size_t plainLength;
             char* plainBuf = readBytesWithTimeout(client, _clientContentLength, plainLength, HTTP_MAX_POST_WAIT);
             if ((int)plainLength < (int)_clientContentLength) {
@@ -333,7 +356,7 @@ void HTTPServer::_uploadWriteByte(uint8_t b) {
     _currentUpload->buf[_currentUpload->currentSize++] = b;
 }
 
-int HTTPServer::_uploadReadByte(WiFiClient* client) {
+int HTTPServer::_uploadReadByte(WiFiClient * client) {
     int res = client->read();
     if (res < 0) {
         // keep trying until you either read a valid byte or timeout
@@ -376,7 +399,7 @@ int HTTPServer::_uploadReadByte(WiFiClient* client) {
     return res;
 }
 
-bool HTTPServer::_parseForm(WiFiClient* client, String boundary, uint32_t len) {
+bool HTTPServer::_parseForm(WiFiClient * client, String boundary, uint32_t len) {
     (void) len;
     log_v("Parse Form: Boundary: %s Length: %d", boundary.c_str(), len);
     String line;
@@ -595,7 +618,7 @@ bool HTTPServer::_parseForm(WiFiClient* client, String boundary, uint32_t len) {
     return false;
 }
 
-String HTTPServer::urlDecode(const String& text) {
+String HTTPServer::urlDecode(const String & text) {
     String decoded = "";
     char temp[] = "0x00";
     unsigned int len = text.length();

libraries/WebServer/src/WebServerTemplate.h

@@ -171,6 +171,7 @@ void WebServerTemplate<ServerType, DefaultPort>::handleClient() {
         }
         _currentStatus = HC_NONE;
         _currentUpload.reset();
+        _currentRaw.reset();
     }
 
     if (callYield) {

libraries/WebServer/src/detail/RequestHandler.h

@@ -15,6 +15,10 @@ class RequestHandler {
         (void) uri;
         return false;
     }
+    virtual bool canRaw(String uri) {
+        (void) uri;
+        return false;
+    }
     virtual bool handle(HTTPServer& server, HTTPMethod requestMethod, String requestUri) {
         (void) server;
         (void) requestMethod;
@@ -26,6 +30,11 @@ class RequestHandler {
         (void) requestUri;
         (void) upload;
     }
+    virtual void raw(HTTPServer& server, String requestUri, HTTPRaw& raw) {
+        (void) server;
+        (void) requestUri;
+        (void) raw;
+    }
 
     RequestHandler* next() {
         return _next;

libraries/WebServer/src/detail/RequestHandlersImpl.h

@@ -42,6 +42,14 @@ class FunctionRequestHandler : public RequestHandler {
 
         return true;
     }
+    bool canRaw(String requestUri) override {
+        (void) requestUri;
+        if (!_ufn || _method == HTTP_GET) {
+            return false;
+        }
+
+        return true;
+    }
 
     bool handle(HTTPServer& server, HTTPMethod requestMethod, String requestUri) override {
         (void) server;
@@ -61,6 +69,14 @@ class FunctionRequestHandler : public RequestHandler {
         }
     }
 
+    void raw(HTTPServer& server, String requestUri, HTTPRaw& raw) override {
+        (void)server;
+        (void)raw;
+        if (canRaw(requestUri)) {
+            _ufn();
+        }
+    }
+
 protected:
     HTTPServer::THandlerFunction _fn;
     HTTPServer::THandlerFunction _ufn;