Extensive example of streaming data without hogging memory

This is an example as a reply to
#47

examples/example_chunked.c will open up a file (of any size),
and stream data back to the client in 128 byte chunked segments.

This is the proper way of sending large amounts of data back to
a client without consuming much memory.

examples/CMakeLists.txt

@@ -8,6 +8,7 @@ add_executable(test_query EXCLUDE_FROM_ALL test_query.c)
 add_executable(test_perf EXCLUDE_FROM_ALL test_perf.c)
 add_executable(example_vhost EXCLUDE_FROM_ALL example_vhost.c)
 add_executable(example_pause EXCLUDE_FROM_ALL example_pause.c)
+add_executable(example_chunked EXCLUDE_FROM_ALL example_chunked.c)
 
 
 if (NOT EVHTP_DISABLE_EVTHR)
@@ -24,6 +25,7 @@ target_link_libraries(test_query   evhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
 target_link_libraries(test_perf    evhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
 target_link_libraries(example_vhost evhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
 target_link_libraries(example_pause evhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
+target_link_libraries(example_chunked evhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
 
 
 if (NOT EVHTP_DISABLE_SSL)
@@ -42,4 +44,4 @@ if (NOT EVHTP_DISABLE_SSL)
     add_dependencies(examples example_https)
 endif()
 
-add_dependencies(examples example_pause example_vhost test_extensive test_basic test_vhost test_client test_query test_perf)
+add_dependencies(examples example_chunked example_pause example_vhost test_extensive test_basic test_vhost test_client test_query test_perf)

examples/eutils.h

@@ -0,0 +1,10 @@
+#pragma once
+
+static void * mm__dup_(const void * src, size_t size) {
+    void * mem = malloc(size);
+
+    return mem ? memcpy(mem, src, size) : NULL;
+}
+
+#define mm__alloc_(type, ...) \
+    (type *)mm__dup_((type[]) {__VA_ARGS__ }, sizeof(type))

examples/example_chunked.c

@@ -0,0 +1,162 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+
+#include "../log.h"
+#include "./eutils.h"
+#include "internal.h"
+#include "evhtp/evhtp.h"
+
+struct reply_ {
+    evhtp_request_t * request;
+    FILE            * file_desc;
+    struct evbuffer * buffer;
+};
+
+
+/* This function is called each time the client has been sent
+ * all outstanding data. We use this to send the next part of
+ * the file in a chunk at 128 byte increments.
+ *
+ * When there is no more data to be read from the file, this
+ * will send the final chunked reply and free our struct reply_.
+ */
+static evhtp_res
+http__send_chunk_(evhtp_connection_t * conn, void * arg) {
+    struct reply_ * reply = (struct reply_ *)arg;
+    char            buf[128];
+    size_t          bytes_read;
+
+    /* try to read 128 bytes from the file pointer */
+    bytes_read = fread(buf, 1, sizeof(buf), reply->file_desc);
+
+    log_info("Sending %zu bytes", bytes_read);
+
+    if (bytes_read > 0) {
+        /* add our data we read from the file into our reply buffer */
+        evbuffer_add(reply->buffer, buf, bytes_read);
+
+        /* send the reply buffer as a http chunked message */
+        evhtp_send_reply_chunk(reply->request, reply->buffer);
+
+        /* we can now drain our reply buffer as to not be a resource
+         * hog.
+         */
+        evbuffer_drain(reply->buffer, bytes_read);
+    }
+
+    /* check if we have read everything from the file */
+    if (feof(reply->file_desc)) {
+        log_info("Sending last chunk");
+
+        /* now that we have read everything from the file, we must
+         * first unset our on_write hook, then inform evhtp to send
+         * this message as the final chunk.
+         */
+        evhtp_connection_unset_hook(conn, evhtp_hook_on_write);
+        evhtp_send_reply_chunk_end(reply->request);
+
+        /* we can now free up our little reply_ structure */
+        {
+            fclose(reply->file_desc);
+            evbuffer_free(reply->buffer);
+            free(reply);
+        }
+    }
+
+    return EVHTP_RES_OK;
+}
+
+/* This function is called when a request has been fully received.
+ *
+ * This function assumes the `arg` value is the filename that was
+ * passed via `evhtp_set_gencb` in `main`.
+ *
+ * 1. open the file
+ * 2. create a `struct reply_`
+ * 3. create an evbuffer that we will write into.
+ * 4. set a hook to call the function `http__send_chunk_` each
+ *    time all data has been sent from the previous write call.
+ * 5. start the chunked stream via `evhtp_send_reply_chunk_start`
+ */
+static void
+http__callback_(evhtp_request_t * req, void * arg) {
+    const char    * filename = arg;
+    FILE          * file_desc;
+    struct reply_ * reply;
+
+    evhtp_assert(arg != NULL);
+
+    /* open up the file as passed to us via evhtp_set_gencb */
+    file_desc = fopen((const char *)arg, "r");
+    evhtp_assert(file_desc != NULL);
+
+    /* create our little internal reply structure which will
+     * be used by `http__send_chunk_`
+     */
+    reply     = mm__alloc_(struct reply_, {
+        req,
+        file_desc,
+        evbuffer_new()
+    });
+
+    /* here we set a connection hook of the type `evhtp_hook_on_write`
+     *
+     * this will execute the function `http__send_chunk_` each time
+     * all data has been written to the client.
+     */
+    evhtp_connection_set_hook(req->conn,
+                              evhtp_hook_on_write,
+                              http__send_chunk_, reply);
+
+    /* we do not have to start sending data from the file from here -
+     * this function will write data to the client, thus when finished,
+     * will call our `http__send_chunk_` callback.
+     */
+    evhtp_send_reply_chunk_start(req, EVHTP_RES_OK);
+}
+
+int
+main(int argc, char ** argv) {
+    evhtp_t           * htp;
+    struct event_base * evbase;
+
+    if (argc < 2) {
+        printf("Usage: %s <file>\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    evbase = event_base_new();
+    evhtp_alloc_assert(evbase);
+
+    htp    = evhtp_new(evbase, NULL);
+    evhtp_alloc_assert(htp);
+
+    /* here we set our default request response callback, the argument
+     * that is passed will be the filename we want to stream to the
+     * client in chunked form.
+     */
+    evhtp_set_gencb(htp, http__callback_, strdup(argv[1]));
+
+    evhtp_bind_socket(htp, "127.0.0.1", 0, 128);
+    {
+        struct sockaddr_in sin;
+        socklen_t          len = sizeof(struct sockaddr);
+        uint16_t           port;
+
+        getsockname(
+            evconnlistener_get_fd(htp->server),
+            (struct sockaddr *)&sin, &len);
+
+        port = ntohs(sin.sin_port);
+
+        log_info("curl http://127.0.0.1:%d/", port);
+    }
+
+    event_base_loop(evbase, 0);
+
+
+    return 0;
+}

include/evhtp/evhtp.h

@@ -775,6 +775,10 @@ EVHTP_EXPORT int evhtp_connection_set_hook(evhtp_connection_t * c, evhtp_hook_ty
 EVHTP_EXPORT int evhtp_request_set_hook(evhtp_request_t * r, evhtp_hook_type type, evhtp_hook cb, void * arg);
 EVHTP_EXPORT int evhtp_callback_set_hook(evhtp_callback_t * cb, evhtp_hook_type type, evhtp_hook hookcb, void * arg);
 
+EVHTP_EXPORT evhtp_hooks_t * evhtp_connection_get_hooks(evhtp_connection_t * c);
+EVHTP_EXPORT evhtp_hooks_t * evhtp_request_get_hooks(evhtp_request_t * r);
+EVHTP_EXPORT evhtp_hooks_t * evhtp_callback_get_hooks(evhtp_callback_t * cb);
+
 /**
  * @brief removes all hooks.
  *
@@ -784,6 +788,11 @@ EVHTP_EXPORT int evhtp_callback_set_hook(evhtp_callback_t * cb, evhtp_hook_type
  */
 EVHTP_EXPORT int evhtp_unset_all_hooks(evhtp_hooks_t ** hooks);
 
+EVHTP_EXPORT int evhtp_request_unset_hook(evhtp_request_t * req, evhtp_hook_type type);
+EVHTP_EXPORT int evhtp_connection_unset_hook(evhtp_connection_t * conn, evhtp_hook_type type);
+EVHTP_EXPORT int evhtp_callback_unset_hook(evhtp_callback_t * callback, evhtp_hook_type type);
+
+
 /**
  * @brief bind to a socket, optionally with specific protocol support
  *        formatting. The addr can be defined as one of the following: