[WJ-1196] [WJ-1031] Add support for S3 presign URLs to upload blobs

deepwell/Cargo.toml

@@ -8,7 +8,7 @@ keywords = ["wikijump", "api", "backend", "wiki"]
 categories = ["asynchronous", "database", "web-programming::http-server"]
 exclude = [".gitignore", ".editorconfig"]
 
-version = "2024.9.14"
+version = "2024.10.6"
 authors = ["Emmie Smith <emmie.maeda@gmail.com>"]
 edition = "2021"
 

deepwell/README.md

@@ -90,6 +90,60 @@ $ cargo fmt     # Ensure code is formatted
 $ cargo clippy  # Check code for lints
 ```
 
+### Running requests
+
+When you have a local instance of DEEPWELL running, probably in the developement `docker-compose` instance, you may want to run requests against it. You can easily accomplish this with a tool like `curl`. The basic format is:
+
+```sh
+$ curl -X POST --json '{"jsonrpc":"2.0","method":"<method here>","params":<json data of request>,"id":<request id>}' http://localhost:2747/jsonrpc
+```
+
+Where you pass in the JSONRPC method name and corresponding JSON data. The ID value distinguishes between notices and requests, see the JSONRPC specification for information.
+
+For instance:
+
+```sh
+$ curl -X POST --json '{"jsonrpc":"2.0","method":"echo","params":{"my":["json","data"]},"id":0}' http://localhost:2747/jsonrpc
+
+{"jsonrpc":"2.0","id":0,"result":{"my":["json","data"]}}
+
+$ curl -X POST --json '{"jsonrpc":"2.0","method":"ping","id":0}' http://localhost:2747/jsonrpc
+
+{"jsonrpc":"2.0","id":0,"result":"Pong!"}
+```
+
+If you are unfamiliar with JSONRPC, you can read about it [on its website](https://www.jsonrpc.org/specification). For instance, one quirk is that for methods which take a non-list or object argument, you specify it as a list of one element.
+
+There is also a helper script to assist with making JSONRPC requests, `scripts/request.py`. It requires the popular [`requests`](https://requests.readthedocs.io/) library to be installed.
+
+Example usage:
+
+```sh
+$ scripts/request.py echo '{ "my": ["json","data"] }'
+OK  {'my': ['json', 'data']}
+
+$ scripts/request.py ping
+OK  Pong!
+
+$ scripts/request.py error
+ERR
+{'code': 4000,
+ 'data': None,
+ 'message': 'The request is in some way malformed or incorrect'}
+```
+
+**NOTE:** When you are uploading files to local minio as part of testing file upload flows, **you must leave the URL unmodified**. The host `files` is used as the S3 provider, which is a problem since this is not a valid host on your development machine, which necessitates use of `--connect-to` to tell `curl` to connect to the appropriate location instead:
+
+```sh
+$ curl --connect-to files:9000:localhost:9000 --upload-file <path-to-file> <s3-presign-url>
+```
+
+Alternatively, you can use the helper script:
+
+```sh
+$ scripts/upload.sh <path-to-file> <s3-presign-url>
+```
+
 ### Database
 
 There are two important directories related to the management of the database (which DEEPWELL can be said to "own"). They are both fairly self-explanatory:

deepwell/config.example.toml

@@ -355,6 +355,28 @@ minimum-name-bytes = 3
 # Set to 0 to disable.
 refill-name-change-days = 90
 
+
+[file]
+
+# The length of paths used for S3 presigned URLs.
+#
+# The value doesn't particularly matter so long as it is sufficiently long
+# to avoid collisions.
+#
+# Just to be safe, the generation mechanism is the same as for session tokens.
+presigned-path-length = 32
+
+# How long a presigned URL lasts before expiry.
+#
+# The value should only be a few minutes, and no longer than 12 hours.
+presigned-expiration-minutes = 5
+
+# The maximum blob size allowed globally, in KiB.
+maximum-blob-size-kb = 1_048_576
+
+# The maximum blob size allowed for user avatars, in KiB.
+maximum-avatar-size-kb = 250
+
 [message]
 
 # The maximum size of a message's subject line, in bytes.

deepwell/migrations/20220906103252_deepwell.sql

@@ -411,6 +411,26 @@ CREATE TABLE page_vote (
     CHECK ((disabled_at IS NULL) = (disabled_by IS NULL))
 );
 
+--
+-- Blobs
+--
+
+-- Manages blobs that are being uploaded by the user
+CREATE TABLE blob_pending (
+    external_id TEXT PRIMARY KEY,
+    created_by BIGINT NOT NULL REFERENCES "user"(user_id),
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    expires_at TIMESTAMP WITH TIME ZONE NOT NULL,
+    expected_length BIGINT NOT NULL CHECK (expected_length >= 0),
+    s3_path TEXT NOT NULL CHECK (length(s3_path) > 1),
+    s3_hash BYTEA,  -- NULL means not yet moved, NOT NULL means deleted from s3_path
+    presign_url TEXT NOT NULL CHECK (length(presign_url) > 1),
+
+    CHECK (expires_at > created_at),                 -- expiration time is not in the relative past
+    CHECK (length(external_id) = 24),                -- default length for a cuid2
+    CHECK (s3_hash IS NULL OR length(s3_hash) = 64)  -- SHA-512 hash size, if present
+);
+
 --
 -- Files
 --
@@ -514,7 +534,7 @@ CREATE TYPE message_recipient_type AS ENUM (
 -- A "record" is the underlying message data, with its contents, attachments,
 -- and associated metadata such as sender and recipient(s).
 CREATE TABLE message_record (
-    external_id TEXT PRIMARY KEY,
+    external_id TEXT PRIMARY KEY, -- ID comes from message_draft
     created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
     drafted_at TIMESTAMP WITH TIME ZONE NOT NULL,
     retracted_at TIMESTAMP WITH TIME ZONE,

deepwell/scripts/request.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import sys
+
+import requests
+
+
+def color_settings(value):
+    match value:
+        case "auto":
+            fd = sys.stdout.fileno()
+            return os.isatty(fd)
+        case "always":
+            return True
+        case "never":
+            return False
+
+
+def print_data(data):
+    if isinstance(data, str):
+        print(data)
+    else:
+        # Only print on multiple lines if it's "large"
+        output = json.dumps(data)
+        if len(output) > 16:
+            output = json.dumps(data, indent=4)
+            print()
+        print(output)
+
+
+def deepwell_request(endpoint, method, data, id=0, color=False):
+    r = requests.post(
+        endpoint,
+        json={
+            "jsonrpc": "2.0",
+            "method": method,
+            "params": data,
+            "id": id,
+        },
+    )
+
+    if color:
+        green_start = "\x1b[32m"
+        red_start = "\x1b[31m"
+        color_end = "\x1b[0m"
+    else:
+        green_start = ""
+        red_start = ""
+        color_end = ""
+
+    match r.json():
+        case {"jsonrpc": "2.0", "id": id, "result": data}:
+            print(f"{green_start}OK  {color_end}", end="")
+            print_data(data)
+            return 0
+        case {"jsonrpc": "2.0", "id": id, "error": data}:
+            print(f"{red_start}ERR {color_end}", end="")
+            print_data(data)
+            return 1
+
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser(
+        "deepwell-request",
+        description="Helper script to run DEEPWELL JSONRPC requests",
+    )
+    argparser.add_argument(
+        "-H",
+        "--host",
+        default="localhost",
+    )
+    argparser.add_argument(
+        "-p",
+        "--port",
+        type=int,
+        default=2747,
+    )
+    argparser.add_argument(
+        "-s",
+        "--https",
+        dest="scheme",
+        action="store_const",
+        const="https",
+        default="http",
+    )
+    argparser.add_argument(
+        "-I",
+        "--id",
+        default=0,
+    )
+    argparser.add_argument(
+        "-C",
+        "--color",
+        choices=["never", "auto", "always"],
+        default="auto",
+    )
+    argparser.add_argument("method")
+    argparser.add_argument("data", nargs="?", type=json.loads, default="{}")
+    args = argparser.parse_args()
+    enable_color = color_settings(args.color)
+
+    endpoint = f"{args.scheme}://{args.host}:{args.port}/jsonrpc"
+    exit_code = deepwell_request(
+        endpoint,
+        args.method,
+        args.data,
+        args.id,
+        color=enable_color,
+    )
+
+    sys.exit(exit_code)

deepwell/scripts/upload.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -eu
+
+#
+# Helper script to upload files to a local S3 store for testing upload flows.
+#
+
+if [[ $# -ne 2 ]]; then
+	echo >&2 "Usage: $0 <path-to-file> <s3-presign-url>"
+	exit 1
+fi
+
+# Allow either order of arguments, for convenience.
+# If it starts with HTTP or HTTPS, we assume it's the presign URL.
+if [[ $1 = http:* || $1 = https:* ]]; then
+	path="$2"
+	url="$1"
+else
+	path="$1"
+	url="$2"
+fi
+
+exec \
+	curl \
+		--connect-to 'files:9000:localhost:9000' \
+		--upload-file "$path" \
+		"$url"

deepwell/src/api.rs

@@ -28,9 +28,9 @@
 
 use crate::config::{Config, Secrets};
 use crate::endpoints::{
-    auth::*, category::*, domain::*, email::*, file::*, file_revision::*, link::*,
-    locale::*, message::*, misc::*, page::*, page_revision::*, parent::*, site::*,
-    site_member::*, text::*, user::*, user_bot::*, view::*, vote::*,
+    auth::*, blob::*, category::*, domain::*, email::*, file::*, file_revision::*,
+    link::*, locale::*, message::*, misc::*, page::*, page_revision::*, parent::*,
+    site::*, site_member::*, text::*, user::*, user_bot::*, view::*, vote::*,
 };
 use crate::locales::Localizations;
 use crate::services::blob::MimeAnalyzer;
@@ -174,6 +174,7 @@ async fn build_module(app_state: ServerState) -> anyhow::Result<RpcModule<Server
 
     // Miscellaneous
     register!("ping", ping);
+    register!("echo", echo);
     register!("error", yield_error);
     register!("version", version);
     register!("version_full", full_version);
@@ -260,11 +261,13 @@ async fn build_module(app_state: ServerState) -> anyhow::Result<RpcModule<Server
 
     // Blob data
     register!("blob_get", blob_get);
+    register!("blob_upload", blob_upload);
+    register!("blob_cancel", blob_cancel);
 
     // Files
-    register!("file_upload", file_upload);
-    register!("file_get", file_get);
+    register!("file_create", file_create);
     register!("file_edit", file_edit);
+    register!("file_get", file_get);
     register!("file_delete", file_delete);
     register!("file_move", file_move);
     register!("file_restore", file_restore);

deepwell/src/config/file.rs

@@ -53,6 +53,7 @@ pub struct ConfigFile {
     ftml: Ftml,
     special_pages: SpecialPages,
     user: User,
+    file: FileSection,
     message: Message,
 }
 
@@ -181,6 +182,16 @@ struct User {
     minimum_name_bytes: usize,
 }
 
+// NOTE: Name conflict with std::fs::File
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+struct FileSection {
+    presigned_path_length: usize,
+    presigned_expiration_minutes: u32,
+    maximum_blob_size_kb: i64,
+    maximum_avatar_size_kb: i64,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "kebab-case")]
 struct Message {
@@ -303,6 +314,13 @@ impl ConfigFile {
                     refill_name_change_days,
                     minimum_name_bytes,
                 },
+            file:
+                FileSection {
+                    presigned_path_length,
+                    presigned_expiration_minutes,
+                    maximum_blob_size_kb,
+                    maximum_avatar_size_kb,
+                },
             message:
                 Message {
                     maximum_subject_bytes: maximum_message_subject_bytes,
@@ -424,6 +442,10 @@ impl ConfigFile {
                 ))
             },
             minimum_name_bytes,
+            presigned_path_length,
+            presigned_expiry_secs: presigned_expiration_minutes * 60,
+            maximum_blob_size: maximum_blob_size_kb * 1024,
+            maximum_avatar_size: maximum_avatar_size_kb * 1024,
             maximum_message_subject_bytes,
             maximum_message_body_bytes,
             maximum_message_recipients,

deepwell/src/config/object.rs

@@ -200,6 +200,18 @@ pub struct Config {
     /// Minimum length of bytes in a username.
     pub minimum_name_bytes: usize,
 
+    /// Length of randomly-generated portion of S3 presigned URLs.
+    pub presigned_path_length: usize,
+
+    /// How long S3 presigned URLs will last before expiry.
+    pub presigned_expiry_secs: u32,
+
+    /// Maximum size of a blob globally.
+    pub maximum_blob_size: i64,
+
+    /// Maximum size of a user's avatar image.
+    pub maximum_avatar_size: i64,
+
     /// Maximum size of the subject line allowed in a direct message.
     pub maximum_message_subject_bytes: usize,