Skip to content

Commit

Permalink
Merge pull request #9 from NatLabs/dev
Browse files Browse the repository at this point in the history
access files via 'filename' instead of 'name'
  • Loading branch information
tomijaga authored Jun 18, 2024
2 parents abcfb4d + 583b397 commit 0387c36
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 91 deletions.
6 changes: 3 additions & 3 deletions docs.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
| `protocol` | always **https** | `Text` |
| `port` | URL Port. Default to `443` if it's not defined | `Nat16` |
| `host` | An object with the url host stored as a string and as an array. The array is the result of the string split at every dot.<br/> <br/> ```original = "www.google.com"```<br/> `array = ["www", "google", "com"]` | `{original: Text; array: [Text]}` |
| `path` | An object with the url paths stored as a string and as an array. The array is the result of the string split at every backslash. <br/> <br/> `original = "/categories/items/32"`, <br/>```array = ["categories", "items", "32"] ```| `{original: Text; array: [Text]}` |
| `path` | An object with the url paths stored as a string and as an array. The array is the result of the string split at every backslash. <br/> <br/> `original = "/categories/items/32"`, <br/>```array = ["categories", "items", "32"] ``` <br/> <br/> A path with a trailing backslash will have an empty string at the end of `path.array` <br/> <br/> `original = "/categories/items/"`, <br/>```array = ["categories", "items", ""] ``` | `{original: Text; array: [Text]}` |
| `queryObj` | An object for accessing fields and values of a query string | [SearchParams](#searchParams) |
| `anchor` | Everything after the symbol `#` | `Text` |

Expand Down Expand Up @@ -52,8 +52,8 @@
| `get` | Retrieves field values | `(Text) -> ?[Text]` |
| `keys` | An array of all field keys | `[Text]` |
| `trieMap` | Stores all field entries (data without a filename) | `TrieMap<Text, [Text]>` |
| `fileKeys` | An array of all file keys | `[Text]` |
| `files` | Retrieves files of a specific key/name | `(Text) -> ?[`[File](#file)`]` |
| `fileKeys` | An array of all filenames | `[Text]` |
| `files` | Retrieves files associated with a specific filename | `(Text) -> ?[`[File](#file)`]` |

#### File
| Property | Description | Type |
Expand Down
2 changes: 1 addition & 1 deletion mops.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "http-parser"
version = "0.2.0"
version = "0.3.0"
description = "A http request parser for parsing url, search query, headers and form data."
repository = "https://github.com/NatLabs/http-parser.mo"

Expand Down
42 changes: 19 additions & 23 deletions src/form-data.mo
Original file line number Diff line number Diff line change
Expand Up @@ -37,30 +37,28 @@ module {
func parseContentDisposition(buffer : Buffer<Nat8>) : (Text, Text) {
var line = "";

var i = 25;
var i = 31;
while (i < buffer.size()) {
line #= Char.toText(Utils.nat8ToChar(buffer.get(i)));
i += 1;
};

let splitTextArr = Iter.toArray(Text.tokens(line, #char ';'));
let n = splitTextArr.size();

var name = "";
if (n > 1) {
let arr = Iter.toArray(Text.split(splitTextArr[1], #text("name=")));
if (arr.size() == 2) {
name := trimQuotesAndSpaces(arr[1]);
};
let arr = Iter.toArray(Text.split(splitTextArr[0], #text("name=")));
if (arr.size() == 2) {
name := trimQuotesAndSpaces(arr[1]);
};

var filename = "";
if (n > 2) {
let arr = Iter.toArray(Text.split(splitTextArr[2], #text("filename=")));
if (n > 1) {
let arr = Iter.toArray(Text.split(splitTextArr[1], #text("filename=")));
if (arr.size() == 2) {

filename := trimQuotesAndSpaces(arr[1]);
};
};

(name, filename);
};

Expand All @@ -69,23 +67,19 @@ module {
func parseContentType(buffer : Buffer<Nat8>) : (Text, Text) {
var line = "";

var i = 20;
var i = 13;
while (i < buffer.size()) {
line #= Char.toText(Utils.nat8ToChar(buffer.get(i)));
i += 1;
};

let arr = Iter.toArray(Text.tokens(line, #char ':'));

return if (arr.size() > 1) {
let mime = Iter.toArray(Text.tokens(trimQuotesAndSpaces(arr[1]), #char '/'));
let mimeType = mime[0];
let mimeSubType = if (mime.size() > 1) {
mime[1];
} else { "" };
let mime = Iter.toArray(Text.tokens(trimQuotesAndSpaces(line), #char '/'));
let mimeType = mime[0];
let mimeSubType = if (mime.size() > 1) {
mime[1];
} else { "" };

(mimeType, mimeSubType);
} else { ("", "") };
(mimeType, mimeSubType);
};

func startsWith(a : Buffer<Nat8>, b : Buffer<Nat8>) : Bool {
Expand Down Expand Up @@ -199,7 +193,6 @@ module {
};

if (char == NEWLINE or char == CARRIAGE_RETURN or equals(line, exitBoundary)) {

// skips the next char if EOL == '\r\n'
if (char == CARRIAGE_RETURN and blobArray[i +1] == NEWLINE) {
if (is_EOL_LF_CR == false) {
Expand All @@ -211,6 +204,7 @@ module {

// Get's the boundary from the first line if it wasn't specified
if (lineIndexFromBoundary == 0) {

if (boundary.size() == 0) {
if (startsWith(line, delim)) {
boundary.append(line);
Expand Down Expand Up @@ -239,6 +233,7 @@ module {
};

if (lineIndexFromBoundary == 2) {

if (startsWith(line, content_type)) {
let (_mimeType, _mimeSubType) = parseContentType(line);
mimeType := _mimeType;
Expand All @@ -249,6 +244,7 @@ module {
};

if (lineIndexFromBoundary == 3 or lineIndexFromBoundary == 4) {

if ((not includesContentType) and start == 0) {
start := prevLineEnd + (if (is_EOL_LF_CR) { 2 } else { 1 });
};
Expand All @@ -262,7 +258,7 @@ module {
// if it doesn't, add it to fields
if (filename != "") {
filesMVMap.add(
name,
filename,
{
name = name;
filename = filename;
Expand Down
25 changes: 12 additions & 13 deletions src/lib.mo
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ module HttpRequestParser {
let raw_domain = (Option.get(headers.get("host"), [""]))[0];

var domain = Option.get(Utils.decodeURIComponent(raw_domain), raw_domain);
let url = Option.get(Utils.decodeURIComponent(raw_url), raw_url);
var url = Option.get(Utils.decodeURIComponent(raw_url), raw_url);

public let original = domain # url;

Expand All @@ -118,44 +118,43 @@ module HttpRequestParser {
public let array = Iter.toArray(Text.tokens(_host, #char('.')));
};

domain := url;

let p = Iter.toArray(Text.tokens(domain, #char('#')));
let p = Iter.toArray(Text.tokens(url, #char('#')));

public let anchor = if (p.size() > 1) {
domain := p[0];
url := p[0];
p[1];
} else {
domain := p[0];
url := p[0];
"";
};

let re = Iter.toArray(Text.tokens(domain, #char('?')));
let re = Iter.toArray(Text.tokens(url, #char('?')));

let queryString : Text = switch (re.size()) {
case (0) {
domain := "";
url := "";
re[1];
};
case (1) {
domain := re[0];
url := re[0];
"";
};

case (_) {
domain := re[0];
url := re[0];
re[1];
};

};

public let queryObj : SearchParams = SearchParams(queryString);

let path_iter = Text.tokens(domain, #char('/'));

public let path = object {
public let original = url;

let path_iter = Text.split(url, #char('/'));
ignore path_iter.next();
public let array = Iter.toArray(path_iter);
public let original = "/" # Text.join("/", Iter.fromArray(array));
};

};
Expand Down
73 changes: 22 additions & 51 deletions tests/Parser.test.mo
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ let success = run([
host.original == "m7sm4-2iaaa-aaaab-qabra-cai.raw.ic0.app",
host.array == ["m7sm4-2iaaa-aaaab-qabra-cai", "raw", "ic0", "app"],

path.original == "/counter",
path.array == ["counter"],
path.original == "/counter/",
path.array == ["counter", ""],

queryObj.original == "tag=2526172523",
queryObj.keys == ["tag"],
Expand Down Expand Up @@ -356,7 +356,7 @@ let success = run([
case (null) true;
},

// body.bytes(9, 23).toArray() == ArrayModule.slice(bytes, 9, 23)
body.bytes(9, 23).toArray() == ArrayModule.slice(bytes, 9, 23)
]);
},
),
Expand Down Expand Up @@ -385,30 +385,6 @@ let success = run([
let body = HttpParser.Body(blob, ?"multipart/form-data");
let { form } = body;

Debug.print(debug_show ("original", body.original));
Debug.print(debug_show ("size", body.size, blob.size()));
Debug.print(debug_show ("text", body.text()));
Debug.print(debug_show ("form.keys", form.keys));
Debug.print(debug_show ("form.get(\"field1\")", form.get("field1")));
Debug.print(debug_show ("form.fileKeys", form.fileKeys));
Debug.print(
debug_show (
"form.files(\"field2\")",
switch (form.files("field2")) {
case (?arr) ?(
arr[0].name,
arr[0].filename,
arr[0].mimeType,
arr[0].mimeSubType,
arr[0].start,
arr[0].end,
(arr[0].bytes.toArray(), Text.decodeUtf8(Blob.fromArray(arr[0].bytes.toArray()))),
(Utils.textToBytes("value2"), "value2"),
);
case (_) null;
},
)
);
assertAllTrue([
body.original == blob,
body.size == blob.size(),
Expand All @@ -417,30 +393,25 @@ let success = run([
form.keys == ["field1"],
form.get("field1") == ?["value1"],

form.fileKeys == ["field2"],

// switch (form.files("field2")) {
// case (?arr) {
// let file = arr[0];

// assertAllTrue([
// file.name == "field2",
// file.filename == "example.txt",

// file.mimeType == "text",
// file.mimeSubType == "plain",

// file.start == 172,
// file.end == 178,

// file.bytes.toArray() == Utils.textToBytes("value2"),
// file.bytes.toArray() == ArrayModule.slice(blobArray, 172, 178)

// ]);
// };

// case (_) false;
// },
form.fileKeys == ["example.txt"],

switch (form.files("example.txt")) {
case (?arr) {
let file = arr[0];

assertAllTrue([
file.name == "field2",
file.filename == "example.txt",
file.mimeType == "text",
file.mimeSubType == "plain",
file.start == 172,
file.end == 178,
file.bytes.toArray() == Utils.textToBytes("value2"),
file.bytes.toArray() == ArrayModule.slice(blobArray, 172, 178)
]);
};
case (_) false;
},
])

},
Expand Down
73 changes: 73 additions & 0 deletions tests/form.test.mo
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// @testmode wasi
import Iter "mo:base/Iter";
import Debug "mo:base/Debug";
import Prelude "mo:base/Prelude";
import Char "mo:base/Char";
import Text "mo:base/Text";

import Bench "mo:bench";

import HttpParser "../src";

let boundary = "----WebKitFormBoundary";

var body_text = "";

func add_file(filename : Text, content_type : Text, size : Nat) {
body_text #= "--" # boundary # "\r\n";
body_text #= "Content-Disposition: form-data; name=\"file\"; filename=\"" # filename # "\"\r\n";
body_text #= "Content-Type: " # content_type # "\r\n\r\n";

for (i in Iter.range(0, size - 1)) {
if (i % 100 == 99) {
body_text #= "\n";
} else {
body_text #= "a";
};
};

body_text #= "\r\n";
};

for (i in Iter.range(0, 1000)) {
add_file("file" # debug_show (i) # ".txt", "text/plain", i);
};

add_file("file.md", "text/markdown", (10 * 1024));
add_file("file.c", "text/plain", (1024 ** 2) * 2);
add_file("file.json", "application/json", (1024 ** 2) * 4);

// end form
body_text #= "--" # boundary # "--\r\n";
let body = Text.encodeUtf8(body_text);

let #ok(form) = HttpParser.parseForm(body, #multipart(null));

for (i in Iter.range(0, 1000)){
let filename = "file" # debug_show (i) # ".txt";
let ?files = form.files(filename);

let file = files[0];

assert (file.mimeType # "/" # file.mimeSubType == "text/plain");
assert (filename == file.filename);
assert file.bytes.size() == i;
assert file.end - file.start == i;
};

let ?markdown = form.files("file.md");
assert markdown[0].mimeType # "/" # markdown[0].mimeSubType == "text/markdown";
assert markdown[0].filename == "file.md";
assert markdown[0].bytes.size() == (10 * 1024);
assert markdown[0].end - markdown[0].start == (10 * 1024);

let ?c = form.files("file.c");
assert c[0].mimeType # "/" # c[0].mimeSubType == "text/plain";
assert c[0].filename == "file.c";
assert c[0].bytes.size() == (1024 ** 2) * 2;

let ?json = form.files("file.json");
assert json[0].mimeType # "/" # json[0].mimeSubType == "application/json";
assert json[0].filename == "file.json";
assert json[0].bytes.size() == (1024 ** 2) * 4;

0 comments on commit 0387c36

Please sign in to comment.