3party/mongoose
0
0
Fork 0
mirror of https://github.com/cesanta/mongoose.git synced 2025-01-14 09:48:01 +08:00
Code Issues Projects Releases Wiki Activity

Use MG_EV_HTTP_CHUNK for streaming upload

Browse Source
This commit is contained in:
Sergey Lyubka 2022-02-09 12:24:06 +00:00
parent 33666e7d6b
commit 9e634310b4
6 changed files with 99 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
docs/README.md
View File

@ -1176,61 +1176,6 @@ void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
}
```
### mg\_http\_upload()
```c
int mg_http_upload(struct mg_connection *c, struct mg_http_message *hm,
struct mg_fs *fs, const char *dir);
```
Handle file upload. See [file upload example](https://github.com/cesanta/mongoose/tree/master/examples/file-uploads/).
This function expects a series of POST requests with file data. POST requests
should have `name` and `offset` query string parameters set:
```text
POST /whatever_uri?name=myfile.txt&offset=1234 HTTP/1.0
Content-Length: 5
hello
```
- `name` - A mandatory query string parameter, specifies a file name. It it
created in the `dir` directory
- `offset` - An optional parameter, default `0`. If it set to `0`, or omitted,
then a file gets truncated before write. Otherwise, the body of
the POST request gets appended to the file
- Server must call `mg_http_upload()` when `/whatever_uri` is hit
The expected usage of this API function follows:
- A client splits a file into small enough chunks, to ensure that a chunk
fits into the server's RAM
- Then, each chunk is POST-ed to the server with using URI, like
`/some_uri?name=FILENAME&offset=OFFSET`
- Initial OFFSET is `0`, and subsequent offsets are non-zero
- Each chunk gets appended to the file
- When the last chunk is POST-ed, upload finishes
- POST data must not be encoded in any way; it it saved as-is
Parameters:
- `c` - Connection to use
- `hm` - POST message, containing parameters described above
- `fs` - Filesystem to use
- `dir` - Path to directory
Return value: Request body length or negative value on error
Usage example:
```c
// Mongoose events handler
void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
if (ev == MG_EV_HTTP_MSG) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
mg_http_upload(c, hm, &mg_fs_posix, "."); // Upload to current folder
}
```
### mg\_http\_bauth()
```c

31
examples/file-upload/main.c
View File

@ -1,5 +1,11 @@
// Copyright (c) 2020 Cesanta Software Limited
// All rights reserved
//
// Streaming upload example. Demonstrates how to use chunked encoding
// to send large payload in smaller chunks. To test, use curl utility:
//
// curl http://localhost:8000/upload \
// --data-binary @my_large_file.txt -H 'Transfer-Encoding: chunked'
#include "mongoose.h"
@ -7,20 +13,29 @@
// /upload - Saves the next file chunk
// all other URI - serves web_root/ directory
static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
if (ev == MG_EV_HTTP_MSG) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
if (mg_http_match_uri(hm, "/upload")) {
mg_http_upload(c, hm, &mg_fs_posix, "/tmp");
} else {
struct mg_http_serve_opts opts = {.root_dir = "web_root"};
mg_http_serve_dir(c, hm, &opts);
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
LOG(LL_INFO, ("Got chunk len %lu", (unsigned long) hm->chunk.len));
LOG(LL_INFO, ("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
LOG(LL_INFO, ("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
mg_http_delete_chunk(c, hm);
if (hm->chunk.len == 0) {
LOG(LL_INFO, ("Last chunk received, sending response"));
mg_http_reply(c, 200, "", "ok (chunked)\n");
}
} else if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
LOG(LL_INFO, ("Got all %lu bytes!", (unsigned long) hm->body.len));
LOG(LL_INFO, ("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
LOG(LL_INFO, ("Body:\n%.*s", (int) hm->body.len, hm->body.ptr));
mg_http_reply(c, 200, "", "ok (%lu)\n", (unsigned long) hm->body.len);
} else if (ev == MG_EV_HTTP_MSG) {
struct mg_http_serve_opts opts = {.root_dir = "web_root"};
mg_http_serve_dir(c, hm, &opts);
}
}
int main(void) {
struct mg_mgr mgr;
struct mg_timer t1;
mg_mgr_init(&mgr);
mg_log_set("3");

41
examples/file-upload/web_root/app.js
View File

@ -1,41 +0,0 @@
// Copyright (c) 2020 Cesanta Software Limited
// All rights reserved
// Helper function to display upload status
var setStatus = function(text) {
document.getElementById('el3').innerText = text;
};
// When user clicks on a button, trigger file selection dialog
var button = document.getElementById('el2');
button.onclick = function(ev) {
input.click();
};
// Send a large blob of data chunk by chunk
var sendFileData = function(name, data, chunkSize) {
var sendChunk = function(offset) {
var chunk = data.subarray(offset, offset + chunkSize) || '';
var opts = {method: 'POST', body: chunk};
var url = '/upload?offset=' + offset + '&name=' + encodeURIComponent(name);
setStatus(
'sending bytes ' + offset + '..' + (offset + chunk.length) + ' of ' +
data.length);
fetch(url, opts).then(function(res) {
if (chunk.length > 0) sendChunk(offset + chunk.length);
});
};
sendChunk(0);
};
// If user selected a file, read it into memory and trigger sendFileData()
var input = document.getElementById('el1');
input.onchange = function(ev) {
if (!ev.target.files[0]) return;
var f = ev.target.files[0], r = new FileReader();
r.readAsArrayBuffer(f);
r.onload = function() {
ev.target.value = '';
sendFileData(f.name, new Uint8Array(r.result), 4096);
};
};

48
examples/file-upload/web_root/index.html
View File

@ -16,21 +16,49 @@
<div id="info">
Mongoose always buffers a full HTTP message before invoking
MG_EV_HTTP_MSG event. Big POST request require of lot
of RAM to buffer everything. Therefore, in order to upload large
files on memory-constrained system, a large file should be send
in small chunks.
<br/><br/>
In this example, JavaScript code on this page sends uploaded
file in 2k chunks using an <code>/upload</code> endpoint.
Uploaded file is stored in <code>/tmp</code> directory by the helper
API function <code>mg_http_upload()</code>
of RAM to buffer everything.
<br><br>
In order to upload large files to a memory-constrained system, use
<code>MG_EV_HTTP_CHUNK</code> on a server side. It fires when
a partial HTTP message has been received (or a chunk-encoded chunk).
Use <code>mg_http_delete_chunk()</code> to release chunk memory.
When 0-sized chunk is received, that's the end of the message.
Use <code>MG_MAX_RECV_BUF_SIZE</code> build constant to limit
maximum chunk size on a server side.
<br><br>
In this example, JavaScript code uses "fetch()" browser API.
Uploaded file is not saved, but rather printed by server side.
</div>
<div id="wrapper">
<input type="file" id="el1" style="display: none"/>
<button id="el2">choose file...</button>
<div id="el3"></div>
<div id="el3" style="margin-top: 1em;"></div>
</div>
</div>
</body>
<script src="app.js"></script>
<script>
// When user clicks on a button, trigger file selection dialog
document.getElementById('el2').onclick = function(ev) {
document.getElementById('el1').click();
};
// If user selected a file, read it into memory and trigger sendFileData()
document.getElementById('el1').onchange = function(ev) {
if (!ev.target.files[0]) return;
var f = ev.target.files[0], r = new FileReader();
r.readAsArrayBuffer(f);
r.onload = function() {
ev.target.value = '';
document.getElementById('el3').innerText = 'Uploading...';
fetch('/upload?name=' + encodeURIComponent(f.name), {
method: 'POST',
body: r.result,
}).then(function(res) {
document.getElementById('el3').innerText = 'Uploaded ' + r.result.byteLength + ' bytes';
});
};
};
</script>
</html>

33
mongoose.c
View File

@ -1407,7 +1407,7 @@ static struct mg_str guess_content_type(struct mg_str path, const char *extra) {
static int getrange(struct mg_str *s, int64_t *a, int64_t *b) {
size_t i, numparsed = 0;
LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr));
// LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr));
for (i = 0; i + 6 < s->len; i++) {
if (memcmp(&s->ptr[i], "bytes=", 6) == 0) {
struct mg_str p = mg_str_n(s->ptr + i + 6, s->len - i - 6);
@ -1573,7 +1573,7 @@ static void listdir(struct mg_connection *c, struct mg_http_message *hm,
"<!DOCTYPE html><html><head><title>Index of %.*s</title>%s%s"
"<style>th,td {text-align: left; padding-right: 1em; "
"font-family: monospace; }</style></head>"
"<body><h1>Index of %.*s</h1><table cellpadding=\"0\"><thead>"
"<body><h1>Innex of %.*s</h1><table cellpadding=\"0\"><thead>"
"<tr><th><a href=\"#\" rel=\"0\">Name</a></th><th>"
"<a href=\"#\" rel=\"1\">Modified</a></th>"
"<th><a href=\"#\" rel=\"2\">Size</a></th></tr>"
@ -1821,21 +1821,17 @@ static bool mg_is_chunked(struct mg_http_message *hm) {
void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
struct mg_str ch = hm->chunk;
if (mg_is_chunked(hm)) {
ch.len += 4; // \r\n before and after the chunk
ch.ptr -= 2;
const char *end = (char *) &c->recv.buf[c->recv.len], *ce;
bool chunked = mg_is_chunked(hm);
if (!mg_is_chunked(hm)) return;
if (chunked) {
ch.len += 4, ch.ptr -= 2; // \r\n before and after the chunk
while (ch.ptr > hm->body.ptr && *ch.ptr != '\n') ch.ptr--, ch.len++;
}
{
const char *end = &ch.ptr[ch.len];
size_t n = (size_t) (end - (char *) c->recv.buf);
if (c->recv.len > n) {
memmove((char *) ch.ptr, end, (size_t) (c->recv.len - n));
}
// LOG(LL_INFO, ("DELETING CHUNK: %zu %zu %zu\n%.*s", c->recv.len, n,
// ch.len, (int) ch.len, ch.ptr));
}
ce = &ch.ptr[ch.len];
if (ce < end) memmove((void *) ch.ptr, ce, (size_t) (end - ce));
c->recv.len -= ch.len;
if (c->pfn_data == NULL) c->pfn_data = (char *) c->pfn_data - ch.len;
}
int mg_http_upload(struct mg_connection *c, struct mg_http_message *hm,
@ -1889,7 +1885,16 @@ static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
if (n > 0 && !is_chunked) {
hm.chunk =
mg_str_n((char *) &c->recv.buf[n], c->recv.len - (size_t) n);
// Store remaining body length in c->pfn_data
if (c->pfn_data == NULL)
c->pfn_data = (void *) (hm.message.len - (size_t) n);
mg_call(c, MG_EV_HTTP_CHUNK, &hm);
if (c->pfn_data == NULL) {
hm.chunk.len = 0; // Last chunk!
mg_call(c, MG_EV_HTTP_CHUNK, &hm); // Lest user know
memmove(c->recv.buf, c->recv.buf + n, c->recv.len - (size_t) n);
c->recv.len -= (size_t) n;
}
}
break;
}

33
src/http.c
View File

@ -455,7 +455,7 @@ static struct mg_str guess_content_type(struct mg_str path, const char *extra) {
static int getrange(struct mg_str *s, int64_t *a, int64_t *b) {
size_t i, numparsed = 0;
LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr));
// LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr));
for (i = 0; i + 6 < s->len; i++) {
if (memcmp(&s->ptr[i], "bytes=", 6) == 0) {
struct mg_str p = mg_str_n(s->ptr + i + 6, s->len - i - 6);
@ -621,7 +621,7 @@ static void listdir(struct mg_connection *c, struct mg_http_message *hm,
"<!DOCTYPE html><html><head><title>Index of %.*s</title>%s%s"
"<style>th,td {text-align: left; padding-right: 1em; "
"font-family: monospace; }</style></head>"
"<body><h1>Index of %.*s</h1><table cellpadding=\"0\"><thead>"
"<body><h1>Innex of %.*s</h1><table cellpadding=\"0\"><thead>"
"<tr><th><a href=\"#\" rel=\"0\">Name</a></th><th>"
"<a href=\"#\" rel=\"1\">Modified</a></th>"
"<th><a href=\"#\" rel=\"2\">Size</a></th></tr>"
@ -869,21 +869,17 @@ static bool mg_is_chunked(struct mg_http_message *hm) {
void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
struct mg_str ch = hm->chunk;
if (mg_is_chunked(hm)) {
ch.len += 4; // \r\n before and after the chunk
ch.ptr -= 2;
const char *end = (char *) &c->recv.buf[c->recv.len], *ce;
bool chunked = mg_is_chunked(hm);
if (!mg_is_chunked(hm)) return;
if (chunked) {
ch.len += 4, ch.ptr -= 2; // \r\n before and after the chunk
while (ch.ptr > hm->body.ptr && *ch.ptr != '\n') ch.ptr--, ch.len++;
}
{
const char *end = &ch.ptr[ch.len];
size_t n = (size_t) (end - (char *) c->recv.buf);
if (c->recv.len > n) {
memmove((char *) ch.ptr, end, (size_t) (c->recv.len - n));
}
// LOG(LL_INFO, ("DELETING CHUNK: %zu %zu %zu\n%.*s", c->recv.len, n,
// ch.len, (int) ch.len, ch.ptr));
}
ce = &ch.ptr[ch.len];
if (ce < end) memmove((void *) ch.ptr, ce, (size_t) (end - ce));
c->recv.len -= ch.len;
if (c->pfn_data == NULL) c->pfn_data = (char *) c->pfn_data - ch.len;
}
int mg_http_upload(struct mg_connection *c, struct mg_http_message *hm,
@ -937,7 +933,16 @@ static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
if (n > 0 && !is_chunked) {
hm.chunk =
mg_str_n((char *) &c->recv.buf[n], c->recv.len - (size_t) n);
// Store remaining body length in c->pfn_data
if (c->pfn_data == NULL)
c->pfn_data = (void *) (hm.message.len - (size_t) n);
mg_call(c, MG_EV_HTTP_CHUNK, &hm);
if (c->pfn_data == NULL) {
hm.chunk.len = 0; // Last chunk!
mg_call(c, MG_EV_HTTP_CHUNK, &hm); // Lest user know
memmove(c->recv.buf, c->recv.buf + n, c->recv.len - (size_t) n);
c->recv.len -= (size_t) n;
}
}
break;
}