3party/mongoose
0
0
Fork 0
mirror of https://github.com/cesanta/mongoose.git synced 2024-12-26 22:41:03 +08:00
Code Issues Projects Releases Wiki Activity

Get rid of MG_EV_HTTP_CHUNK

Browse Source
This commit is contained in:
cpq 2023-09-26 19:59:42 +01:00
parent dde699ffda
commit 7884a2f3c7
10 changed files with 224 additions and 373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
examples/file-upload-single-post/main.c
View File

@ -1,8 +1,8 @@
// Copyright (c) 2020 Cesanta Software Limited
// All rights reserved
//
// Streaming upload example. Demonstrates how to use MG_EV_HTTP_CHUNK events
// to send large payload in smaller chunks. To test, use curl utility:
// Streaming upload example. Demonstrates how to use MG_EV_READ events
// to get large payload in smaller chunks. To test, use curl utility:
//
// curl http://localhost:8000/upload?name=a.txt --data-binary @large_file.txt
@ -12,21 +12,33 @@
// /upload - Saves the next file chunk
// all other URI - serves web_root/ directory
static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
MG_INFO(("Got chunk len %lu", (unsigned long) hm->chunk.len));
MG_INFO(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
// MG_INFO(("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
mg_http_delete_chunk(c, hm);
if (hm->chunk.len == 0) {
MG_INFO(("Last chunk received, sending response"));
mg_http_reply(c, 200, "", "ok (chunked)\n");
if (ev == MG_EV_READ) {
// Parse the incoming data ourselves. If we can parse the request,
// store two size_t variables in the c->data: expected len and recv len.
size_t *data = (size_t *) c->data;
if (data[0]) { // Already parsed, simply print received data
data[1] += c->recv.len;
MG_INFO(("Got chunk len %lu, %lu total", c->recv.len, data[1]));
c->recv.len = 0; // And cleanup the receive buffer. Streaming!
if (data[1] >= data[0]) mg_http_reply(c, 200, "", "ok\n");
} else {
struct mg_http_message hm;
int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
if (n < 0) mg_error(c, "Bad response");
if (n > 0) {
if (mg_http_match_uri(&hm, "/upload")) {
MG_INFO(("Got chunk len %lu", c->recv.len - n));
data[0] = hm.body.len;
data[1] = c->recv.len - n;
if (data[1] >= data[0]) mg_http_reply(c, 200, "", "ok\n");
} else {
struct mg_http_serve_opts opts = {.root_dir = "web_root"};
mg_http_serve_dir(c, &hm, &opts);
}
}
}
} else if (ev == MG_EV_HTTP_MSG) {
struct mg_http_serve_opts opts = {.root_dir = "web_root"};
mg_http_serve_dir(c, hm, &opts);
}
(void) fn_data;
(void) fn_data, (void) ev_data;
}
int main(void) {

10
examples/file-upload-single-post/web_root/index.html
View File

@ -19,12 +19,10 @@
of RAM to buffer everything.
<br><br>
In order to upload large files to a memory-constrained system, use
<code>MG_EV_HTTP_CHUNK</code> on a server side. It fires when
a partial HTTP message has been received (or a chunk-encoded chunk).
Use <code>mg_http_delete_chunk()</code> to release chunk memory.
When 0-sized chunk is received, that's the end of the message.
Use <code>MG_MAX_RECV_SIZE</code> build constant to limit
maximum chunk size on a server side.
<code>MG_EV_READ</code> on a server side. It fires when
a partial HTTP message has been received. We discourage this method,
since the client can send chunked-encoded data. Instead, please
split the upload into smaller pieces and send sequentially.
<br><br>
In this example, JavaScript code uses "fetch()" browser API.
Uploaded file is not saved, but rather printed by server side.

50
examples/http-streaming-client/main.c
View File

@ -1,14 +1,12 @@
// Copyright (c) 2021 Cesanta Software Limited
// All rights reserved
//
// Example HTTP client.
// Instead of buffering the whole HTTP response in memory and wait for the
// MG_EV_HTTP_MSG event, this client catches MG_EV_HTTP_CHUNK events and
// prints them. This is useful to download large content without spooling
// everything to memory, or catch chunked content.
// Example streaming HTTP client.
// The default HTTP handler waits until the whole HTTP message is buffered in
// c->recv IO buffer. It can be large. This example shows how to receive
// a potentially large response in chunks by handling the MG_EV_READ events.
//
// You can change `s_url` from the command line by executing: ./example YOUR_URL
//
// To enable SSL/TLS, see https://mongoose.ws/tutorials/tls/#how-to-build
#include "mongoose.h"
@ -21,13 +19,11 @@ static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
if (ev == MG_EV_CONNECT) {
// Connected to server. Extract host name from URL
struct mg_str host = mg_url_host(s_url);
if (mg_url_is_ssl(s_url)) {
struct mg_tls_opts opts = {.ca = mg_unpacked("/certs/ca.pem"),
.name = host};
mg_tls_init(c, &opts);
}
// Send request
mg_printf(c,
"GET %s HTTP/1.1\r\n"
@ -36,21 +32,33 @@ static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
"Host: %.*s\r\n"
"\r\n",
mg_url_uri(s_url), (int) host.len, host.ptr);
} else if (ev == MG_EV_HTTP_CHUNK) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
fwrite(hm->chunk.ptr, 1, hm->chunk.len, stdout);
// fprintf(stderr, "c %u\n", (unsigned) hm->chunk.len);
mg_http_delete_chunk(c, hm);
if (hm->chunk.len == 0) *(bool *) fn_data = true; // Last chunk
} else if (ev == MG_EV_HTTP_MSG) {
// Response is received. Print it
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
fwrite(hm->body.ptr, 1, hm->body.len, stdout);
c->is_draining = 1; // Tell mongoose to close this connection
*(bool *) fn_data = true; // Tell event loop to stop
} else if (ev == MG_EV_READ) {
// Parse the incoming data ourselves. If we can parse the request,
// store two size_t variables in the c->data: expected len and recv len.
size_t *data = (size_t *) c->data;
if (data[0]) { // Already parsed, simply print received data
data[1] += c->recv.len;
fwrite(c->recv.buf, 1, c->recv.len, stdout);
c->recv.len = 0; // And cleanup the receive buffer. Streming!
if (data[1] >= data[0]) * (bool *) fn_data = true;
} else {
struct mg_http_message hm;
int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
if (n < 0) mg_error(c, "Bad response");
if (n > 0) {
fwrite(c->recv.buf + n, 1, c->recv.len - n, stdout); // Print body
data[0] = n + hm.body.len;
data[1] = c->recv.len;
c->recv.len = 0; // Cleanup receive buffer
if (data[1] >= data[0]) * (bool *) fn_data = true;
}
}
} else if (ev == MG_EV_CLOSE) {
*(bool *) fn_data = true; // tell event loop to stop
} else if (ev == MG_EV_ERROR) {
*(bool *) fn_data = true; // Error, tell event loop to stop
}
(void) ev_data;
}
int main(int argc, char *argv[]) {
@ -59,7 +67,7 @@ int main(int argc, char *argv[]) {
const char *log_level = getenv("V"); // Log level
mg_mgr_init(&mgr); // Initialise event manager
if (log_level == NULL) log_level = "3"; // If not set, set to DEBUG
if (log_level == NULL) log_level = "2"; // If not set, set to DEBUG
mg_log_set(atoi(log_level)); // Set to 0 to disable debug log
if (argc > 1) s_url = argv[1]; // Use URL from command line

163
mongoose.c
View File

@ -1984,8 +1984,7 @@ int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
hm->message.ptr = hm->head.ptr = s;
hm->body.ptr = end;
hm->head.len = (size_t) req_len;
hm->chunk.ptr = end;
hm->message.len = hm->body.len = (size_t) ~0; // Set body length to infinite
hm->message.len = hm->body.len = (size_t) -1; // Set body length to infinite
// Parse request line
hm->method.ptr = s;
@ -2690,131 +2689,75 @@ static bool is_hex_digit(int c) {
(c >= 'A' && c <= 'F');
}
// If a server sends data to the client using chunked encoding, Mongoose strips
// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event. When zero
// chunk is received, we fire MG_EV_HTTP_MSG, and the body already has all
// chunking prefixes/suffixes stripped.
//
// If a server sends data without chunked encoding, we also fire a series of
// MG_EV_HTTP_CHUNK events for every received piece of data, and then we fire
// MG_EV_HTTP_MSG event in the end.
//
// We track total processed length in the c->pfn_data, which is a void *
// pointer: we store a size_t value there.
static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
size_t i = 0, n;
while (i < s.len && is_hex_digit(s.ptr[i])) i++;
n = mg_unhexn(s.ptr, i);
// MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
if (s.len < i + n + 4) return false; // Chunk not yet fully buffered
if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
*prefixlen = i + 2;
*datalen = n;
return true;
static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
int i = 0, n = 0;
if (len < 3) return 0;
while (i < len && is_hex_digit(buf[i])) i++;
if (len < i + 1 || buf[i] != '\r' || buf[i + 1] != '\n') return -1; // Error
n = (int) mg_unhexn(buf, (size_t) i); // Decode hex length
if (n < 0) return -1; // Error
if (len < i + n + 4) return 0; // Chunk not yet fully buffered
if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1; // Error
*pl = i + 2, *dl = n;
return i + 2 + n + 2;
}
static bool mg_is_chunked(struct mg_http_message *hm) {
static bool is_chunked(struct mg_http_message *hm) {
const char *needle = "chunked";
struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
return te != NULL && mg_vcasecmp(te, needle) == 0;
}
void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
}
static void deliver_chunked_chunks(struct mg_connection *c, size_t hlen,
struct mg_http_message *hm, bool *next) {
// | ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
// +------------------+--------------------------+----
// | hlen | chunk1 | ......
char *buf = (char *) &c->recv.buf[hlen], *p = buf;
size_t len = c->recv.len - hlen;
size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
size_t mark, pl, dl, del = 0, ofs = 0;
bool last = false;
if (processed <= len) len -= processed, buf += processed;
while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
size_t saved = c->recv.len;
memmove(p + processed, buf + ofs + pl, dl);
// MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
hm->chunk = mg_str_n(p + processed, dl);
mg_call(c, MG_EV_HTTP_CHUNK, hm);
ofs += pl + dl + 2, del += pl + 2; // 2 is for \r\n suffix
processed += dl;
if (c->recv.len != saved) processed -= dl, buf -= dl;
// mg_hexdump(c->recv.buf, hlen + processed);
last = (dl == 0);
}
mg_iobuf_del(&c->recv, hlen + processed, del);
mark = ((size_t) c->pfn_data) & MG_DMARK;
c->pfn_data = (void *) (processed | mark);
if (last) {
hm->body.len = processed;
hm->message.len = hlen + processed;
c->pfn_data = NULL;
if (mark) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
// MG_INFO(("LAST, mark: %lx", mark));
// mg_hexdump(c->recv.buf, c->recv.len);
}
}
static void deliver_normal_chunks(struct mg_connection *c, size_t hlen,
struct mg_http_message *hm, bool *next) {
size_t left, processed = ((size_t) c->pfn_data) & ~MG_DMARK;
size_t deleted = ((size_t) c->pfn_data) & MG_DMARK;
hm->chunk = mg_str_n((char *) &c->recv.buf[hlen], c->recv.len - hlen);
if (processed <= hm->chunk.len && !deleted) {
hm->chunk.len -= processed;
hm->chunk.ptr += processed;
}
left = hm->body.len < processed ? 0 : hm->body.len - processed;
if (hm->chunk.len > left) hm->chunk.len = left;
if (hm->chunk.len > 0) mg_call(c, MG_EV_HTTP_CHUNK, hm);
processed += hm->chunk.len;
deleted = ((size_t) c->pfn_data) & MG_DMARK; // Re-evaluate after user call
if (processed >= hm->body.len) { // Last, 0-len chunk
hm->chunk.len = 0; // Reset length
mg_call(c, MG_EV_HTTP_CHUNK, hm); // Call user handler
c->pfn_data = NULL; // Reset processed counter
if (processed && deleted) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
} else {
c->pfn_data = (void *) (processed | deleted); // if it is set
}
}
static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
struct mg_http_message hm;
while (c->recv.buf != NULL && c->recv.len > 0) {
bool next = false;
int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
if (hlen < 0) {
mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
break;
size_t ofs = 0; // Parsing offset
while (c->is_resp == 0 && ofs < c->recv.len) {
const char *buf = (char *) c->recv.buf + ofs;
int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
if (n < 0) {
mg_error(c, "HTTP parse");
return;
}
if (c->is_resp) break; // Response is still generated
if (hlen == 0) break; // Request is not buffered yet
if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
hm.message.len = c->recv.len; // and closes now, deliver a MSG
if (n == 0) break; // Request is not buffered yet
if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
hm.message.len = c->recv.len - ofs; // and closes now, deliver MSG
hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
}
if (mg_is_chunked(&hm)) {
deliver_chunked_chunks(c, (size_t) hlen, &hm, &next);
} else {
deliver_normal_chunks(c, (size_t) hlen, &hm, &next);
if (is_chunked(&hm)) {
// For chunked data, strip off prefixes and suffixes from chunks
// and relocate them right after the headers, then report a message
char *s = (char *) c->recv.buf + ofs + n;
int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);
// Find zero-length chunk (the end of the body)
while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
if (cl == 0) break; // No zero-len chunk, buffer more data
if (cl < 0) {
mg_error(c, "Invalid chunk");
break;
}
// Zero chunk found. Second pass: strip + relocate
o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
memmove(s + hm.body.len, s + o + pl, (size_t) dl);
o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
if (dl == 0) break;
}
ofs += (size_t) (n + o);
} else { // Normal, non-chunked data
size_t len = c->recv.len - ofs - (size_t) n;
if (hm.body.len > len) break; // Buffer more data
ofs += (size_t) n + hm.body.len;
}
if (next) continue; // Chunks & request were deleted
// Chunk events are delivered. If we have full body, deliver MSG
if (c->recv.len < hm.message.len) break;
if (c->is_accepted) c->is_resp = 1; // Start generating response
mg_call(c, MG_EV_HTTP_MSG, &hm); // User handler can clear is_resp
mg_iobuf_del(&c->recv, 0, hm.message.len);
}
if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs); // Delete processed data
}
(void) evd, (void) fnd;
}

40
mongoose.h
View File

@ -1145,26 +1145,25 @@ void mg_call(struct mg_connection *c, int ev, void *ev_data);
void mg_error(struct mg_connection *c, const char *fmt, ...);
enum {
MG_EV_ERROR, // Error char *error_message
MG_EV_OPEN, // Connection created NULL
MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
MG_EV_RESOLVE, // Host name is resolved NULL
MG_EV_CONNECT, // Connection established NULL
MG_EV_ACCEPT, // Connection accepted NULL
MG_EV_TLS_HS, // TLS handshake succeeded NULL
MG_EV_READ, // Data received from socket long *bytes_read
MG_EV_WRITE, // Data written to socket long *bytes_written
MG_EV_CLOSE, // Connection closed NULL
MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
MG_EV_HTTP_CHUNK, // HTTP chunk (partial msg) struct mg_http_message *
MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
MG_EV_USER // Starting ID for user events
MG_EV_ERROR, // Error char *error_message
MG_EV_OPEN, // Connection created NULL
MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
MG_EV_RESOLVE, // Host name is resolved NULL
MG_EV_CONNECT, // Connection established NULL
MG_EV_ACCEPT, // Connection accepted NULL
MG_EV_TLS_HS, // TLS handshake succeeded NULL
MG_EV_READ, // Data received from socket long *bytes_read
MG_EV_WRITE, // Data written to socket long *bytes_written
MG_EV_CLOSE, // Connection closed NULL
MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
MG_EV_USER // Starting ID for user events
};
@ -1290,7 +1289,6 @@ struct mg_http_message {
struct mg_http_header headers[MG_MAX_HTTP_HEADERS]; // Headers
struct mg_str body; // Body
struct mg_str head; // Request + headers
struct mg_str chunk; // Chunk for chunked encoding, or partial body
struct mg_str message; // Request + headers + body
};

39
src/event.h
View File

@ -7,24 +7,23 @@ void mg_call(struct mg_connection *c, int ev, void *ev_data);
void mg_error(struct mg_connection *c, const char *fmt, ...);
enum {
MG_EV_ERROR, // Error char *error_message
MG_EV_OPEN, // Connection created NULL
MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
MG_EV_RESOLVE, // Host name is resolved NULL
MG_EV_CONNECT, // Connection established NULL
MG_EV_ACCEPT, // Connection accepted NULL
MG_EV_TLS_HS, // TLS handshake succeeded NULL
MG_EV_READ, // Data received from socket long *bytes_read
MG_EV_WRITE, // Data written to socket long *bytes_written
MG_EV_CLOSE, // Connection closed NULL
MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
MG_EV_HTTP_CHUNK, // HTTP chunk (partial msg) struct mg_http_message *
MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
MG_EV_USER // Starting ID for user events
MG_EV_ERROR, // Error char *error_message
MG_EV_OPEN, // Connection created NULL
MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
MG_EV_RESOLVE, // Host name is resolved NULL
MG_EV_CONNECT, // Connection established NULL
MG_EV_ACCEPT, // Connection accepted NULL
MG_EV_TLS_HS, // TLS handshake succeeded NULL
MG_EV_READ, // Data received from socket long *bytes_read
MG_EV_WRITE, // Data written to socket long *bytes_written
MG_EV_CLOSE, // Connection closed NULL
MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
MG_EV_USER // Starting ID for user events
};

163
src/http.c
View File

@ -245,8 +245,7 @@ int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
hm->message.ptr = hm->head.ptr = s;
hm->body.ptr = end;
hm->head.len = (size_t) req_len;
hm->chunk.ptr = end;
hm->message.len = hm->body.len = (size_t) ~0; // Set body length to infinite
hm->message.len = hm->body.len = (size_t) -1; // Set body length to infinite
// Parse request line
hm->method.ptr = s;
@ -951,131 +950,75 @@ static bool is_hex_digit(int c) {
(c >= 'A' && c <= 'F');
}
// If a server sends data to the client using chunked encoding, Mongoose strips
// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event. When zero
// chunk is received, we fire MG_EV_HTTP_MSG, and the body already has all
// chunking prefixes/suffixes stripped.
//
// If a server sends data without chunked encoding, we also fire a series of
// MG_EV_HTTP_CHUNK events for every received piece of data, and then we fire
// MG_EV_HTTP_MSG event in the end.
//
// We track total processed length in the c->pfn_data, which is a void *
// pointer: we store a size_t value there.
static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
size_t i = 0, n;
while (i < s.len && is_hex_digit(s.ptr[i])) i++;
n = mg_unhexn(s.ptr, i);
// MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
if (s.len < i + n + 4) return false; // Chunk not yet fully buffered
if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
*prefixlen = i + 2;
*datalen = n;
return true;
static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
int i = 0, n = 0;
if (len < 3) return 0;
while (i < len && is_hex_digit(buf[i])) i++;
if (len < i + 1 || buf[i] != '\r' || buf[i + 1] != '\n') return -1; // Error
n = (int) mg_unhexn(buf, (size_t) i); // Decode hex length
if (n < 0) return -1; // Error
if (len < i + n + 4) return 0; // Chunk not yet fully buffered
if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1; // Error
*pl = i + 2, *dl = n;
return i + 2 + n + 2;
}
static bool mg_is_chunked(struct mg_http_message *hm) {
static bool is_chunked(struct mg_http_message *hm) {
const char *needle = "chunked";
struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
return te != NULL && mg_vcasecmp(te, needle) == 0;
}
void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
}
static void deliver_chunked_chunks(struct mg_connection *c, size_t hlen,
struct mg_http_message *hm, bool *next) {
// | ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
// +------------------+--------------------------+----
// | hlen | chunk1 | ......
char *buf = (char *) &c->recv.buf[hlen], *p = buf;
size_t len = c->recv.len - hlen;
size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
size_t mark, pl, dl, del = 0, ofs = 0;
bool last = false;
if (processed <= len) len -= processed, buf += processed;
while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
size_t saved = c->recv.len;
memmove(p + processed, buf + ofs + pl, dl);
// MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
hm->chunk = mg_str_n(p + processed, dl);
mg_call(c, MG_EV_HTTP_CHUNK, hm);
ofs += pl + dl + 2, del += pl + 2; // 2 is for \r\n suffix
processed += dl;
if (c->recv.len != saved) processed -= dl, buf -= dl;
// mg_hexdump(c->recv.buf, hlen + processed);
last = (dl == 0);
}
mg_iobuf_del(&c->recv, hlen + processed, del);
mark = ((size_t) c->pfn_data) & MG_DMARK;
c->pfn_data = (void *) (processed | mark);
if (last) {
hm->body.len = processed;
hm->message.len = hlen + processed;
c->pfn_data = NULL;
if (mark) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
// MG_INFO(("LAST, mark: %lx", mark));
// mg_hexdump(c->recv.buf, c->recv.len);
}
}
static void deliver_normal_chunks(struct mg_connection *c, size_t hlen,
struct mg_http_message *hm, bool *next) {
size_t left, processed = ((size_t) c->pfn_data) & ~MG_DMARK;
size_t deleted = ((size_t) c->pfn_data) & MG_DMARK;
hm->chunk = mg_str_n((char *) &c->recv.buf[hlen], c->recv.len - hlen);
if (processed <= hm->chunk.len && !deleted) {
hm->chunk.len -= processed;
hm->chunk.ptr += processed;
}
left = hm->body.len < processed ? 0 : hm->body.len - processed;
if (hm->chunk.len > left) hm->chunk.len = left;
if (hm->chunk.len > 0) mg_call(c, MG_EV_HTTP_CHUNK, hm);
processed += hm->chunk.len;
deleted = ((size_t) c->pfn_data) & MG_DMARK; // Re-evaluate after user call
if (processed >= hm->body.len) { // Last, 0-len chunk
hm->chunk.len = 0; // Reset length
mg_call(c, MG_EV_HTTP_CHUNK, hm); // Call user handler
c->pfn_data = NULL; // Reset processed counter
if (processed && deleted) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
} else {
c->pfn_data = (void *) (processed | deleted); // if it is set
}
}
static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
struct mg_http_message hm;
while (c->recv.buf != NULL && c->recv.len > 0) {
bool next = false;
int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
if (hlen < 0) {
mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
break;
size_t ofs = 0; // Parsing offset
while (c->is_resp == 0 && ofs < c->recv.len) {
const char *buf = (char *) c->recv.buf + ofs;
int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
if (n < 0) {
mg_error(c, "HTTP parse");
return;
}
if (c->is_resp) break; // Response is still generated
if (hlen == 0) break; // Request is not buffered yet
if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
hm.message.len = c->recv.len; // and closes now, deliver a MSG
if (n == 0) break; // Request is not buffered yet
if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
hm.message.len = c->recv.len - ofs; // and closes now, deliver MSG
hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
}
if (mg_is_chunked(&hm)) {
deliver_chunked_chunks(c, (size_t) hlen, &hm, &next);
} else {
deliver_normal_chunks(c, (size_t) hlen, &hm, &next);
if (is_chunked(&hm)) {
// For chunked data, strip off prefixes and suffixes from chunks
// and relocate them right after the headers, then report a message
char *s = (char *) c->recv.buf + ofs + n;
int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);
// Find zero-length chunk (the end of the body)
while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
if (cl == 0) break; // No zero-len chunk, buffer more data
if (cl < 0) {
mg_error(c, "Invalid chunk");
break;
}
// Zero chunk found. Second pass: strip + relocate
o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
memmove(s + hm.body.len, s + o + pl, (size_t) dl);
o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
if (dl == 0) break;
}
ofs += (size_t) (n + o);
} else { // Normal, non-chunked data
size_t len = c->recv.len - ofs - (size_t) n;
if (hm.body.len > len) break; // Buffer more data
ofs += (size_t) n + hm.body.len;
}
if (next) continue; // Chunks & request were deleted
// Chunk events are delivered. If we have full body, deliver MSG
if (c->recv.len < hm.message.len) break;
if (c->is_accepted) c->is_resp = 1; // Start generating response
mg_call(c, MG_EV_HTTP_MSG, &hm); // User handler can clear is_resp
mg_iobuf_del(&c->recv, 0, hm.message.len);
}
if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs); // Delete processed data
}
(void) evd, (void) fnd;
}

1
src/http.h
View File

@ -16,7 +16,6 @@ struct mg_http_message {
struct mg_http_header headers[MG_MAX_HTTP_HEADERS]; // Headers
struct mg_str body; // Body
struct mg_str head; // Request + headers
struct mg_str chunk; // Chunk for chunked encoding, or partial body
struct mg_str message; // Request + headers + body
};

2
test/fuzz.c
View File

@ -2,7 +2,7 @@
#define MG_ENABLE_LOG 0
#define MG_ENABLE_LINES 1
#define MG_ENABLE_TCPIP 1
#define MG_IO_SIZE (4 * 1024 * 1024) // 4M IO size, unless resize is too slow
#define MG_IO_SIZE (32 * 1024 * 1024) // Big IO size for fast resizes
#include "mongoose.c"

87
test/unit_test.c
View File

@ -1342,8 +1342,6 @@ static void f4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
"abcdef");
strcat((char *) fn_data, "m");
c->is_draining = 1;
} else if (ev == MG_EV_HTTP_CHUNK) {
strcat((char *) fn_data, "f");
} else if (ev == MG_EV_CLOSE) {
strcat((char *) fn_data, "c");
}
@ -1356,10 +1354,6 @@ static void f4c(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
ASSERT(mg_strcmp(hm->body, mg_str("/foo/bar/abcdef")) == 0);
strcat((char *) fn_data, "m");
} else if (ev == MG_EV_HTTP_CHUNK) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
MG_INFO(("FS [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
strcat((char *) fn_data, "f");
} else if (ev == MG_EV_CLOSE) {
strcat((char *) fn_data, "c");
}
@ -1375,8 +1369,8 @@ static void test_http_no_content_length(void) {
mg_http_connect(&mgr, url, f4c, (void *) buf2);
for (i = 0; i < 1000 && strchr(buf2, 'c') == NULL; i++) mg_mgr_poll(&mgr, 10);
MG_INFO(("[%s] [%s]", buf1, buf2));
ASSERT(strcmp(buf1, "fmc") == 0);
ASSERT(strcmp(buf2, "fcfm") == 0); // See #1475
ASSERT(strcmp(buf1, "mc") == 0);
ASSERT(strcmp(buf2, "cm") == 0); // See #1475
mg_mgr_free(&mgr);
ASSERT(mgr.conns == NULL);
}
@ -2290,20 +2284,8 @@ static void test_crc32(void) {
}
static void us(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
int del = *(int *) fn_data;
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
MG_DEBUG(("Got chunk len %lu", (unsigned long) hm->chunk.len));
MG_DEBUG(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
// MG_DEBUG(("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
if (del) {
mg_http_delete_chunk(c, hm);
if (hm->chunk.len == 0) {
MG_DEBUG(("Last chunk received, sending response"));
mg_http_reply(c, 200, "", "ok (chunked)\n");
}
}
} else if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
MG_DEBUG(("Got all %lu bytes!", (unsigned long) hm->body.len));
MG_DEBUG(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
// MG_DEBUG(("Body:\n%.*s", (int) hm->body.len, hm->body.ptr));
@ -2318,7 +2300,7 @@ static void us(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
const char **s = (const char **) fn_data;
if (ev == MG_EV_OPEN) {
c->is_hexdumping = 1;
// c->is_hexdumping = 1;
} else if (ev == MG_EV_CONNECT) {
mg_printf(c,
"POST /upload HTTP/1.0\r\n"
@ -2328,6 +2310,7 @@ static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
mg_http_printf_chunk(c, "");
} else if (ev == MG_EV_HTTP_MSG) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
// MG_INFO(("---> [%s] [%.*s]", *s, hm->body.len, hm->body.ptr));
ASSERT(mg_strcmp(hm->body, mg_str(*s)) == 0);
*s = NULL;
}
@ -2337,22 +2320,14 @@ static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
static void test_http_upload(void) {
struct mg_mgr mgr;
const char *url = "http://127.0.0.1:12352";
int i, del = 1;
const char *s1 = "ok (chunked)\n";
const char *s2 = "ok (8 foo\nbar\n)\n";
int i;
const char *s = "ok (8 foo\nbar\n)\n";
mg_mgr_init(&mgr);
mg_http_listen(&mgr, url, us, (void *) &del);
mg_http_connect(&mgr, url, uc, (void *) &s1);
mg_http_listen(&mgr, url, us, NULL);
mg_http_connect(&mgr, url, uc, (void *) &s);
for (i = 0; i < 20; i++) mg_mgr_poll(&mgr, 5);
ASSERT(s1 == NULL);
del = 0;
mg_http_connect(&mgr, url, uc, (void *) &s2);
for (i = 0; i < 20; i++) mg_mgr_poll(&mgr, 5);
ASSERT(s2 == NULL);
ASSERT(s == NULL);
mg_mgr_free(&mgr);
ASSERT(mgr.conns == NULL);
}
@ -2363,7 +2338,7 @@ static void eX(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
if (ev == MG_EV_HTTP_MSG) {
mg_printf(c, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n");
c->data[0] = 1;
c->is_hexdumping = 1;
// c->is_hexdumping = 1;
} else if (ev == MG_EV_POLL && c->data[0] != 0) {
c->data[0]++;
if (c->data[0] == 10) mg_http_printf_chunk(c, "a");
@ -2403,32 +2378,13 @@ static void eZ(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
// Do not delete chunks as they arrive
static void eh4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
uint32_t *crc = (uint32_t *) fn_data;
if (ev == MG_EV_HTTP_CHUNK) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
*crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
*crc = mg_crc32(*crc, "x", 1);
MG_INFO(("%lu C [%.*s]", c->id, (int) hm->chunk.len, hm->chunk.ptr));
} else if (ev == MG_EV_HTTP_MSG) {
if (ev == MG_EV_HTTP_MSG) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
*crc = mg_crc32(*crc, hm->body.ptr, hm->body.len);
MG_INFO(("%lu M [%.*s]", c->id, (int) hm->body.len, hm->body.ptr));
}
}
// Streaming client event handler. Delete chunks as they arrive
static void eh5(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
uint32_t *crc = (uint32_t *) fn_data;
if (ev == MG_EV_HTTP_CHUNK) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
*crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
*crc = mg_crc32(*crc, "x", 1);
MG_INFO(("%lu DELC [%.*s]", c->id, (int) hm->chunk.len, hm->chunk.ptr));
mg_http_delete_chunk(c, hm);
} else if (ev == MG_EV_HTTP_MSG) {
ASSERT(0); // Must not be here, MSG must not be fired: chunks deleted!
}
}
static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
int req_count, const char *expected) {
char url[100];
@ -2446,6 +2402,7 @@ static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
for (i = 0; i < 100 && crc != expected_crc; i++) {
mg_mgr_poll(&mgr, 1);
}
// MG_INFO(("-------- %d [%s]", i, expected));
ASSERT(i < 100);
ASSERT(crc == expected_crc);
mg_mgr_free(&mgr);
@ -2454,20 +2411,14 @@ static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
static void test_http_chunked(void) {
// Non-chunked encoding
test_http_chunked_case(eY, eh4, 1, "axbcxdxxabcd"); // Chunks not deleted
test_http_chunked_case(eY, eh4, 2, "axbcxdxxabcdaxbcxdxxabcd");
test_http_chunked_case(eY, eh5, 1, "axbcxdxx"); // Chunks deleted
test_http_chunked_case(eY, eh5, 2, "axbcxdxxaxbcxdxx");
test_http_chunked_case(eZ, eh4, 1, "abcdxxabcd"); // Not deleted
test_http_chunked_case(eZ, eh4, 2, "abcdxxabcdabcdxxabcd");
test_http_chunked_case(eZ, eh5, 1, "abcdxx"); // Deleted
test_http_chunked_case(eZ, eh5, 2, "abcdxxabcdxx");
test_http_chunked_case(eY, eh4, 1, "abcd"); // Chunks not deleted
test_http_chunked_case(eY, eh4, 2, "abcdabcd");
test_http_chunked_case(eZ, eh4, 1, "abcd"); // Not deleted
test_http_chunked_case(eZ, eh4, 2, "abcdabcd");
// Chunked encoding
test_http_chunked_case(eX, eh4, 1, "axbxcxdxxabcd"); // Chunks not deleted
test_http_chunked_case(eX, eh5, 1, "axbxcxdxx"); // Chunks deleted
test_http_chunked_case(eX, eh4, 2, "axbxcxdxxabcdaxbxcxdxxabcd");
test_http_chunked_case(eX, eh5, 2, "axbxcxdxxaxbxcxdxx");
test_http_chunked_case(eX, eh4, 1, "abcd"); // Chunks not deleted
test_http_chunked_case(eX, eh4, 2, "abcdabcd");
}
static void test_invalid_listen_addr(void) {