diff --git a/examples/file-upload-single-post/main.c b/examples/file-upload-single-post/main.c
index 1c462cc4..982e918e 100644
--- a/examples/file-upload-single-post/main.c
+++ b/examples/file-upload-single-post/main.c
@@ -1,8 +1,8 @@
// Copyright (c) 2020 Cesanta Software Limited
// All rights reserved
//
-// Streaming upload example. Demonstrates how to use MG_EV_HTTP_CHUNK events
-// to send large payload in smaller chunks. To test, use curl utility:
+// Streaming upload example. Demonstrates how to use MG_EV_READ events
+// to get large payload in smaller chunks. To test, use curl utility:
//
// curl http://localhost:8000/upload?name=a.txt --data-binary @large_file.txt
@@ -12,21 +12,33 @@
// /upload - Saves the next file chunk
// all other URI - serves web_root/ directory
static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
- MG_INFO(("Got chunk len %lu", (unsigned long) hm->chunk.len));
- MG_INFO(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
- // MG_INFO(("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
- mg_http_delete_chunk(c, hm);
- if (hm->chunk.len == 0) {
- MG_INFO(("Last chunk received, sending response"));
- mg_http_reply(c, 200, "", "ok (chunked)\n");
+ if (ev == MG_EV_READ) {
+ // Parse the incoming data ourselves. If we can parse the request,
+ // store two size_t variables in the c->data: expected len and recv len.
+ size_t *data = (size_t *) c->data;
+ if (data[0]) { // Already parsed, simply print received data
+ data[1] += c->recv.len;
+ MG_INFO(("Got chunk len %lu, %lu total", c->recv.len, data[1]));
+ c->recv.len = 0; // And cleanup the receive buffer. Streaming!
+ if (data[1] >= data[0]) mg_http_reply(c, 200, "", "ok\n");
+ } else {
+ struct mg_http_message hm;
+ int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+ if (n < 0) mg_error(c, "Bad response");
+ if (n > 0) {
+ if (mg_http_match_uri(&hm, "/upload")) {
+ MG_INFO(("Got chunk len %lu", c->recv.len - n));
+ data[0] = hm.body.len;
+ data[1] = c->recv.len - n;
+ if (data[1] >= data[0]) mg_http_reply(c, 200, "", "ok\n");
+ } else {
+ struct mg_http_serve_opts opts = {.root_dir = "web_root"};
+ mg_http_serve_dir(c, &hm, &opts);
+ }
+ }
}
- } else if (ev == MG_EV_HTTP_MSG) {
- struct mg_http_serve_opts opts = {.root_dir = "web_root"};
- mg_http_serve_dir(c, hm, &opts);
}
- (void) fn_data;
+ (void) fn_data, (void) ev_data;
}
int main(void) {
diff --git a/examples/file-upload-single-post/web_root/index.html b/examples/file-upload-single-post/web_root/index.html
index 6ae10166..4e692b31 100644
--- a/examples/file-upload-single-post/web_root/index.html
+++ b/examples/file-upload-single-post/web_root/index.html
@@ -19,12 +19,10 @@
of RAM to buffer everything.
In order to upload large files to a memory-constrained system, use
- MG_EV_HTTP_CHUNK on a server side. It fires when
- a partial HTTP message has been received (or a chunk-encoded chunk).
- Use mg_http_delete_chunk() to release chunk memory.
- When 0-sized chunk is received, that's the end of the message.
- Use MG_MAX_RECV_SIZE build constant to limit
- maximum chunk size on a server side.
+ MG_EV_READ on a server side. It fires when
+ a partial HTTP message has been received. We discourage this method,
+ since the client can send chunked-encoded data. Instead, please
+ split the upload into smaller pieces and send sequentially.
In this example, JavaScript code uses "fetch()" browser API.
Uploaded file is not saved, but rather printed by server side.
diff --git a/examples/http-streaming-client/main.c b/examples/http-streaming-client/main.c
index d1a83f74..5c46a1a8 100644
--- a/examples/http-streaming-client/main.c
+++ b/examples/http-streaming-client/main.c
@@ -1,14 +1,12 @@
// Copyright (c) 2021 Cesanta Software Limited
// All rights reserved
//
-// Example HTTP client.
-// Instead of buffering the whole HTTP response in memory and wait for the
-// MG_EV_HTTP_MSG event, this client catches MG_EV_HTTP_CHUNK events and
-// prints them. This is useful to download large content without spooling
-// everything to memory, or catch chunked content.
+// Example streaming HTTP client.
+// The default HTTP handler waits until the whole HTTP message is buffered in
+// c->recv IO buffer. It can be large. This example shows how to receive
+// a potentially large response in chunks by handling the MG_EV_READ events.
//
// You can change `s_url` from the command line by executing: ./example YOUR_URL
-//
// To enable SSL/TLS, see https://mongoose.ws/tutorials/tls/#how-to-build
#include "mongoose.h"
@@ -21,13 +19,11 @@ static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
if (ev == MG_EV_CONNECT) {
// Connected to server. Extract host name from URL
struct mg_str host = mg_url_host(s_url);
-
if (mg_url_is_ssl(s_url)) {
struct mg_tls_opts opts = {.ca = mg_unpacked("/certs/ca.pem"),
.name = host};
mg_tls_init(c, &opts);
}
-
// Send request
mg_printf(c,
"GET %s HTTP/1.1\r\n"
@@ -36,21 +32,33 @@ static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
"Host: %.*s\r\n"
"\r\n",
mg_url_uri(s_url), (int) host.len, host.ptr);
- } else if (ev == MG_EV_HTTP_CHUNK) {
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- fwrite(hm->chunk.ptr, 1, hm->chunk.len, stdout);
- // fprintf(stderr, "c %u\n", (unsigned) hm->chunk.len);
- mg_http_delete_chunk(c, hm);
- if (hm->chunk.len == 0) *(bool *) fn_data = true; // Last chunk
- } else if (ev == MG_EV_HTTP_MSG) {
- // Response is received. Print it
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- fwrite(hm->body.ptr, 1, hm->body.len, stdout);
- c->is_draining = 1; // Tell mongoose to close this connection
- *(bool *) fn_data = true; // Tell event loop to stop
+ } else if (ev == MG_EV_READ) {
+ // Parse the incoming data ourselves. If we can parse the request,
+ // store two size_t variables in the c->data: expected len and recv len.
+ size_t *data = (size_t *) c->data;
+ if (data[0]) { // Already parsed, simply print received data
+ data[1] += c->recv.len;
+ fwrite(c->recv.buf, 1, c->recv.len, stdout);
+ c->recv.len = 0; // And cleanup the receive buffer. Streming!
+ if (data[1] >= data[0]) * (bool *) fn_data = true;
+ } else {
+ struct mg_http_message hm;
+ int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+ if (n < 0) mg_error(c, "Bad response");
+ if (n > 0) {
+ fwrite(c->recv.buf + n, 1, c->recv.len - n, stdout); // Print body
+ data[0] = n + hm.body.len;
+ data[1] = c->recv.len;
+ c->recv.len = 0; // Cleanup receive buffer
+ if (data[1] >= data[0]) * (bool *) fn_data = true;
+ }
+ }
+ } else if (ev == MG_EV_CLOSE) {
+ *(bool *) fn_data = true; // tell event loop to stop
} else if (ev == MG_EV_ERROR) {
*(bool *) fn_data = true; // Error, tell event loop to stop
}
+ (void) ev_data;
}
int main(int argc, char *argv[]) {
@@ -59,7 +67,7 @@ int main(int argc, char *argv[]) {
const char *log_level = getenv("V"); // Log level
mg_mgr_init(&mgr); // Initialise event manager
- if (log_level == NULL) log_level = "3"; // If not set, set to DEBUG
+ if (log_level == NULL) log_level = "2"; // If not set, set to DEBUG
mg_log_set(atoi(log_level)); // Set to 0 to disable debug log
if (argc > 1) s_url = argv[1]; // Use URL from command line
diff --git a/mongoose.c b/mongoose.c
index 6fdd7444..0930d481 100644
--- a/mongoose.c
+++ b/mongoose.c
@@ -1984,8 +1984,7 @@ int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
hm->message.ptr = hm->head.ptr = s;
hm->body.ptr = end;
hm->head.len = (size_t) req_len;
- hm->chunk.ptr = end;
- hm->message.len = hm->body.len = (size_t) ~0; // Set body length to infinite
+ hm->message.len = hm->body.len = (size_t) -1; // Set body length to infinite
// Parse request line
hm->method.ptr = s;
@@ -2690,131 +2689,75 @@ static bool is_hex_digit(int c) {
(c >= 'A' && c <= 'F');
}
-// If a server sends data to the client using chunked encoding, Mongoose strips
-// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
-// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event. When zero
-// chunk is received, we fire MG_EV_HTTP_MSG, and the body already has all
-// chunking prefixes/suffixes stripped.
-//
-// If a server sends data without chunked encoding, we also fire a series of
-// MG_EV_HTTP_CHUNK events for every received piece of data, and then we fire
-// MG_EV_HTTP_MSG event in the end.
-//
-// We track total processed length in the c->pfn_data, which is a void *
-// pointer: we store a size_t value there.
-static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
- size_t i = 0, n;
- while (i < s.len && is_hex_digit(s.ptr[i])) i++;
- n = mg_unhexn(s.ptr, i);
- // MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
- if (s.len < i + n + 4) return false; // Chunk not yet fully buffered
- if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
- if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
- *prefixlen = i + 2;
- *datalen = n;
- return true;
+static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
+ int i = 0, n = 0;
+ if (len < 3) return 0;
+ while (i < len && is_hex_digit(buf[i])) i++;
+ if (len < i + 1 || buf[i] != '\r' || buf[i + 1] != '\n') return -1; // Error
+ n = (int) mg_unhexn(buf, (size_t) i); // Decode hex length
+ if (n < 0) return -1; // Error
+ if (len < i + n + 4) return 0; // Chunk not yet fully buffered
+ if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1; // Error
+ *pl = i + 2, *dl = n;
+ return i + 2 + n + 2;
}
-static bool mg_is_chunked(struct mg_http_message *hm) {
+static bool is_chunked(struct mg_http_message *hm) {
const char *needle = "chunked";
struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
return te != NULL && mg_vcasecmp(te, needle) == 0;
}
-void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
- size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
- mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
- c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
-}
-
-static void deliver_chunked_chunks(struct mg_connection *c, size_t hlen,
- struct mg_http_message *hm, bool *next) {
- // | ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
- // +------------------+--------------------------+----
- // | hlen | chunk1 | ......
- char *buf = (char *) &c->recv.buf[hlen], *p = buf;
- size_t len = c->recv.len - hlen;
- size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
- size_t mark, pl, dl, del = 0, ofs = 0;
- bool last = false;
- if (processed <= len) len -= processed, buf += processed;
- while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
- size_t saved = c->recv.len;
- memmove(p + processed, buf + ofs + pl, dl);
- // MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
- hm->chunk = mg_str_n(p + processed, dl);
- mg_call(c, MG_EV_HTTP_CHUNK, hm);
- ofs += pl + dl + 2, del += pl + 2; // 2 is for \r\n suffix
- processed += dl;
- if (c->recv.len != saved) processed -= dl, buf -= dl;
- // mg_hexdump(c->recv.buf, hlen + processed);
- last = (dl == 0);
- }
- mg_iobuf_del(&c->recv, hlen + processed, del);
- mark = ((size_t) c->pfn_data) & MG_DMARK;
- c->pfn_data = (void *) (processed | mark);
- if (last) {
- hm->body.len = processed;
- hm->message.len = hlen + processed;
- c->pfn_data = NULL;
- if (mark) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
- // MG_INFO(("LAST, mark: %lx", mark));
- // mg_hexdump(c->recv.buf, c->recv.len);
- }
-}
-
-static void deliver_normal_chunks(struct mg_connection *c, size_t hlen,
- struct mg_http_message *hm, bool *next) {
- size_t left, processed = ((size_t) c->pfn_data) & ~MG_DMARK;
- size_t deleted = ((size_t) c->pfn_data) & MG_DMARK;
- hm->chunk = mg_str_n((char *) &c->recv.buf[hlen], c->recv.len - hlen);
- if (processed <= hm->chunk.len && !deleted) {
- hm->chunk.len -= processed;
- hm->chunk.ptr += processed;
- }
- left = hm->body.len < processed ? 0 : hm->body.len - processed;
- if (hm->chunk.len > left) hm->chunk.len = left;
- if (hm->chunk.len > 0) mg_call(c, MG_EV_HTTP_CHUNK, hm);
- processed += hm->chunk.len;
- deleted = ((size_t) c->pfn_data) & MG_DMARK; // Re-evaluate after user call
- if (processed >= hm->body.len) { // Last, 0-len chunk
- hm->chunk.len = 0; // Reset length
- mg_call(c, MG_EV_HTTP_CHUNK, hm); // Call user handler
- c->pfn_data = NULL; // Reset processed counter
- if (processed && deleted) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
- } else {
- c->pfn_data = (void *) (processed | deleted); // if it is set
- }
-}
-
static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
struct mg_http_message hm;
- while (c->recv.buf != NULL && c->recv.len > 0) {
- bool next = false;
- int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
- if (hlen < 0) {
- mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
- break;
+ size_t ofs = 0; // Parsing offset
+
+ while (c->is_resp == 0 && ofs < c->recv.len) {
+ const char *buf = (char *) c->recv.buf + ofs;
+ int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
+ if (n < 0) {
+ mg_error(c, "HTTP parse");
+ return;
}
- if (c->is_resp) break; // Response is still generated
- if (hlen == 0) break; // Request is not buffered yet
- if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
- hm.message.len = c->recv.len; // and closes now, deliver a MSG
+ if (n == 0) break; // Request is not buffered yet
+ if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
+ hm.message.len = c->recv.len - ofs; // and closes now, deliver MSG
hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
}
- if (mg_is_chunked(&hm)) {
- deliver_chunked_chunks(c, (size_t) hlen, &hm, &next);
- } else {
- deliver_normal_chunks(c, (size_t) hlen, &hm, &next);
+
+ if (is_chunked(&hm)) {
+ // For chunked data, strip off prefixes and suffixes from chunks
+ // and relocate them right after the headers, then report a message
+ char *s = (char *) c->recv.buf + ofs + n;
+ int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);
+
+ // Find zero-length chunk (the end of the body)
+ while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
+ if (cl == 0) break; // No zero-len chunk, buffer more data
+ if (cl < 0) {
+ mg_error(c, "Invalid chunk");
+ break;
+ }
+
+ // Zero chunk found. Second pass: strip + relocate
+ o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
+ while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
+ memmove(s + hm.body.len, s + o + pl, (size_t) dl);
+ o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
+ if (dl == 0) break;
+ }
+ ofs += (size_t) (n + o);
+ } else { // Normal, non-chunked data
+ size_t len = c->recv.len - ofs - (size_t) n;
+ if (hm.body.len > len) break; // Buffer more data
+ ofs += (size_t) n + hm.body.len;
}
- if (next) continue; // Chunks & request were deleted
- // Chunk events are delivered. If we have full body, deliver MSG
- if (c->recv.len < hm.message.len) break;
+
if (c->is_accepted) c->is_resp = 1; // Start generating response
mg_call(c, MG_EV_HTTP_MSG, &hm); // User handler can clear is_resp
- mg_iobuf_del(&c->recv, 0, hm.message.len);
}
+ if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs); // Delete processed data
}
(void) evd, (void) fnd;
}
diff --git a/mongoose.h b/mongoose.h
index de7debda..73058509 100644
--- a/mongoose.h
+++ b/mongoose.h
@@ -1145,26 +1145,25 @@ void mg_call(struct mg_connection *c, int ev, void *ev_data);
void mg_error(struct mg_connection *c, const char *fmt, ...);
enum {
- MG_EV_ERROR, // Error char *error_message
- MG_EV_OPEN, // Connection created NULL
- MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
- MG_EV_RESOLVE, // Host name is resolved NULL
- MG_EV_CONNECT, // Connection established NULL
- MG_EV_ACCEPT, // Connection accepted NULL
- MG_EV_TLS_HS, // TLS handshake succeeded NULL
- MG_EV_READ, // Data received from socket long *bytes_read
- MG_EV_WRITE, // Data written to socket long *bytes_written
- MG_EV_CLOSE, // Connection closed NULL
- MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
- MG_EV_HTTP_CHUNK, // HTTP chunk (partial msg) struct mg_http_message *
- MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
- MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
- MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
- MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
- MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
- MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
- MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
- MG_EV_USER // Starting ID for user events
+ MG_EV_ERROR, // Error char *error_message
+ MG_EV_OPEN, // Connection created NULL
+ MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
+ MG_EV_RESOLVE, // Host name is resolved NULL
+ MG_EV_CONNECT, // Connection established NULL
+ MG_EV_ACCEPT, // Connection accepted NULL
+ MG_EV_TLS_HS, // TLS handshake succeeded NULL
+ MG_EV_READ, // Data received from socket long *bytes_read
+ MG_EV_WRITE, // Data written to socket long *bytes_written
+ MG_EV_CLOSE, // Connection closed NULL
+ MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
+ MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
+ MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
+ MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
+ MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
+ MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
+ MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
+ MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
+ MG_EV_USER // Starting ID for user events
};
@@ -1290,7 +1289,6 @@ struct mg_http_message {
struct mg_http_header headers[MG_MAX_HTTP_HEADERS]; // Headers
struct mg_str body; // Body
struct mg_str head; // Request + headers
- struct mg_str chunk; // Chunk for chunked encoding, or partial body
struct mg_str message; // Request + headers + body
};
diff --git a/src/event.h b/src/event.h
index 769c12af..da7a938a 100644
--- a/src/event.h
+++ b/src/event.h
@@ -7,24 +7,23 @@ void mg_call(struct mg_connection *c, int ev, void *ev_data);
void mg_error(struct mg_connection *c, const char *fmt, ...);
enum {
- MG_EV_ERROR, // Error char *error_message
- MG_EV_OPEN, // Connection created NULL
- MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
- MG_EV_RESOLVE, // Host name is resolved NULL
- MG_EV_CONNECT, // Connection established NULL
- MG_EV_ACCEPT, // Connection accepted NULL
- MG_EV_TLS_HS, // TLS handshake succeeded NULL
- MG_EV_READ, // Data received from socket long *bytes_read
- MG_EV_WRITE, // Data written to socket long *bytes_written
- MG_EV_CLOSE, // Connection closed NULL
- MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
- MG_EV_HTTP_CHUNK, // HTTP chunk (partial msg) struct mg_http_message *
- MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
- MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
- MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
- MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
- MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
- MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
- MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
- MG_EV_USER // Starting ID for user events
+ MG_EV_ERROR, // Error char *error_message
+ MG_EV_OPEN, // Connection created NULL
+ MG_EV_POLL, // mg_mgr_poll iteration uint64_t *uptime_millis
+ MG_EV_RESOLVE, // Host name is resolved NULL
+ MG_EV_CONNECT, // Connection established NULL
+ MG_EV_ACCEPT, // Connection accepted NULL
+ MG_EV_TLS_HS, // TLS handshake succeeded NULL
+ MG_EV_READ, // Data received from socket long *bytes_read
+ MG_EV_WRITE, // Data written to socket long *bytes_written
+ MG_EV_CLOSE, // Connection closed NULL
+ MG_EV_HTTP_MSG, // HTTP request/response struct mg_http_message *
+ MG_EV_WS_OPEN, // Websocket handshake done struct mg_http_message *
+ MG_EV_WS_MSG, // Websocket msg, text or bin struct mg_ws_message *
+ MG_EV_WS_CTL, // Websocket control msg struct mg_ws_message *
+ MG_EV_MQTT_CMD, // MQTT low-level command struct mg_mqtt_message *
+ MG_EV_MQTT_MSG, // MQTT PUBLISH received struct mg_mqtt_message *
+ MG_EV_MQTT_OPEN, // MQTT CONNACK received int *connack_status_code
+ MG_EV_SNTP_TIME, // SNTP time received uint64_t *epoch_millis
+ MG_EV_USER // Starting ID for user events
};
diff --git a/src/http.c b/src/http.c
index f4850fa0..d617a661 100644
--- a/src/http.c
+++ b/src/http.c
@@ -245,8 +245,7 @@ int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
hm->message.ptr = hm->head.ptr = s;
hm->body.ptr = end;
hm->head.len = (size_t) req_len;
- hm->chunk.ptr = end;
- hm->message.len = hm->body.len = (size_t) ~0; // Set body length to infinite
+ hm->message.len = hm->body.len = (size_t) -1; // Set body length to infinite
// Parse request line
hm->method.ptr = s;
@@ -951,131 +950,75 @@ static bool is_hex_digit(int c) {
(c >= 'A' && c <= 'F');
}
-// If a server sends data to the client using chunked encoding, Mongoose strips
-// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
-// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event. When zero
-// chunk is received, we fire MG_EV_HTTP_MSG, and the body already has all
-// chunking prefixes/suffixes stripped.
-//
-// If a server sends data without chunked encoding, we also fire a series of
-// MG_EV_HTTP_CHUNK events for every received piece of data, and then we fire
-// MG_EV_HTTP_MSG event in the end.
-//
-// We track total processed length in the c->pfn_data, which is a void *
-// pointer: we store a size_t value there.
-static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
- size_t i = 0, n;
- while (i < s.len && is_hex_digit(s.ptr[i])) i++;
- n = mg_unhexn(s.ptr, i);
- // MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
- if (s.len < i + n + 4) return false; // Chunk not yet fully buffered
- if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
- if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
- *prefixlen = i + 2;
- *datalen = n;
- return true;
+static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
+ int i = 0, n = 0;
+ if (len < 3) return 0;
+ while (i < len && is_hex_digit(buf[i])) i++;
+ if (len < i + 1 || buf[i] != '\r' || buf[i + 1] != '\n') return -1; // Error
+ n = (int) mg_unhexn(buf, (size_t) i); // Decode hex length
+ if (n < 0) return -1; // Error
+ if (len < i + n + 4) return 0; // Chunk not yet fully buffered
+ if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1; // Error
+ *pl = i + 2, *dl = n;
+ return i + 2 + n + 2;
}
-static bool mg_is_chunked(struct mg_http_message *hm) {
+static bool is_chunked(struct mg_http_message *hm) {
const char *needle = "chunked";
struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
return te != NULL && mg_vcasecmp(te, needle) == 0;
}
-void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
- size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
- mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
- c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
-}
-
-static void deliver_chunked_chunks(struct mg_connection *c, size_t hlen,
- struct mg_http_message *hm, bool *next) {
- // | ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
- // +------------------+--------------------------+----
- // | hlen | chunk1 | ......
- char *buf = (char *) &c->recv.buf[hlen], *p = buf;
- size_t len = c->recv.len - hlen;
- size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
- size_t mark, pl, dl, del = 0, ofs = 0;
- bool last = false;
- if (processed <= len) len -= processed, buf += processed;
- while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
- size_t saved = c->recv.len;
- memmove(p + processed, buf + ofs + pl, dl);
- // MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
- hm->chunk = mg_str_n(p + processed, dl);
- mg_call(c, MG_EV_HTTP_CHUNK, hm);
- ofs += pl + dl + 2, del += pl + 2; // 2 is for \r\n suffix
- processed += dl;
- if (c->recv.len != saved) processed -= dl, buf -= dl;
- // mg_hexdump(c->recv.buf, hlen + processed);
- last = (dl == 0);
- }
- mg_iobuf_del(&c->recv, hlen + processed, del);
- mark = ((size_t) c->pfn_data) & MG_DMARK;
- c->pfn_data = (void *) (processed | mark);
- if (last) {
- hm->body.len = processed;
- hm->message.len = hlen + processed;
- c->pfn_data = NULL;
- if (mark) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
- // MG_INFO(("LAST, mark: %lx", mark));
- // mg_hexdump(c->recv.buf, c->recv.len);
- }
-}
-
-static void deliver_normal_chunks(struct mg_connection *c, size_t hlen,
- struct mg_http_message *hm, bool *next) {
- size_t left, processed = ((size_t) c->pfn_data) & ~MG_DMARK;
- size_t deleted = ((size_t) c->pfn_data) & MG_DMARK;
- hm->chunk = mg_str_n((char *) &c->recv.buf[hlen], c->recv.len - hlen);
- if (processed <= hm->chunk.len && !deleted) {
- hm->chunk.len -= processed;
- hm->chunk.ptr += processed;
- }
- left = hm->body.len < processed ? 0 : hm->body.len - processed;
- if (hm->chunk.len > left) hm->chunk.len = left;
- if (hm->chunk.len > 0) mg_call(c, MG_EV_HTTP_CHUNK, hm);
- processed += hm->chunk.len;
- deleted = ((size_t) c->pfn_data) & MG_DMARK; // Re-evaluate after user call
- if (processed >= hm->body.len) { // Last, 0-len chunk
- hm->chunk.len = 0; // Reset length
- mg_call(c, MG_EV_HTTP_CHUNK, hm); // Call user handler
- c->pfn_data = NULL; // Reset processed counter
- if (processed && deleted) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
- } else {
- c->pfn_data = (void *) (processed | deleted); // if it is set
- }
-}
-
static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
struct mg_http_message hm;
- while (c->recv.buf != NULL && c->recv.len > 0) {
- bool next = false;
- int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
- if (hlen < 0) {
- mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
- break;
+ size_t ofs = 0; // Parsing offset
+
+ while (c->is_resp == 0 && ofs < c->recv.len) {
+ const char *buf = (char *) c->recv.buf + ofs;
+ int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
+ if (n < 0) {
+ mg_error(c, "HTTP parse");
+ return;
}
- if (c->is_resp) break; // Response is still generated
- if (hlen == 0) break; // Request is not buffered yet
- if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
- hm.message.len = c->recv.len; // and closes now, deliver a MSG
+ if (n == 0) break; // Request is not buffered yet
+ if (ev == MG_EV_CLOSE) { // If client did not set Content-Length
+ hm.message.len = c->recv.len - ofs; // and closes now, deliver MSG
hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
}
- if (mg_is_chunked(&hm)) {
- deliver_chunked_chunks(c, (size_t) hlen, &hm, &next);
- } else {
- deliver_normal_chunks(c, (size_t) hlen, &hm, &next);
+
+ if (is_chunked(&hm)) {
+ // For chunked data, strip off prefixes and suffixes from chunks
+ // and relocate them right after the headers, then report a message
+ char *s = (char *) c->recv.buf + ofs + n;
+ int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);
+
+ // Find zero-length chunk (the end of the body)
+ while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
+ if (cl == 0) break; // No zero-len chunk, buffer more data
+ if (cl < 0) {
+ mg_error(c, "Invalid chunk");
+ break;
+ }
+
+ // Zero chunk found. Second pass: strip + relocate
+ o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
+ while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
+ memmove(s + hm.body.len, s + o + pl, (size_t) dl);
+ o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
+ if (dl == 0) break;
+ }
+ ofs += (size_t) (n + o);
+ } else { // Normal, non-chunked data
+ size_t len = c->recv.len - ofs - (size_t) n;
+ if (hm.body.len > len) break; // Buffer more data
+ ofs += (size_t) n + hm.body.len;
}
- if (next) continue; // Chunks & request were deleted
- // Chunk events are delivered. If we have full body, deliver MSG
- if (c->recv.len < hm.message.len) break;
+
if (c->is_accepted) c->is_resp = 1; // Start generating response
mg_call(c, MG_EV_HTTP_MSG, &hm); // User handler can clear is_resp
- mg_iobuf_del(&c->recv, 0, hm.message.len);
}
+ if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs); // Delete processed data
}
(void) evd, (void) fnd;
}
diff --git a/src/http.h b/src/http.h
index 619f36be..0e810bb8 100644
--- a/src/http.h
+++ b/src/http.h
@@ -16,7 +16,6 @@ struct mg_http_message {
struct mg_http_header headers[MG_MAX_HTTP_HEADERS]; // Headers
struct mg_str body; // Body
struct mg_str head; // Request + headers
- struct mg_str chunk; // Chunk for chunked encoding, or partial body
struct mg_str message; // Request + headers + body
};
diff --git a/test/fuzz.c b/test/fuzz.c
index 1c5dec32..e68ae79a 100644
--- a/test/fuzz.c
+++ b/test/fuzz.c
@@ -2,7 +2,7 @@
#define MG_ENABLE_LOG 0
#define MG_ENABLE_LINES 1
#define MG_ENABLE_TCPIP 1
-#define MG_IO_SIZE (4 * 1024 * 1024) // 4M IO size, unless resize is too slow
+#define MG_IO_SIZE (32 * 1024 * 1024) // Big IO size for fast resizes
#include "mongoose.c"
diff --git a/test/unit_test.c b/test/unit_test.c
index 38181d57..f7330745 100644
--- a/test/unit_test.c
+++ b/test/unit_test.c
@@ -1342,8 +1342,6 @@ static void f4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
"abcdef");
strcat((char *) fn_data, "m");
c->is_draining = 1;
- } else if (ev == MG_EV_HTTP_CHUNK) {
- strcat((char *) fn_data, "f");
} else if (ev == MG_EV_CLOSE) {
strcat((char *) fn_data, "c");
}
@@ -1356,10 +1354,6 @@ static void f4c(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
ASSERT(mg_strcmp(hm->body, mg_str("/foo/bar/abcdef")) == 0);
strcat((char *) fn_data, "m");
- } else if (ev == MG_EV_HTTP_CHUNK) {
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- MG_INFO(("FS [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
- strcat((char *) fn_data, "f");
} else if (ev == MG_EV_CLOSE) {
strcat((char *) fn_data, "c");
}
@@ -1375,8 +1369,8 @@ static void test_http_no_content_length(void) {
mg_http_connect(&mgr, url, f4c, (void *) buf2);
for (i = 0; i < 1000 && strchr(buf2, 'c') == NULL; i++) mg_mgr_poll(&mgr, 10);
MG_INFO(("[%s] [%s]", buf1, buf2));
- ASSERT(strcmp(buf1, "fmc") == 0);
- ASSERT(strcmp(buf2, "fcfm") == 0); // See #1475
+ ASSERT(strcmp(buf1, "mc") == 0);
+ ASSERT(strcmp(buf2, "cm") == 0); // See #1475
mg_mgr_free(&mgr);
ASSERT(mgr.conns == NULL);
}
@@ -2290,20 +2284,8 @@ static void test_crc32(void) {
}
static void us(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
- int del = *(int *) fn_data;
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
- MG_DEBUG(("Got chunk len %lu", (unsigned long) hm->chunk.len));
- MG_DEBUG(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
- // MG_DEBUG(("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
- if (del) {
- mg_http_delete_chunk(c, hm);
- if (hm->chunk.len == 0) {
- MG_DEBUG(("Last chunk received, sending response"));
- mg_http_reply(c, 200, "", "ok (chunked)\n");
- }
- }
- } else if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
+ if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
MG_DEBUG(("Got all %lu bytes!", (unsigned long) hm->body.len));
MG_DEBUG(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
// MG_DEBUG(("Body:\n%.*s", (int) hm->body.len, hm->body.ptr));
@@ -2318,7 +2300,7 @@ static void us(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
const char **s = (const char **) fn_data;
if (ev == MG_EV_OPEN) {
- c->is_hexdumping = 1;
+ // c->is_hexdumping = 1;
} else if (ev == MG_EV_CONNECT) {
mg_printf(c,
"POST /upload HTTP/1.0\r\n"
@@ -2328,6 +2310,7 @@ static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
mg_http_printf_chunk(c, "");
} else if (ev == MG_EV_HTTP_MSG) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+ // MG_INFO(("---> [%s] [%.*s]", *s, hm->body.len, hm->body.ptr));
ASSERT(mg_strcmp(hm->body, mg_str(*s)) == 0);
*s = NULL;
}
@@ -2337,22 +2320,14 @@ static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
static void test_http_upload(void) {
struct mg_mgr mgr;
const char *url = "http://127.0.0.1:12352";
- int i, del = 1;
- const char *s1 = "ok (chunked)\n";
- const char *s2 = "ok (8 foo\nbar\n)\n";
+ int i;
+ const char *s = "ok (8 foo\nbar\n)\n";
mg_mgr_init(&mgr);
- mg_http_listen(&mgr, url, us, (void *) &del);
-
- mg_http_connect(&mgr, url, uc, (void *) &s1);
+ mg_http_listen(&mgr, url, us, NULL);
+ mg_http_connect(&mgr, url, uc, (void *) &s);
for (i = 0; i < 20; i++) mg_mgr_poll(&mgr, 5);
- ASSERT(s1 == NULL);
-
- del = 0;
- mg_http_connect(&mgr, url, uc, (void *) &s2);
- for (i = 0; i < 20; i++) mg_mgr_poll(&mgr, 5);
- ASSERT(s2 == NULL);
-
+ ASSERT(s == NULL);
mg_mgr_free(&mgr);
ASSERT(mgr.conns == NULL);
}
@@ -2363,7 +2338,7 @@ static void eX(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
if (ev == MG_EV_HTTP_MSG) {
mg_printf(c, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n");
c->data[0] = 1;
- c->is_hexdumping = 1;
+ // c->is_hexdumping = 1;
} else if (ev == MG_EV_POLL && c->data[0] != 0) {
c->data[0]++;
if (c->data[0] == 10) mg_http_printf_chunk(c, "a");
@@ -2403,32 +2378,13 @@ static void eZ(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
// Do not delete chunks as they arrive
static void eh4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
uint32_t *crc = (uint32_t *) fn_data;
- if (ev == MG_EV_HTTP_CHUNK) {
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- *crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
- *crc = mg_crc32(*crc, "x", 1);
- MG_INFO(("%lu C [%.*s]", c->id, (int) hm->chunk.len, hm->chunk.ptr));
- } else if (ev == MG_EV_HTTP_MSG) {
+ if (ev == MG_EV_HTTP_MSG) {
struct mg_http_message *hm = (struct mg_http_message *) ev_data;
*crc = mg_crc32(*crc, hm->body.ptr, hm->body.len);
MG_INFO(("%lu M [%.*s]", c->id, (int) hm->body.len, hm->body.ptr));
}
}
-// Streaming client event handler. Delete chunks as they arrive
-static void eh5(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
- uint32_t *crc = (uint32_t *) fn_data;
- if (ev == MG_EV_HTTP_CHUNK) {
- struct mg_http_message *hm = (struct mg_http_message *) ev_data;
- *crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
- *crc = mg_crc32(*crc, "x", 1);
- MG_INFO(("%lu DELC [%.*s]", c->id, (int) hm->chunk.len, hm->chunk.ptr));
- mg_http_delete_chunk(c, hm);
- } else if (ev == MG_EV_HTTP_MSG) {
- ASSERT(0); // Must not be here, MSG must not be fired: chunks deleted!
- }
-}
-
static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
int req_count, const char *expected) {
char url[100];
@@ -2446,6 +2402,7 @@ static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
for (i = 0; i < 100 && crc != expected_crc; i++) {
mg_mgr_poll(&mgr, 1);
}
+ // MG_INFO(("-------- %d [%s]", i, expected));
ASSERT(i < 100);
ASSERT(crc == expected_crc);
mg_mgr_free(&mgr);
@@ -2454,20 +2411,14 @@ static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
static void test_http_chunked(void) {
// Non-chunked encoding
- test_http_chunked_case(eY, eh4, 1, "axbcxdxxabcd"); // Chunks not deleted
- test_http_chunked_case(eY, eh4, 2, "axbcxdxxabcdaxbcxdxxabcd");
- test_http_chunked_case(eY, eh5, 1, "axbcxdxx"); // Chunks deleted
- test_http_chunked_case(eY, eh5, 2, "axbcxdxxaxbcxdxx");
- test_http_chunked_case(eZ, eh4, 1, "abcdxxabcd"); // Not deleted
- test_http_chunked_case(eZ, eh4, 2, "abcdxxabcdabcdxxabcd");
- test_http_chunked_case(eZ, eh5, 1, "abcdxx"); // Deleted
- test_http_chunked_case(eZ, eh5, 2, "abcdxxabcdxx");
+ test_http_chunked_case(eY, eh4, 1, "abcd"); // Chunks not deleted
+ test_http_chunked_case(eY, eh4, 2, "abcdabcd");
+ test_http_chunked_case(eZ, eh4, 1, "abcd"); // Not deleted
+ test_http_chunked_case(eZ, eh4, 2, "abcdabcd");
// Chunked encoding
- test_http_chunked_case(eX, eh4, 1, "axbxcxdxxabcd"); // Chunks not deleted
- test_http_chunked_case(eX, eh5, 1, "axbxcxdxx"); // Chunks deleted
- test_http_chunked_case(eX, eh4, 2, "axbxcxdxxabcdaxbxcxdxxabcd");
- test_http_chunked_case(eX, eh5, 2, "axbxcxdxxaxbxcxdxx");
+ test_http_chunked_case(eX, eh4, 1, "abcd"); // Chunks not deleted
+ test_http_chunked_case(eX, eh4, 2, "abcdabcd");
}
static void test_invalid_listen_addr(void) {