From 28f3c35c215ffbe0241685901338fad484660454 Mon Sep 17 00:00:00 2001 From: Fabrice Fontaine Date: Tue, 24 Sep 2019 19:44:29 +0200 Subject: [PATCH 01/16] Use -f option when calling ln at install time This allows "make install; make install" to work properly. Retrieved from: https://git.buildroot.net/buildroot/tree/package/libhttpparser/0001-Use-f-option-when-calling-ln-at-install-time.patch PR-URL: https://github.com/nodejs/http-parser/pull/492 Reviewed-By: Ben Noordhuis Signed-off-by: Fabrice Fontaine Signed-off-by: Renaud AUBIN Signed-off-by: Thomas Petazzoni --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e7f6d427..29100568 100644 --- a/Makefile +++ b/Makefile @@ -133,14 +133,14 @@ tags: http_parser.c http_parser.h test.c install: library $(INSTALL) -D http_parser.h $(DESTDIR)$(INCLUDEDIR)/http_parser.h $(INSTALL) -D $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(LIBNAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) install-strip: library $(INSTALL) -D http_parser.h $(DESTDIR)$(INCLUDEDIR)/http_parser.h $(INSTALL) -D -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(LIBNAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) uninstall: rm $(DESTDIR)$(INCLUDEDIR)/http_parser.h From 7d5c99d09f6743b055d53fc3f642746d9801479b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Mon, 3 Feb 2020 21:10:35 +0100 Subject: [PATCH 02/16] Support multi-coding Transfer-Encoding `Transfer-Encoding` header might have multiple codings in it. Even though llhttp cares only about `chunked`, it must check that `chunked` is the last coding (if present). ABNF from RFC 7230: ``` Transfer-Encoding = *( "," OWS ) transfer-coding *( OWS "," [ OWS transfer-coding ] ) transfer-coding = "chunked" / "compress" / "deflate" / "gzip" / transfer-extension transfer-extension = token *( OWS ";" OWS transfer-parameter ) transfer-parameter = token BWS "=" BWS ( token / quoted-string ) ``` However, if `chunked` is not last - llhttp must assume that the encoding and size of the body is unknown (according to 3.3.3 of RFC 7230) and read the response until EOF. For request - the error must be raised for an unknown `Transfer-Encoding`. Furthermore, 3.3.3 of RFC 7230 explicitly states that presence of both `Transfer-Encoding` and `Content-Length` indicates the smuggling attack and "ought to be handled as an error". For the lenient mode: * Unknown `Transfer-Encoding` in requests is not an error and request body is simply read until EOF (end of connection) * Only `Transfer-Encoding: chunked` together with `Content-Length` would result an error (just like before the patch) PR-URL: https://github.com/nodejs-private/http-parser-private/pull/4 Reviewed-By: Matteo Collina Reviewed-By: Sam Roberts Reviewed-By: James M Snell --- http_parser.c | 85 ++++++++++++++++++++++++++++++++++++++++++------ http_parser.h | 5 ++- test.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 165 insertions(+), 14 deletions(-) diff --git a/http_parser.c b/http_parser.c index 48963853..0f76b6ac 100644 --- a/http_parser.c +++ b/http_parser.c @@ -381,7 +381,10 @@ enum header_states , h_transfer_encoding , h_upgrade + , h_matching_transfer_encoding_token_start , h_matching_transfer_encoding_chunked + , h_matching_transfer_encoding_token + , h_matching_connection_token_start , h_matching_connection_keep_alive , h_matching_connection_close @@ -1335,6 +1338,7 @@ size_t http_parser_execute (http_parser *parser, parser->header_state = h_general; } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { parser->header_state = h_transfer_encoding; + parser->flags |= F_TRANSFER_ENCODING; } break; @@ -1416,10 +1420,14 @@ size_t http_parser_execute (http_parser *parser, if ('c' == c) { parser->header_state = h_matching_transfer_encoding_chunked; } else { - parser->header_state = h_general; + parser->header_state = h_matching_transfer_encoding_token; } break; + /* Multi-value `Transfer-Encoding` header */ + case h_matching_transfer_encoding_token_start: + break; + case h_content_length: if (UNLIKELY(!IS_NUM(ch))) { SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); @@ -1563,16 +1571,41 @@ size_t http_parser_execute (http_parser *parser, goto error; /* Transfer-Encoding: chunked */ + case h_matching_transfer_encoding_token_start: + /* looking for 'Transfer-Encoding: chunked' */ + if ('c' == c) { + h_state = h_matching_transfer_encoding_chunked; + } else if (STRICT_TOKEN(c)) { + /* TODO(indutny): similar code below does this, but why? + * At the very least it seems to be inconsistent given that + * h_matching_transfer_encoding_token does not check for + * `STRICT_TOKEN` + */ + h_state = h_matching_transfer_encoding_token; + } else if (c == ' ' || c == '\t') { + /* Skip lws */ + } else { + h_state = h_general; + } + break; + case h_matching_transfer_encoding_chunked: parser->index++; if (parser->index > sizeof(CHUNKED)-1 || c != CHUNKED[parser->index]) { - h_state = h_general; + h_state = h_matching_transfer_encoding_token; } else if (parser->index == sizeof(CHUNKED)-2) { h_state = h_transfer_encoding_chunked; } break; + case h_matching_transfer_encoding_token: + if (ch == ',') { + h_state = h_matching_transfer_encoding_token_start; + parser->index = 0; + } + break; + case h_matching_connection_token_start: /* looking for 'Connection: keep-alive' */ if (c == 'k') { @@ -1631,7 +1664,7 @@ size_t http_parser_execute (http_parser *parser, break; case h_transfer_encoding_chunked: - if (ch != ' ') h_state = h_general; + if (ch != ' ') h_state = h_matching_transfer_encoding_token; break; case h_connection_keep_alive: @@ -1765,12 +1798,17 @@ size_t http_parser_execute (http_parser *parser, REEXECUTE(); } - /* Cannot use chunked encoding and a content-length header together - per the HTTP specification. */ - if ((parser->flags & F_CHUNKED) && + /* Cannot us transfer-encoding and a content-length header together + per the HTTP specification. (RFC 7230 Section 3.3.3) */ + if ((parser->flags & F_TRANSFER_ENCODING) && (parser->flags & F_CONTENTLENGTH)) { - SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); - goto error; + /* Allow it for lenient parsing as long as `Transfer-Encoding` is + * not `chunked` + */ + if (!lenient || (parser->flags & F_CHUNKED)) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } } UPDATE_STATE(s_headers_done); @@ -1845,8 +1883,31 @@ size_t http_parser_execute (http_parser *parser, UPDATE_STATE(NEW_MESSAGE()); CALLBACK_NOTIFY(message_complete); } else if (parser->flags & F_CHUNKED) { - /* chunked encoding - ignore Content-Length header */ + /* chunked encoding - ignore Content-Length header, + * prepare for a chunk */ UPDATE_STATE(s_chunk_size_start); + } else if (parser->flags & F_TRANSFER_ENCODING) { + if (parser->type == HTTP_REQUEST && !lenient) { + /* RFC 7230 3.3.3 */ + + /* If a Transfer-Encoding header field + * is present in a request and the chunked transfer coding is not + * the final encoding, the message body length cannot be determined + * reliably; the server MUST respond with the 400 (Bad Request) + * status code and then close the connection. + */ + SET_ERRNO(HPE_INVALID_TRANSFER_ENCODING); + RETURN(p - data); /* Error */ + } else { + /* RFC 7230 3.3.3 */ + + /* If a Transfer-Encoding header field is present in a response and + * the chunked transfer coding is not the final encoding, the + * message body length is determined by reading the connection until + * it is closed by the server. + */ + UPDATE_STATE(s_body_identity_eof); + } } else { if (parser->content_length == 0) { /* Content-Length header given but zero: Content-Length: 0\r\n */ @@ -2100,6 +2161,12 @@ http_message_needs_eof (const http_parser *parser) return 0; } + /* RFC 7230 3.3.3, see `s_headers_almost_done` */ + if ((parser->flags & F_TRANSFER_ENCODING) && + (parser->flags & F_CHUNKED) == 0) { + return 1; + } + if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) { return 0; } diff --git a/http_parser.h b/http_parser.h index 16b5281d..4f840c23 100644 --- a/http_parser.h +++ b/http_parser.h @@ -225,6 +225,7 @@ enum flags , F_UPGRADE = 1 << 5 , F_SKIPBODY = 1 << 6 , F_CONTENTLENGTH = 1 << 7 + , F_TRANSFER_ENCODING = 1 << 8 }; @@ -271,6 +272,8 @@ enum flags "unexpected content-length header") \ XX(INVALID_CHUNK_SIZE, \ "invalid character in chunk size header") \ + XX(INVALID_TRANSFER_ENCODING, \ + "request has invalid transfer-encoding") \ XX(INVALID_CONSTANT, "invalid constant string") \ XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\ XX(STRICT, "strict mode assertion failed") \ @@ -293,11 +296,11 @@ enum http_errno { struct http_parser { /** PRIVATE **/ unsigned int type : 2; /* enum http_parser_type */ - unsigned int flags : 8; /* F_* values from 'flags' enum; semi-public */ unsigned int state : 7; /* enum state from http_parser.c */ unsigned int header_state : 7; /* enum header_state from http_parser.c */ unsigned int index : 7; /* index into current matcher */ unsigned int lenient_http_headers : 1; + unsigned int flags : 16; /* F_* values from 'flags' enum; semi-public */ uint32_t nread; /* # bytes read in various scenarios */ uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */ diff --git a/test.c b/test.c index 0140a18b..c979467c 100644 --- a/test.c +++ b/test.c @@ -262,7 +262,6 @@ const struct message requests[] = ,.type= HTTP_REQUEST ,.raw= "POST /post_identity_body_world?q=search#hey HTTP/1.1\r\n" "Accept: */*\r\n" - "Transfer-Encoding: identity\r\n" "Content-Length: 5\r\n" "\r\n" "World" @@ -275,10 +274,9 @@ const struct message requests[] = ,.fragment= "hey" ,.request_path= "/post_identity_body_world" ,.request_url= "/post_identity_body_world?q=search#hey" - ,.num_headers= 3 + ,.num_headers= 2 ,.headers= { { "Accept", "*/*" } - , { "Transfer-Encoding", "identity" } , { "Content-Length", "5" } } ,.body= "World" @@ -1193,6 +1191,61 @@ const struct message requests[] = ,.headers= { { "Host", "example.com" } } ,.body= "" } + +#define POST_MULTI_TE_LAST_CHUNKED 43 +, {.name= "post - multi coding transfer-encoding chunked body" + ,.type= HTTP_REQUEST + ,.raw= "POST / HTTP/1.1\r\n" + "Transfer-Encoding: deflate, chunked\r\n" + "\r\n" + "1e\r\nall your base are belong to us\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.query_string= "" + ,.fragment= "" + ,.request_path= "/" + ,.request_url= "/" + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding" , "deflate, chunked" } + } + ,.body= "all your base are belong to us" + ,.num_chunks_complete= 2 + ,.chunk_lengths= { 0x1e } + } + +#define POST_MULTI_LINE_TE_LAST_CHUNKED 43 +, {.name= "post - multi coding transfer-encoding chunked body" + ,.type= HTTP_REQUEST + ,.raw= "POST / HTTP/1.1\r\n" + "Transfer-Encoding: deflate,\r\n" + " chunked\r\n" + "\r\n" + "1e\r\nall your base are belong to us\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.query_string= "" + ,.fragment= "" + ,.request_path= "/" + ,.request_url= "/" + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding" , "deflate, chunked" } + } + ,.body= "all your base are belong to us" + ,.num_chunks_complete= 2 + ,.chunk_lengths= { 0x1e } + } }; /* * R E S P O N S E S * */ @@ -1970,6 +2023,28 @@ const struct message responses[] = ,.num_chunks_complete= 3 ,.chunk_lengths= { 2, 2 } } +#define HTTP_200_MULTI_TE_NOT_LAST_CHUNKED 28 +, {.name= "HTTP 200 response with `chunked` being *not last* Transfer-Encoding" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked, identity\r\n" + "\r\n" + "2\r\n" + "OK\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= TRUE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_status= "OK" + ,.num_headers= 1 + ,.headers= { { "Transfer-Encoding", "chunked, identity" } + } + ,.body= "2\r\nOK\r\n0\r\n\r\n" + ,.num_chunks_complete= 0 + } }; /* strnlen() is a POSIX.2008 addition. Can't rely on it being available so @@ -3663,7 +3738,7 @@ test_chunked_content_length_error (int req) parsed = http_parser_execute(&parser, &settings_null, buf, strlen(buf)); assert(parsed == strlen(buf)); - buf = "Transfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n"; + buf = "Transfer-Encoding: anything\r\nContent-Length: 1\r\n\r\n"; size_t buflen = strlen(buf); parsed = http_parser_execute(&parser, &settings_null, buf, buflen); @@ -4332,6 +4407,12 @@ main (void) "fooba", HPE_OK); + // Unknown Transfer-Encoding in request + test_simple("GET / HTTP/1.1\r\n" + "Transfer-Encoding: unknown\r\n" + "\r\n", + HPE_INVALID_TRANSFER_ENCODING); + static const char *all_methods[] = { "DELETE", "GET", From a0c034c0c7698c08f8dc8c8d0257305f6280c27b Mon Sep 17 00:00:00 2001 From: Fedor Indutny Date: Thu, 6 Feb 2020 10:04:53 +0100 Subject: [PATCH 03/16] v2.9.3 --- Makefile | 2 +- http_parser.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 29100568..9e8c1f03 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ BINEXT ?= SOLIBNAME = libhttp_parser SOMAJOR = 2 SOMINOR = 9 -SOREV = 2 +SOREV = 3 ifeq (darwin,$(PLATFORM)) SOEXT ?= dylib SONAME ?= $(SOLIBNAME).$(SOMAJOR).$(SOMINOR).$(SOEXT) diff --git a/http_parser.h b/http_parser.h index 4f840c23..983d88a9 100644 --- a/http_parser.h +++ b/http_parser.h @@ -27,7 +27,7 @@ extern "C" { /* Also update SONAME in the Makefile whenever you change these. */ #define HTTP_PARSER_VERSION_MAJOR 2 #define HTTP_PARSER_VERSION_MINOR 9 -#define HTTP_PARSER_VERSION_PATCH 2 +#define HTTP_PARSER_VERSION_PATCH 3 #include #if defined(_WIN32) && !defined(__MINGW32__) && \ From 1c02cb97fa551f92ed0467033250fcda3554d14a Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 12 Feb 2020 16:31:12 -0800 Subject: [PATCH 04/16] Correct test name and numbering It doesn't matter yet, but two tests had the same name, and same test position macro. PR-URL: https://github.com/nodejs/http-parser/pull/497 Reviewed-By: Fedor Indutny --- test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.c b/test.c index c979467c..d0881797 100644 --- a/test.c +++ b/test.c @@ -1219,8 +1219,8 @@ const struct message requests[] = ,.chunk_lengths= { 0x1e } } -#define POST_MULTI_LINE_TE_LAST_CHUNKED 43 -, {.name= "post - multi coding transfer-encoding chunked body" +#define POST_MULTI_LINE_TE_LAST_CHUNKED 44 +, {.name= "post - multi line coding transfer-encoding chunked body" ,.type= HTTP_REQUEST ,.raw= "POST / HTTP/1.1\r\n" "Transfer-Encoding: deflate,\r\n" From 714cbb2dfbea9a521960b82b6239a991a64e3b9a Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 23 Mar 2020 12:44:24 +0100 Subject: [PATCH 05/16] Fix ABI breakage Fix ABI breakage introduced in commit 7d5c99d ("Support multi-coding Transfer-Encoding") by undoing the change in `sizeof(http_parser)`. Restore the size of the `flags` field and shrink the `index` field from 7 to 5 bits. It track strings up to `strlen("Transfer-Encoding")` bytes so 2^5 == 32 is wide enough. Fixes: https://github.com/nodejs/http-parser/issues/502 PR-URL: https://github.com/nodejs/http-parser/pull/503 Reviewed-By: Fedor Indutny Reviewed-By: Matteo Collina Reviewed-By: Sam Roberts --- http_parser.c | 11 +++++++---- http_parser.h | 13 +++++++------ test.c | 1 + 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/http_parser.c b/http_parser.c index 0f76b6ac..95ff42f7 100644 --- a/http_parser.c +++ b/http_parser.c @@ -731,6 +731,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; + parser->extra_flags = 0; parser->content_length = ULLONG_MAX; if (ch == 'H') { @@ -768,6 +769,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; + parser->extra_flags = 0; parser->content_length = ULLONG_MAX; if (ch == 'H') { @@ -925,6 +927,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; + parser->extra_flags = 0; parser->content_length = ULLONG_MAX; if (UNLIKELY(!IS_ALPHA(ch))) { @@ -1338,7 +1341,7 @@ size_t http_parser_execute (http_parser *parser, parser->header_state = h_general; } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { parser->header_state = h_transfer_encoding; - parser->flags |= F_TRANSFER_ENCODING; + parser->extra_flags |= F_TRANSFER_ENCODING >> 8; } break; @@ -1800,7 +1803,7 @@ size_t http_parser_execute (http_parser *parser, /* Cannot us transfer-encoding and a content-length header together per the HTTP specification. (RFC 7230 Section 3.3.3) */ - if ((parser->flags & F_TRANSFER_ENCODING) && + if ((parser->extra_flags & (F_TRANSFER_ENCODING >> 8)) && (parser->flags & F_CONTENTLENGTH)) { /* Allow it for lenient parsing as long as `Transfer-Encoding` is * not `chunked` @@ -1886,7 +1889,7 @@ size_t http_parser_execute (http_parser *parser, /* chunked encoding - ignore Content-Length header, * prepare for a chunk */ UPDATE_STATE(s_chunk_size_start); - } else if (parser->flags & F_TRANSFER_ENCODING) { + } else if (parser->extra_flags & (F_TRANSFER_ENCODING >> 8)) { if (parser->type == HTTP_REQUEST && !lenient) { /* RFC 7230 3.3.3 */ @@ -2162,7 +2165,7 @@ http_message_needs_eof (const http_parser *parser) } /* RFC 7230 3.3.3, see `s_headers_almost_done` */ - if ((parser->flags & F_TRANSFER_ENCODING) && + if ((parser->extra_flags & (F_TRANSFER_ENCODING >> 8)) && (parser->flags & F_CHUNKED) == 0) { return 1; } diff --git a/http_parser.h b/http_parser.h index 983d88a9..d36c80d4 100644 --- a/http_parser.h +++ b/http_parser.h @@ -225,7 +225,7 @@ enum flags , F_UPGRADE = 1 << 5 , F_SKIPBODY = 1 << 6 , F_CONTENTLENGTH = 1 << 7 - , F_TRANSFER_ENCODING = 1 << 8 + , F_TRANSFER_ENCODING = 1 << 8 /* Never set in http_parser.flags */ }; @@ -272,13 +272,13 @@ enum flags "unexpected content-length header") \ XX(INVALID_CHUNK_SIZE, \ "invalid character in chunk size header") \ - XX(INVALID_TRANSFER_ENCODING, \ - "request has invalid transfer-encoding") \ XX(INVALID_CONSTANT, "invalid constant string") \ XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\ XX(STRICT, "strict mode assertion failed") \ XX(PAUSED, "parser is paused") \ - XX(UNKNOWN, "an unknown error occurred") + XX(UNKNOWN, "an unknown error occurred") \ + XX(INVALID_TRANSFER_ENCODING, \ + "request has invalid transfer-encoding") \ /* Define HPE_* values for each errno value above */ @@ -296,11 +296,12 @@ enum http_errno { struct http_parser { /** PRIVATE **/ unsigned int type : 2; /* enum http_parser_type */ + unsigned int flags : 8; /* F_* values from 'flags' enum; semi-public */ unsigned int state : 7; /* enum state from http_parser.c */ unsigned int header_state : 7; /* enum header_state from http_parser.c */ - unsigned int index : 7; /* index into current matcher */ + unsigned int index : 5; /* index into current matcher */ + unsigned int extra_flags : 2; unsigned int lenient_http_headers : 1; - unsigned int flags : 16; /* F_* values from 'flags' enum; semi-public */ uint32_t nread; /* # bytes read in various scenarios */ uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */ diff --git a/test.c b/test.c index d0881797..79834245 100644 --- a/test.c +++ b/test.c @@ -4221,6 +4221,7 @@ main (void) printf("http_parser v%u.%u.%u (0x%06lx)\n", major, minor, patch, version); printf("sizeof(http_parser) = %u\n", (unsigned int)sizeof(http_parser)); + assert(sizeof(http_parser) == 4 + 4 + 8 + 2 + 2 + 4 + sizeof(void *)); //// API test_preserve_data(); From 2343fd6b5214b2ded2cdcf76de2bf60903bb90cd Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Tue, 24 Mar 2020 11:52:16 +0100 Subject: [PATCH 06/16] v2.9.4 --- Makefile | 2 +- http_parser.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9e8c1f03..5d212215 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ BINEXT ?= SOLIBNAME = libhttp_parser SOMAJOR = 2 SOMINOR = 9 -SOREV = 3 +SOREV = 4 ifeq (darwin,$(PLATFORM)) SOEXT ?= dylib SONAME ?= $(SOLIBNAME).$(SOMAJOR).$(SOMINOR).$(SOEXT) diff --git a/http_parser.h b/http_parser.h index d36c80d4..df882526 100644 --- a/http_parser.h +++ b/http_parser.h @@ -27,7 +27,7 @@ extern "C" { /* Also update SONAME in the Makefile whenever you change these. */ #define HTTP_PARSER_VERSION_MAJOR 2 #define HTTP_PARSER_VERSION_MINOR 9 -#define HTTP_PARSER_VERSION_PATCH 3 +#define HTTP_PARSER_VERSION_PATCH 4 #include #if defined(_WIN32) && !defined(__MINGW32__) && \ From 7af127d3ac497bb036d3ecb4fb24bb5ee7ff4b5b Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 14 Apr 2020 15:38:37 +0200 Subject: [PATCH 07/16] Add a first fuzzer for the parser. PR-URL: https://github.com/nodejs/http-parser/pull/506 Reviewed-By: Ben Noordhuis Reviewed-By: Fedor Indutny --- fuzzers/fuzz_parser.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 fuzzers/fuzz_parser.c diff --git a/fuzzers/fuzz_parser.c b/fuzzers/fuzz_parser.c new file mode 100644 index 00000000..1a8442c9 --- /dev/null +++ b/fuzzers/fuzz_parser.c @@ -0,0 +1,26 @@ +#include +#include +#include +#include "http_parser.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + static const http_parser_settings settings_null = { + .on_message_begin = 0 + , .on_header_field = 0 + ,.on_header_value = 0 + ,.on_url = 0 + ,.on_status = 0 + ,.on_body = 0 + ,.on_headers_complete = 0 + ,.on_message_complete = 0 + ,.on_chunk_header = 0 + ,.on_chunk_complete = 0 + }; + + http_parser parser; + http_parser_init(&parser, HTTP_BOTH); + http_parser_execute(&parser, &settings_null, (char*)data, size); + + return 0; +} From eefbf87c47afbb984f75660f672c53eb9d038276 Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 28 Apr 2020 14:43:55 +0100 Subject: [PATCH 08/16] Add another fuzzer for the urls. PR-URL: https://github.com/nodejs/http-parser/pull/508 Reviewed-By: Ben Noordhuis Reviewed-By: Fedor Indutny --- fuzzers/fuzz_url.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 fuzzers/fuzz_url.c diff --git a/fuzzers/fuzz_url.c b/fuzzers/fuzz_url.c new file mode 100644 index 00000000..eca11a2e --- /dev/null +++ b/fuzzers/fuzz_url.c @@ -0,0 +1,14 @@ +#include +#include +#include +#include "http_parser.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + struct http_parser_url u; + http_parser_url_init(&u); + http_parser_parse_url((char*)data, size, 0, &u); + http_parser_parse_url((char*)data, size, 1, &u); + + return 0; +} From 55e736cd336bf1d1ac8aeeb402d66475060321ad Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Mon, 4 May 2020 12:01:54 +0200 Subject: [PATCH 09/16] Solaris 9 doesn't have stdint.h, use inttypes.h PR-URL: https://github.com/nodejs/http-parser/pull/184 Reviewed-By: Ben Noordhuis --- http_parser.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http_parser.h b/http_parser.h index df882526..60dc85d3 100644 --- a/http_parser.h +++ b/http_parser.h @@ -41,6 +41,8 @@ typedef __int32 int32_t; typedef unsigned __int32 uint32_t; typedef __int64 int64_t; typedef unsigned __int64 uint64_t; +#elif (defined(__sun) || defined(__sun__)) && defined(__SunOS_5_9) +#include #else #include #endif From 805a0d162b1de81e589bde978407e6374cb3eb62 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 4 May 2020 02:11:29 -0700 Subject: [PATCH 10/16] Fix test numbers PR-URL: https://github.com/nodejs/http-parser/pull/511 Reviewed-By: Ben Noordhuis --- test.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test.c b/test.c index 79834245..0026a7f6 100644 --- a/test.c +++ b/test.c @@ -1172,7 +1172,7 @@ const struct message requests[] = ,.body= "" } -#define SOURCE_ICE_REQUEST 42 +#define SOURCE_ICE_REQUEST 43 , {.name = "source request" ,.type= HTTP_REQUEST ,.raw= "SOURCE /music/sweet/music ICE/1.0\r\n" @@ -1192,7 +1192,7 @@ const struct message requests[] = ,.body= "" } -#define POST_MULTI_TE_LAST_CHUNKED 43 +#define POST_MULTI_TE_LAST_CHUNKED 44 , {.name= "post - multi coding transfer-encoding chunked body" ,.type= HTTP_REQUEST ,.raw= "POST / HTTP/1.1\r\n" @@ -1219,7 +1219,7 @@ const struct message requests[] = ,.chunk_lengths= { 0x1e } } -#define POST_MULTI_LINE_TE_LAST_CHUNKED 44 +#define POST_MULTI_LINE_TE_LAST_CHUNKED 45 , {.name= "post - multi line coding transfer-encoding chunked body" ,.type= HTTP_REQUEST ,.raw= "POST / HTTP/1.1\r\n" @@ -1821,7 +1821,7 @@ const struct message responses[] = ,.chunk_lengths= { 1 } } -#define EMPTY_REASON_PHRASE_AFTER_SPACE 20 +#define EMPTY_REASON_PHRASE_AFTER_SPACE 21 , {.name= "empty reason phrase after space" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 \r\n" @@ -1837,7 +1837,7 @@ const struct message responses[] = ,.body= "" } -#define CONTENT_LENGTH_X 21 +#define CONTENT_LENGTH_X 22 , {.name= "Content-Length-X" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1863,7 +1863,7 @@ const struct message responses[] = ,.chunk_lengths= { 2 } } -#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER 22 +#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER 23 , {.name= "HTTP 101 response with Upgrade header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 101 Switching Protocols\r\n" @@ -1885,7 +1885,7 @@ const struct message responses[] = } } -#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 23 +#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 24 , {.name= "HTTP 101 response with Upgrade and Content-Length header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 101 Switching Protocols\r\n" @@ -1911,7 +1911,7 @@ const struct message responses[] = } } -#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 24 +#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 25 , {.name= "HTTP 101 response with Upgrade and Transfer-Encoding header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 101 Switching Protocols\r\n" @@ -1944,7 +1944,7 @@ const struct message responses[] = ,.chunk_lengths= { 2, 2 } } -#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER 25 +#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER 26 , {.name= "HTTP 200 response with Upgrade header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1967,7 +1967,7 @@ const struct message responses[] = } } -#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 26 +#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 27 , {.name= "HTTP 200 response with Upgrade and Content-Length header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1992,7 +1992,7 @@ const struct message responses[] = } } -#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 27 +#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 28 , {.name= "HTTP 200 response with Upgrade and Transfer-Encoding header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -2023,7 +2023,7 @@ const struct message responses[] = ,.num_chunks_complete= 3 ,.chunk_lengths= { 2, 2 } } -#define HTTP_200_MULTI_TE_NOT_LAST_CHUNKED 28 +#define HTTP_200_MULTI_TE_NOT_LAST_CHUNKED 29 , {.name= "HTTP 200 response with `chunked` being *not last* Transfer-Encoding" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" From 5c5b3ac62662736de9e71640a8dc16da45b32503 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 11 May 2020 14:47:09 +0200 Subject: [PATCH 11/16] Update http_parser.content_length doc comment. It's -1 when no Content-Length field is present, not 0. Fixes: https://github.com/nodejs/http-parser/issues/512 PR-URL: https://github.com/nodejs/http-parser/pull/513 Reviewed-By: Fedor Indutny --- http_parser.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/http_parser.h b/http_parser.h index 60dc85d3..cfbffdcd 100644 --- a/http_parser.h +++ b/http_parser.h @@ -306,7 +306,9 @@ struct http_parser { unsigned int lenient_http_headers : 1; uint32_t nread; /* # bytes read in various scenarios */ - uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */ + uint64_t content_length; /* # bytes in body. `(uint64_t) -1` (all bits one) + * if no Content-Length header. + */ /** READ-ONLY **/ unsigned short http_major; From d9275da4650fd1133ddc96480df32a9efe4b059b Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Fri, 15 May 2020 13:29:49 +0200 Subject: [PATCH 12/16] Fix -Wsign-compare warning. The operands to `+` are promoted from `uint16_t` to `int` before addition, making the expression `off + len <= buflen` emit a warning because `buflen` is unsigned. Fixes: https://github.com/nodejs/http-parser/issues/514 PR-URL: https://github.com/nodejs/http-parser/pull/515 Reviewed-By: Fedor Indutny --- http_parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http_parser.c b/http_parser.c index 95ff42f7..f66192da 100644 --- a/http_parser.c +++ b/http_parser.c @@ -2517,7 +2517,7 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, end = buf + off + len; /* NOTE: The characters are already validated and are in the [0-9] range */ - assert(off + len <= buflen && "Port number overflow"); + assert((size_t) (off + len) <= buflen && "Port number overflow"); v = 0; for (p = buf + off; p < end; p++) { v *= 10; From 4b99e42883bbc5a54d7b67e2c2e72c351edd4fe6 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Wed, 8 Jul 2020 02:30:29 +0200 Subject: [PATCH 13/16] Test Content-Length header parsing. The test suite did very little validation of the Content-Length field until now. Verify for each request and response that the parsed numeric value matches the value from the header field. PR-URL: https://github.com/nodejs/http-parser/pull/519 Reviewed-By: Fedor Indutny --- test.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/test.c b/test.c index 0026a7f6..53b19997 100644 --- a/test.c +++ b/test.c @@ -74,6 +74,7 @@ struct message { unsigned short http_major; unsigned short http_minor; + uint64_t content_length; int message_begin_cb_called; int headers_complete_cb_called; @@ -108,6 +109,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "User-Agent", "curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1" } @@ -139,6 +141,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/favicon.ico" ,.request_url= "/favicon.ico" + ,.content_length= -1 ,.num_headers= 8 ,.headers= { { "Host", "0.0.0.0=5000" } @@ -168,6 +171,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/dumbluck" ,.request_url= "/dumbluck" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "aaaaaaaaaaaaa", "++++++++++" } @@ -190,6 +194,7 @@ const struct message requests[] = ,.request_path= "/forums/1/topics/2375" /* XXX request url does include fragment? */ ,.request_url= "/forums/1/topics/2375?page=1#posts-17408" + ,.content_length= -1 ,.num_headers= 0 ,.body= "" } @@ -208,6 +213,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/get_no_headers_no_body/world" ,.request_url= "/get_no_headers_no_body/world" + ,.content_length= -1 ,.num_headers= 0 ,.body= "" } @@ -227,6 +233,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/get_one_header_no_body" ,.request_url= "/get_one_header_no_body" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Accept" , "*/*" } @@ -250,6 +257,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/get_funky_content_length_body_hello" ,.request_url= "/get_funky_content_length_body_hello" + ,.content_length= 5 ,.num_headers= 1 ,.headers= { { "conTENT-Length" , "5" } @@ -274,6 +282,7 @@ const struct message requests[] = ,.fragment= "hey" ,.request_path= "/post_identity_body_world" ,.request_url= "/post_identity_body_world?q=search#hey" + ,.content_length= 5 ,.num_headers= 2 ,.headers= { { "Accept", "*/*" } @@ -300,6 +309,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/post_chunked_all_your_base" ,.request_url= "/post_chunked_all_your_base" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding" , "chunked" } @@ -328,6 +338,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/two_chunks_mult_zero_end" ,.request_url= "/two_chunks_mult_zero_end" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -358,6 +369,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/chunked_w_trailing_headers" ,.request_url= "/chunked_w_trailing_headers" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -388,6 +400,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/chunked_w_nonsense_after_length" ,.request_url= "/chunked_w_nonsense_after_length" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -410,6 +423,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/with_\"stupid\"_quotes" ,.request_url= "/with_\"stupid\"_quotes?foo=\"bar\"" + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -436,6 +450,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Host", "0.0.0.0:5000" } , { "User-Agent", "ApacheBench/2.3" } @@ -459,6 +474,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test.cgi" ,.request_url= "/test.cgi?foo=bar?baz" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -480,6 +496,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -507,6 +524,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 7 ,.upgrade="Hot diggity dogg" ,.headers= { { "Host", "example.com" } @@ -538,6 +556,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "" ,.request_url= "0-home0.netscape.com:443" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="some data\r\nand yet even more data" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -560,6 +579,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -579,6 +599,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -601,6 +622,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "*" ,.request_url= "*" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "HOST", "239.255.255.250:1900" } , { "MAN", "\"ssdp:discover\"" } @@ -636,6 +658,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 5 ,.headers= { { "Line1", "abc\tdef ghi\t\tjkl mno \t \tqrs" } , { "Line2", "line2\t" } @@ -662,6 +685,7 @@ const struct message requests[] = ,.request_path= "" ,.request_url= "http://hypnotoad.org?hail=all" ,.host= "hypnotoad.org" + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -683,6 +707,7 @@ const struct message requests[] = ,.request_url= "http://hypnotoad.org:1234?hail=all" ,.host= "hypnotoad.org" ,.port= 1234 + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -704,6 +729,7 @@ const struct message requests[] = ,.request_url= "http://hypnotoad.org:1234" ,.host= "hypnotoad.org" ,.port= 1234 + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -728,6 +754,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/file.txt" ,.request_url= "/file.txt" + ,.content_length= 10 ,.num_headers= 4 ,.headers= { { "Host", "www.example.com" } , { "Content-Type", "application/example" } @@ -753,6 +780,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "" ,.request_url= "HOME0.NETSCAPE.COM:443" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -777,6 +805,7 @@ const struct message requests[] = ,.fragment= "narf" ,.request_path= "/δ¶/δt/pope" ,.request_url= "/δ¶/δt/pope?q=1#narf" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { {"Host", "github.com" } } @@ -799,6 +828,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "" ,.request_url= "home_0.netscape.com:443" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -826,6 +856,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= 4 ,.num_headers= 3 ,.upgrade= 0 ,.headers= { { "Host", "www.example.com" } @@ -854,6 +885,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= 4 ,.num_headers= 4 ,.upgrade= 0 ,.headers= { { "Host", "www.example.com" } @@ -879,6 +911,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/file.txt" ,.request_url= "/file.txt" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "www.example.com" } } ,.body= "" @@ -899,6 +932,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "www.example.com" } } ,.body= "" @@ -920,6 +954,7 @@ const struct message requests[] = ,.host= "hypnotoad.org" ,.userinfo= "a%12:b!&*$" ,.port= 1234 + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -952,6 +987,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 5 ,.headers= { { "Line1", "abc\tdef ghi\t\tjkl mno \t \tqrs" } , { "Line2", "line2\t" } @@ -985,6 +1021,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 7 ,.upgrade="Hot diggity dogg" ,.headers= { { "Host", "example.com" } @@ -1015,6 +1052,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="Hot diggity dogg" ,.headers= { { "Connection", "keep-alive, upgrade" } @@ -1040,6 +1078,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="Hot diggity dogg" ,.headers= { { "Connection", "keep-alive, upgrade" } @@ -1066,6 +1105,7 @@ const struct message requests[] = ,.method= HTTP_POST ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= 15 ,.num_headers= 4 ,.upgrade="Hot diggity dogg" ,.headers= { { "Host", "example.com" } @@ -1091,6 +1131,7 @@ const struct message requests[] = ,.http_minor= 0 ,.method= HTTP_CONNECT ,.request_url= "foo.bar.com:443" + ,.content_length= 10 ,.num_headers= 3 ,.upgrade="blarfcicle" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -1121,6 +1162,7 @@ const struct message requests[] = ,.request_url= "/images/my_dog.jpg" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Host", "example.com" } , { "Link", "; rel=\"tag\"" } @@ -1145,6 +1187,7 @@ const struct message requests[] = ,.request_url= "/images/my_dog.jpg" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Host", "example.com" } , { "Link", "; rel=\"tag\"" } @@ -1167,6 +1210,7 @@ const struct message requests[] = ,.request_url= "/music/sweet/music" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "example.com" } } ,.body= "" @@ -1187,6 +1231,7 @@ const struct message requests[] = ,.request_url= "/music/sweet/music" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "example.com" } } ,.body= "" @@ -1210,6 +1255,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding" , "deflate, chunked" } @@ -1238,6 +1284,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding" , "deflate, chunked" } @@ -1275,6 +1322,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 301 ,.response_status= "Moved Permanently" + ,.content_length= 219 ,.num_headers= 8 ,.headers= { { "Location", "http://www.google.com/" } @@ -1324,6 +1372,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 5 ,.headers= { { "Date", "Tue, 04 Aug 2009 07:59:32 GMT" } @@ -1353,6 +1402,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 404 ,.response_status= "Not Found" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body_size= 0 @@ -1368,6 +1418,7 @@ const struct message responses[] = ,.http_major= 1 ,.http_minor= 1 ,.status_code= 301 + ,.content_length= -1 ,.response_status= "" ,.num_headers= 0 ,.headers= {} @@ -1395,6 +1446,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { {"Content-Type", "text/plain" } @@ -1422,6 +1474,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { {"Content-Type", "text/html; charset=utf-8" } @@ -1446,6 +1499,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 11 ,.num_headers= 4 ,.headers= { {"Content-Type", "text/html; charset=UTF-8" } @@ -1472,6 +1526,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 0 ,.num_headers= 4 ,.headers= { {"Server", "DCLK-AdSvr" } @@ -1505,6 +1560,7 @@ const struct message responses[] = ,.http_minor= 0 ,.status_code= 301 ,.response_status= "Moved Permanently" + ,.content_length= 0 ,.num_headers= 9 ,.headers= { { "Date", "Thu, 03 Jun 2010 09:56:32 GMT" } @@ -1544,6 +1600,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 11 ,.headers= { { "Date", "Tue, 28 Sep 2010 01:14:13 GMT" } @@ -1578,6 +1635,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 500 ,.response_status= "Oriëntatieprobleem" + ,.content_length= 0 ,.num_headers= 3 ,.headers= { { "Date", "Fri, 5 Nov 2010 23:07:12 GMT+2" } @@ -1599,6 +1657,7 @@ const struct message responses[] = ,.http_minor= 9 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} @@ -1622,6 +1681,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Content-Type", "text/plain" } @@ -1641,6 +1701,7 @@ const struct message responses[] = ,.http_minor= 0 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Connection", "keep-alive" } @@ -1661,6 +1722,7 @@ const struct message responses[] = ,.http_minor= 0 ,.status_code= 204 ,.response_status= "No content" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Connection", "keep-alive" } @@ -1680,6 +1742,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 0 ,.headers={} ,.body_size= 0 @@ -1697,6 +1760,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 204 ,.response_status= "No content" + ,.content_length= -1 ,.num_headers= 0 ,.headers={} ,.body_size= 0 @@ -1715,6 +1779,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 204 ,.response_status= "No content" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Connection", "close" } @@ -1737,6 +1802,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -1767,6 +1833,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 16 ,.num_headers= 7 ,.headers= { { "Server", "Microsoft-IIS/6.0" } @@ -1805,6 +1872,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 301 ,.response_status= "MovedPermanently" + ,.content_length= -1 ,.num_headers= 9 ,.headers= { { "Date", "Wed, 15 May 2013 17:06:33 GMT" } , { "Server", "Server" } @@ -1832,6 +1900,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -1854,6 +1923,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Content-Length-X", "0" } , { "Transfer-Encoding", "chunked" } @@ -1878,6 +1948,7 @@ const struct message responses[] = ,.status_code= 101 ,.response_status= "Switching Protocols" ,.upgrade= "proto" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Connection", "upgrade" } @@ -1903,6 +1974,7 @@ const struct message responses[] = ,.response_status= "Switching Protocols" ,.body= "body" ,.upgrade= "proto" + ,.content_length= 4 ,.num_headers= 3 ,.headers= { { "Connection", "upgrade" } @@ -1934,6 +2006,7 @@ const struct message responses[] = ,.response_status= "Switching Protocols" ,.body= "body" ,.upgrade= "proto" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Connection", "upgrade" } @@ -1960,6 +2033,7 @@ const struct message responses[] = ,.response_status= "OK" ,.body= "body" ,.upgrade= NULL + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Connection", "upgrade" } @@ -1982,6 +2056,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 4 ,.num_headers= 3 ,.body= "body" ,.upgrade= NULL @@ -2012,6 +2087,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 3 ,.body= "body" ,.upgrade= NULL @@ -2039,6 +2115,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked, identity" } } @@ -2202,6 +2279,7 @@ headers_complete_cb (http_parser *p) messages[num_messages].status_code = parser.status_code; messages[num_messages].http_major = parser.http_major; messages[num_messages].http_minor = parser.http_minor; + messages[num_messages].content_length = parser.content_length; messages[num_messages].headers_complete_cb_called = TRUE; messages[num_messages].should_keep_alive = http_should_keep_alive(&parser); return 0; @@ -2650,6 +2728,7 @@ message_eq (int index, int connect, const struct message *expected) MESSAGE_CHECK_NUM_EQ(expected, m, http_major); MESSAGE_CHECK_NUM_EQ(expected, m, http_minor); + MESSAGE_CHECK_NUM_EQ(expected, m, content_length); if (expected->type == HTTP_REQUEST) { MESSAGE_CHECK_NUM_EQ(expected, m, method); @@ -4351,6 +4430,7 @@ main (void) ,.http_minor= 0 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Transfer-Encoding", "chunked" } From e13b274770da9b82a1085dec29182acfea72e7a7 Mon Sep 17 00:00:00 2001 From: Oleg Guba Date: Fri, 10 Jul 2020 11:49:58 +0200 Subject: [PATCH 14/16] Allow Content-Length and Transfer-Encoding: chunked Fixes: https://github.com/nodejs/http-parser/issues/517 PR-URL: https://github.com/nodejs/http-parser/pull/518 Reviewed-By: Ben Noordhuis Reviewed-By: Pierce Lopez --- http_parser.c | 27 +++++++++++++++++---------- http_parser.h | 6 ++++-- test.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/http_parser.c b/http_parser.c index f66192da..9be003e7 100644 --- a/http_parser.c +++ b/http_parser.c @@ -653,6 +653,8 @@ size_t http_parser_execute (http_parser *parser, const char *status_mark = 0; enum state p_state = (enum state) parser->state; const unsigned int lenient = parser->lenient_http_headers; + const unsigned int allow_chunked_length = parser->allow_chunked_length; + uint32_t nread = parser->nread; /* We're in an error state. Don't bother doing anything. */ @@ -731,7 +733,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; - parser->extra_flags = 0; + parser->uses_transfer_encoding = 0; parser->content_length = ULLONG_MAX; if (ch == 'H') { @@ -769,7 +771,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; - parser->extra_flags = 0; + parser->uses_transfer_encoding = 0; parser->content_length = ULLONG_MAX; if (ch == 'H') { @@ -927,7 +929,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; - parser->extra_flags = 0; + parser->uses_transfer_encoding = 0; parser->content_length = ULLONG_MAX; if (UNLIKELY(!IS_ALPHA(ch))) { @@ -1341,7 +1343,7 @@ size_t http_parser_execute (http_parser *parser, parser->header_state = h_general; } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { parser->header_state = h_transfer_encoding; - parser->extra_flags |= F_TRANSFER_ENCODING >> 8; + parser->uses_transfer_encoding = 1; } break; @@ -1801,14 +1803,19 @@ size_t http_parser_execute (http_parser *parser, REEXECUTE(); } - /* Cannot us transfer-encoding and a content-length header together + /* Cannot use transfer-encoding and a content-length header together per the HTTP specification. (RFC 7230 Section 3.3.3) */ - if ((parser->extra_flags & (F_TRANSFER_ENCODING >> 8)) && + if ((parser->uses_transfer_encoding == 1) && (parser->flags & F_CONTENTLENGTH)) { /* Allow it for lenient parsing as long as `Transfer-Encoding` is - * not `chunked` + * not `chunked` or allow_length_with_encoding is set */ - if (!lenient || (parser->flags & F_CHUNKED)) { + if (parser->flags & F_CHUNKED) { + if (!allow_chunked_length) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } + } else if (!lenient) { SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); goto error; } @@ -1889,7 +1896,7 @@ size_t http_parser_execute (http_parser *parser, /* chunked encoding - ignore Content-Length header, * prepare for a chunk */ UPDATE_STATE(s_chunk_size_start); - } else if (parser->extra_flags & (F_TRANSFER_ENCODING >> 8)) { + } else if (parser->uses_transfer_encoding == 1) { if (parser->type == HTTP_REQUEST && !lenient) { /* RFC 7230 3.3.3 */ @@ -2165,7 +2172,7 @@ http_message_needs_eof (const http_parser *parser) } /* RFC 7230 3.3.3, see `s_headers_almost_done` */ - if ((parser->extra_flags & (F_TRANSFER_ENCODING >> 8)) && + if ((parser->uses_transfer_encoding == 1) && (parser->flags & F_CHUNKED) == 0) { return 1; } diff --git a/http_parser.h b/http_parser.h index cfbffdcd..3772b399 100644 --- a/http_parser.h +++ b/http_parser.h @@ -227,7 +227,6 @@ enum flags , F_UPGRADE = 1 << 5 , F_SKIPBODY = 1 << 6 , F_CONTENTLENGTH = 1 << 7 - , F_TRANSFER_ENCODING = 1 << 8 /* Never set in http_parser.flags */ }; @@ -302,7 +301,10 @@ struct http_parser { unsigned int state : 7; /* enum state from http_parser.c */ unsigned int header_state : 7; /* enum header_state from http_parser.c */ unsigned int index : 5; /* index into current matcher */ - unsigned int extra_flags : 2; + unsigned int uses_transfer_encoding : 1; /* Transfer-Encoding header is present */ + unsigned int allow_chunked_length : 1; /* Allow headers with both + * `Content-Length` and + * `Transfer-Encoding: chunked` set */ unsigned int lenient_http_headers : 1; uint32_t nread; /* # bytes read in various scenarios */ diff --git a/test.c b/test.c index 53b19997..53a3163d 100644 --- a/test.c +++ b/test.c @@ -82,6 +82,7 @@ struct message { int status_cb_called; int message_complete_on_eof; int body_is_final; + int allow_chunked_length; }; static int currently_parsing_eof; @@ -1293,6 +1294,37 @@ const struct message requests[] = ,.num_chunks_complete= 2 ,.chunk_lengths= { 0x1e } } + +#define CHUNKED_CONTENT_LENGTH 46 +, {.name= "chunked with content-length set, allow_chunked_length flag is set" + ,.type= HTTP_REQUEST + ,.raw= "POST /chunked_w_content_length HTTP/1.1\r\n" + "Content-Length: 10\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5; ilovew3;whattheluck=aretheseparametersfor\r\nhello\r\n" + "6; blahblah; blah\r\n world\r\n" + "0\r\n" + "\r\n" + ,.allow_chunked_length = 1 + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.query_string= "" + ,.fragment= "" + ,.request_path= "/chunked_w_content_length" + ,.request_url= "/chunked_w_content_length" + ,.content_length= 10 + ,.num_headers= 2 + ,.headers={ { "Content-Length", "10"} + , { "Transfer-Encoding", "chunked" } + } + ,.body= "hello world" + ,.num_chunks_complete= 3 + ,.chunk_lengths= { 5, 6 } + } }; /* * R E S P O N S E S * */ @@ -3582,6 +3614,9 @@ test_message (const struct message *message) size_t msg1len; for (msg1len = 0; msg1len < raw_len; msg1len++) { parser_init(message->type); + if (message->allow_chunked_length) { + parser.allow_chunked_length = 1; + } size_t read; const char *msg1 = message->raw; @@ -4023,6 +4058,11 @@ test_multiple3 (const struct message *r1, const struct message *r2, const struct strcat(total, r3->raw); parser_init(r1->type); + if (r1->allow_chunked_length || + r2->allow_chunked_length || + r3->allow_chunked_length) { + parser.allow_chunked_length = 1; + } size_t read; @@ -4225,6 +4265,9 @@ test_message_pause (const struct message *msg) size_t nread; parser_init(msg->type); + if (msg->allow_chunked_length) { + parser.allow_chunked_length = 1; + } do { nread = parse_pause(buf, buflen); From 4f15b7d510dc7c6361a26a7c6d2f7c3a17f8d878 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Fri, 10 Jul 2020 11:55:11 +0200 Subject: [PATCH 15/16] Fix sizeof(http_parser) assert The result should be 32 on both 32 bits and 64 bits x86 because of struct padding. Fixes: https://github.com/nodejs/http-parser/issues/507 PR-URL: https://github.com/nodejs/http-parser/pull/510 Reviewed-By: Fedor Indutny --- test.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test.c b/test.c index 53a3163d..3f7c77b3 100644 --- a/test.c +++ b/test.c @@ -4343,7 +4343,13 @@ main (void) printf("http_parser v%u.%u.%u (0x%06lx)\n", major, minor, patch, version); printf("sizeof(http_parser) = %u\n", (unsigned int)sizeof(http_parser)); - assert(sizeof(http_parser) == 4 + 4 + 8 + 2 + 2 + 4 + sizeof(void *)); + +#if defined(__i386__) || defined(__x86_64__) + /* Should be 32 on both 32 bits and 64 bits x86 because of struct padding, + * see https://github.com/nodejs/http-parser/issues/507. + */ + assert(sizeof(http_parser) == 32); +#endif //// API test_preserve_data(); From ec8b5ee63f0e51191ea43bb0c6eac7bfbff3141d Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Fri, 2 Oct 2020 03:20:28 +0200 Subject: [PATCH 16/16] doc: add maintenance notice to readme I'm moving on and as the last (semi-)active maintainer, that means http-parser is now effectively unmaintained. Refs: https://github.com/nodejs/http-parser/issues/522 Reviewed-By: Ben Noordhuis --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b265d717..e38d3a57 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ HTTP Parser =========== +http-parser is [**not** actively maintained](https://github.com/nodejs/http-parser/issues/522). +New projects and projects looking to migrate should consider [llhttp](https://github.com/nodejs/llhttp). + [![Build Status](https://api.travis-ci.org/nodejs/http-parser.svg?branch=master)](https://travis-ci.org/nodejs/http-parser) This is a parser for HTTP messages written in C. It parses both requests and