diff --git a/README.md b/README.md index 814c5fd..8155b22 100644 --- a/README.md +++ b/README.md @@ -936,6 +936,7 @@ enum class Error { UnsupportedAddressFamily, HTTPParsing, InvalidRangeHeader, + UnsupportedContentEncoding, }; ``` diff --git a/httplib.h b/httplib.h index 2f07ed0..c0e78cf 100644 --- a/httplib.h +++ b/httplib.h @@ -1514,6 +1514,7 @@ enum class Error { UnsupportedAddressFamily, HTTPParsing, InvalidRangeHeader, + UnsupportedContentEncoding, // For internal use only SSLPeerCouldBeClosed_, @@ -7121,19 +7122,49 @@ inline bool zstd_decompressor::decompress(const char *data, size_t data_length, } #endif +inline bool contains_case_ignore(const std::string &s, const char *token) { + auto token_end = token + std::strlen(token); + return std::search(s.begin(), s.end(), token, token_end, [](char a, char b) { + return case_ignore::to_lower(a) == case_ignore::to_lower(b); + }) != s.end(); +} + +// Content codings are case-insensitive (RFC 9110 8.4.1). Matching them +// case-sensitively would make a response labeled e.g. "GZIP" look like an +// unknown coding, and its payload would be handed back still compressed. +inline bool is_zlib_encoding(const std::string &encoding) { + return case_ignore::equal(encoding, "gzip") || + case_ignore::equal(encoding, "deflate"); +} + +inline bool is_brotli_encoding(const std::string &encoding) { + return contains_case_ignore(encoding, "br"); +} + +inline bool is_zstd_encoding(const std::string &encoding) { + return contains_case_ignore(encoding, "zstd"); +} + +// Returns true if the content coding is one cpp-httplib is able to decompress +// when the corresponding support is compiled in. +inline bool is_known_content_encoding(const std::string &encoding) { + return is_zlib_encoding(encoding) || is_brotli_encoding(encoding) || + is_zstd_encoding(encoding); +} + inline std::unique_ptr create_decompressor(const std::string &encoding) { std::unique_ptr decompressor; - if (encoding == "gzip" || encoding == "deflate") { + if (is_zlib_encoding(encoding)) { #ifdef CPPHTTPLIB_ZLIB_SUPPORT decompressor = detail::make_unique(); #endif - } else if (encoding.find("br") != std::string::npos) { + } else if (is_brotli_encoding(encoding)) { #ifdef CPPHTTPLIB_BROTLI_SUPPORT decompressor = detail::make_unique(); #endif - } else if (encoding == "zstd" || encoding.find("zstd") != std::string::npos) { + } else if (is_zstd_encoding(encoding)) { #ifdef CPPHTTPLIB_ZSTD_SUPPORT decompressor = detail::make_unique(); #endif @@ -7470,9 +7501,12 @@ bool prepare_content_receiver(T &x, int &status, std::unique_ptr decompressor; if (!encoding.empty()) { + // A coding we know about but were not built with is an error. An + // unrecognized coding (including "identity") is left alone and the + // payload is passed through as-is, since some servers misuse the header, + // e.g. by sending a character set such as "Content-Encoding: UTF-8". decompressor = detail::create_decompressor(encoding); - if (!decompressor) { - // Unsupported encoding or no support compiled in + if (!decompressor && detail::is_known_content_encoding(encoding)) { status = StatusCode::UnsupportedMediaType_415; return false; } @@ -9776,6 +9810,7 @@ inline std::string to_string(const Error error) { case Error::UnsupportedAddressFamily: return "Unsupported address family"; case Error::HTTPParsing: return "HTTP parsing failed"; case Error::InvalidRangeHeader: return "Invalid Range header"; + case Error::UnsupportedContentEncoding: return "Unsupported Content-Encoding"; default: break; } @@ -13436,7 +13471,20 @@ ClientImpl::open_stream(const std::string &method, const std::string &path, auto content_encoding = handle.response->get_header_value("Content-Encoding"); if (!content_encoding.empty()) { + // Same policy as prepare_content_receiver(): reject a coding we know about + // but were not built with, pass an unrecognized one through as-is. handle.decompressor_ = detail::create_decompressor(content_encoding); + if (!handle.decompressor_) { + if (detail::is_known_content_encoding(content_encoding)) { + handle.error = Error::UnsupportedContentEncoding; + handle.response.reset(); + return handle; + } + } else if (!handle.decompressor_->is_valid()) { + handle.error = Error::Compression; + handle.response.reset(); + return handle; + } } return handle; @@ -14397,14 +14445,26 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, } if (res.status != StatusCode::NotModified_304) { - int dummy_status; + auto content_status = 0; auto max_length = (!has_payload_max_length_ && req.content_receiver) ? (std::numeric_limits::max)() : payload_max_length_; - if (!detail::read_content(strm, res, max_length, dummy_status, + if (!detail::read_content(strm, res, max_length, content_status, std::move(progress), std::move(out), decompress_)) { - if (error != Error::Canceled) { error = Error::Read; } + if (error != Error::Canceled) { + // Tell the caller apart from a plain read failure when the body could + // not be decoded because of its Content-Encoding. + switch (content_status) { + case StatusCode::UnsupportedMediaType_415: + error = Error::UnsupportedContentEncoding; + break; + case StatusCode::InternalServerError_500: + error = Error::Compression; + break; + default: error = Error::Read; break; + } + } output_error_log(error, &req); return false; } diff --git a/test/test.cc b/test/test.cc index bd2bef8..25ca019 100644 --- a/test/test.cc +++ b/test/test.cc @@ -10433,6 +10433,146 @@ TEST(PayloadLimitBypassTest, StreamingGzipDecompression) { } #endif +// Some servers misuse Content-Encoding to advertise a character set, e.g. +// `Content-Encoding: UTF-8` on a JPEG. Such a value is not a content coding, so +// the payload must be passed through untouched instead of being rejected. +TEST(ContentEncodingTest, UnknownEncodingIsPassedThrough) { + const std::string body = "\xff\xd8\xff\xe0 not really a jpeg"; + + Server svr; + + svr.Get("/image", [&](const Request & /*req*/, Response &res) { + res.set_content(body, "image/jpeg"); + res.set_header("Content-Encoding", "UTF-8"); + }); + + svr.Get("/identity", [&](const Request & /*req*/, Response &res) { + res.set_content(body, "image/jpeg"); + res.set_header("Content-Encoding", "identity"); + }); + + svr.Post("/echo", [](const Request &req, Response &res) { + res.set_content(req.body, "image/jpeg"); + }); + + thread t = thread([&]() { svr.listen(HOST, PORT); }); + auto se = detail::scope_exit([&] { + svr.stop(); + t.join(); + ASSERT_FALSE(svr.is_running()); + }); + + svr.wait_until_ready(); + + Client cli(HOST, PORT); + + { + auto res = cli.Get("/image"); + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ(StatusCode::OK_200, res->status); + EXPECT_EQ(body, res->body); + } + + { + auto res = cli.Get("/identity"); + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ(StatusCode::OK_200, res->status); + EXPECT_EQ(body, res->body); + } + + { + // The same applies to a request body reaching the server. + Headers headers = {{"Content-Encoding", "UTF-8"}}; + auto res = cli.Post("/echo", headers, body, "image/jpeg"); + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ(StatusCode::OK_200, res->status); + EXPECT_EQ(body, res->body); + } +} + +// "Hello World!" as gzip. Hard-coded so that the test below can serve a +// gzip-encoded response even when the build has no zlib support. +static const char GZIPPED_HELLO_WORLD[] = { + '\x1f', '\x8b', '\x08', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x02', '\x03', '\xf3', '\x48', '\xcd', '\xc9', '\xc9', '\x57', + '\x08', '\xcf', '\x2f', '\xca', '\x49', '\x51', '\x04', '\x00', + '\xa3', '\x1c', '\x29', '\x1c', '\x0c', '\x00', '\x00', '\x00'}; + +// A content coding cpp-httplib recognizes but was not built with must be +// reported as such. Handing the still-compressed payload back to the caller +// would silently corrupt it. +TEST(ContentEncodingTest, KnownEncodingWithoutSupportIsReported) { + const std::string gzipped(GZIPPED_HELLO_WORLD, sizeof(GZIPPED_HELLO_WORLD)); + + Server svr; + + // "image/jpeg" keeps the server from applying a content coding of its own, + // so the hand-crafted Content-Encoding below survives. + svr.Get("/gzipped", [&](const Request & /*req*/, Response &res) { + res.set_content(gzipped, "image/jpeg"); + res.set_header("Content-Encoding", "gzip"); + }); + + // Content codings are case-insensitive (RFC 9110 8.4.1). + svr.Get("/gzipped-uppercase", [&](const Request & /*req*/, Response &res) { + res.set_content(gzipped, "image/jpeg"); + res.set_header("Content-Encoding", "GZIP"); + }); + + thread t = thread([&]() { svr.listen(HOST, PORT); }); + auto se = detail::scope_exit([&] { + svr.stop(); + t.join(); + ASSERT_FALSE(svr.is_running()); + }); + + svr.wait_until_ready(); + + Client cli(HOST, PORT); + + { + auto res = cli.Get("/gzipped"); +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ("Hello World!", res->body); +#else + ASSERT_FALSE(res); + EXPECT_EQ(Error::UnsupportedContentEncoding, res.error()); +#endif + } + + { + // open_stream() must behave the same way. + auto handle = cli.open_stream("GET", "/gzipped"); +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + ASSERT_TRUE(handle.is_valid()); + std::string received; + char buf[256]; + ssize_t n; + while ((n = handle.read(buf, sizeof(buf))) > 0) { + received.append(buf, static_cast(n)); + } + EXPECT_EQ("Hello World!", received); +#else + EXPECT_FALSE(handle.is_valid()); + EXPECT_EQ(Error::UnsupportedContentEncoding, handle.error); +#endif + } + + { + // A differently-cased coding must not be mistaken for an unknown one, or + // the body would be handed back still compressed. + auto res = cli.Get("/gzipped-uppercase"); +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ("Hello World!", res->body); +#else + ASSERT_FALSE(res); + EXPECT_EQ(Error::UnsupportedContentEncoding, res.error()); +#endif + } +} + // Regression test for DoS vulnerability: a malicious server sending a response // without Content-Length header must not cause unbounded memory consumption on // the client side. The client should stop reading after a reasonable limit, @@ -16292,6 +16432,10 @@ protected: svr_.Get("/large", [](const Request &, Response &res) { res.set_content(std::string(10000, 'X'), "text/plain"); }); + svr_.Get("/unknown-encoding", [](const Request &, Response &res) { + res.set_content("Hello World!", "image/jpeg"); + res.set_header("Content-Encoding", "UTF-8"); + }); svr_.Get("/chunked", [](const Request &, Response &res) { res.set_chunked_content_provider("text/plain", [](size_t offset, DataSink &sink) { @@ -16380,6 +16524,13 @@ TEST_F(OpenStreamTest, Basic) { EXPECT_EQ("Hello World!", read_all(handle)); } +TEST_F(OpenStreamTest, UnknownContentEncodingIsPassedThrough) { + Client cli("127.0.0.1", port_); + auto handle = cli.open_stream("GET", "/unknown-encoding"); + ASSERT_TRUE(handle.is_valid()); + EXPECT_EQ("Hello World!", read_all(handle)); +} + TEST_F(OpenStreamTest, SmallBuffer) { Client cli("127.0.0.1", port_); auto handle = cli.open_stream("GET", "/hello");