diff --git a/README.md b/README.md index 80d6fc4..b988791 100644 --- a/README.md +++ b/README.md @@ -989,6 +989,17 @@ httplib::UploadFormDataItems items = { auto res = cli.Post("/multipart", items); ``` +To upload files from disk without loading them entirely into memory, use `make_file_provider`. The file is read and sent in chunks with a correct `Content-Length` header. + +```cpp +httplib::FormDataProviderItems providers = { + httplib::make_file_provider("file1", "/path/to/large.bin", "large.bin", "application/octet-stream"), + httplib::make_file_provider("avatar", "/path/to/photo.jpg", "photo.jpg", "image/jpeg"), +}; + +auto res = cli.Post("/upload", {}, {}, providers); +``` + ### PUT ```c++ diff --git a/httplib.h b/httplib.h index 8de2719..2b61c24 100644 --- a/httplib.h +++ b/httplib.h @@ -334,6 +334,7 @@ using socket_t = int; #include #include #include +#include #include #include #include @@ -1001,6 +1002,34 @@ struct FormDataProvider { }; using FormDataProviderItems = std::vector; +inline FormDataProvider +make_file_provider(const std::string &name, const std::string &filepath, + const std::string &filename = std::string(), + const std::string &content_type = std::string()) { + FormDataProvider fdp; + fdp.name = name; + fdp.filename = filename.empty() ? filepath : filename; + fdp.content_type = content_type; + fdp.provider = [filepath](size_t offset, DataSink &sink) -> bool { + std::ifstream f(filepath, std::ios::binary); + if (!f) { return false; } + if (offset > 0) { + f.seekg(static_cast(offset)); + if (!f.good()) { + sink.done(); + return true; + } + } + char buf[8192]; + f.read(buf, sizeof(buf)); + auto n = static_cast(f.gcount()); + if (n > 0) { return sink.write(buf, n); } + sink.done(); // EOF + return true; + }; + return fdp; +} + using ContentReceiverWithProgress = std::function; @@ -7870,6 +7899,64 @@ serialize_multipart_formdata(const UploadFormDataItems &items, return body; } +inline size_t get_multipart_content_length(const UploadFormDataItems &items, + const std::string &boundary) { + size_t total = 0; + for (const auto &item : items) { + total += serialize_multipart_formdata_item_begin(item, boundary).size(); + total += item.content.size(); + total += serialize_multipart_formdata_item_end().size(); + } + total += serialize_multipart_formdata_finish(boundary).size(); + return total; +} + +struct MultipartSegment { + const char *data; + size_t size; +}; + +// NOTE: items must outlive the returned ContentProvider +// (safe for synchronous use inside Post/Put/Patch) +inline ContentProvider +make_multipart_content_provider(const UploadFormDataItems &items, + const std::string &boundary) { + // Own the per-item header strings and the finish string + std::vector owned; + owned.reserve(items.size() + 1); + for (const auto &item : items) + owned.push_back(serialize_multipart_formdata_item_begin(item, boundary)); + owned.push_back(serialize_multipart_formdata_finish(boundary)); + + // Flat segment list: [header, content, "\r\n"] * N + [finish] + std::vector segs; + segs.reserve(items.size() * 3 + 1); + static const char crlf[] = "\r\n"; + for (size_t i = 0; i < items.size(); i++) { + segs.push_back({owned[i].data(), owned[i].size()}); + segs.push_back({items[i].content.data(), items[i].content.size()}); + segs.push_back({crlf, 2}); + } + segs.push_back({owned.back().data(), owned.back().size()}); + + return [owned = std::move(owned), segs = std::move(segs)]( + size_t offset, size_t length, DataSink &sink) -> bool { + size_t pos = 0; + for (const auto &seg : segs) { + // Loop invariant: pos <= offset (proven by advancing pos only when + // offset - pos >= seg.size, i.e., the segment doesn't contain offset) + if (seg.size > 0 && offset - pos < seg.size) { + size_t seg_offset = offset - pos; + size_t available = seg.size - seg_offset; + size_t to_write = std::min(available, length); + return sink.write(seg.data + seg_offset, to_write); + } + pos += seg.size; + } + return true; // past end (shouldn't be reached when content_length is exact) + }; +} + inline void coalesce_ranges(Ranges &ranges, size_t content_length) { if (ranges.size() <= 1) return; @@ -13402,8 +13489,10 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const auto &boundary = detail::make_multipart_data_boundary(); const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); - const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Post(path, headers, body, content_type, progress); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Post(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); } inline Result ClientImpl::Post(const std::string &path, const Headers &headers, @@ -13416,8 +13505,10 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); - const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Post(path, headers, body, content_type, progress); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Post(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); } inline Result ClientImpl::Post(const std::string &path, const Headers &headers, @@ -13595,8 +13686,10 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers, const auto &boundary = detail::make_multipart_data_boundary(); const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); - const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Put(path, headers, body, content_type, progress); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Put(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); } inline Result ClientImpl::Put(const std::string &path, const Headers &headers, @@ -13609,8 +13702,10 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers, const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); - const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Put(path, headers, body, content_type, progress); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Put(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); } inline Result ClientImpl::Put(const std::string &path, const Headers &headers, @@ -13790,8 +13885,10 @@ inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, const auto &boundary = detail::make_multipart_data_boundary(); const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); - const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Patch(path, headers, body, content_type, progress); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Patch(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); } inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, @@ -13804,8 +13901,10 @@ inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, const auto &content_type = detail::serialize_multipart_formdata_get_content_type(boundary); - const auto &body = detail::serialize_multipart_formdata(items, boundary); - return Patch(path, headers, body, content_type, progress); + auto content_length = detail::get_multipart_content_length(items, boundary); + return Patch(path, headers, content_length, + detail::make_multipart_content_provider(items, boundary), + content_type, progress); } inline Result ClientImpl::Patch(const std::string &path, const Headers &headers, diff --git a/test/test.cc b/test/test.cc index 3ae89a0..9eb0c02 100644 --- a/test/test.cc +++ b/test/test.cc @@ -11725,6 +11725,100 @@ TEST(MultipartFormDataTest, LargeHeader) { ASSERT_EQ("200", response.substr(9, 3)); } +TEST(MultipartFormDataTest, UploadItemsHasContentLength) { + // Verify that Post(path, headers, UploadFormDataItems) sends Content-Length + // (not chunked Transfer-Encoding) after the streaming refactor. + auto handled = false; + + Server svr; + svr.Post("/upload", [&](const Request &req, Response &res) { + auto cl_it = req.headers.find("Content-Length"); + EXPECT_TRUE(cl_it != req.headers.end()); + auto te_it = req.headers.find("Transfer-Encoding"); + EXPECT_TRUE(te_it == req.headers.end()); + EXPECT_EQ(2u, req.form.fields.size() + req.form.files.size()); + res.set_content("ok", "text/plain"); + handled = true; + }); + + auto port = svr.bind_to_any_port(HOST); + auto t = thread([&] { svr.listen_after_bind(); }); + auto se = detail::scope_exit([&] { + svr.stop(); + t.join(); + ASSERT_FALSE(svr.is_running()); + ASSERT_TRUE(handled); + }); + + svr.wait_until_ready(); + + UploadFormDataItems items = { + {"field1", "hello", "", "text/plain"}, + {"file1", "world", "test.txt", "application/octet-stream"}, + }; + + Client cli(HOST, port); + auto res = cli.Post("/upload", {}, items); + ASSERT_TRUE(res); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + +TEST(MultipartFormDataTest, MakeFileProvider) { + // Verify make_file_provider sends a file's contents correctly. + const std::string file_content(4096, 'Z'); + const std::string tmp_path = "/tmp/httplib_test_make_file_provider.bin"; + { + std::ofstream ofs(tmp_path, std::ios::binary); + ofs.write(file_content.data(), + static_cast(file_content.size())); + } + + auto handled = false; + + Server svr; + svr.Post("/upload", [&](const Request &req, Response & /*res*/, + const ContentReader &content_reader) { + ASSERT_TRUE(req.is_multipart_form_data()); + std::vector items; + content_reader( + [&](const FormData &file) { + items.push_back(file); + return true; + }, + [&](const char *data, size_t data_length) { + items.back().content.append(data, data_length); + return true; + }); + ASSERT_EQ(1u, items.size()); + EXPECT_EQ("myfile", items[0].name); + EXPECT_EQ("data.bin", items[0].filename); + EXPECT_EQ("application/octet-stream", items[0].content_type); + EXPECT_EQ(file_content, items[0].content); + handled = true; + }); + + auto port = svr.bind_to_any_port(HOST); + auto t = thread([&] { svr.listen_after_bind(); }); + auto se = detail::scope_exit([&] { + svr.stop(); + t.join(); + ASSERT_FALSE(svr.is_running()); + ASSERT_TRUE(handled); + std::remove(tmp_path.c_str()); + }); + + svr.wait_until_ready(); + + FormDataProviderItems providers; + providers.push_back(make_file_provider("myfile", tmp_path, "data.bin", + "application/octet-stream")); + + Client cli(HOST, port); + auto res = cli.Post("/upload", {}, {}, providers); + ASSERT_TRUE(res); + EXPECT_EQ(StatusCode::OK_200, res->status); +} + TEST(TaskQueueTest, IncreaseAtomicInteger) { static constexpr unsigned int number_of_tasks{1000000}; std::atomic_uint count{0};