Improved multipart form data interface

README.md

@@ -119,15 +119,14 @@ svr.Get("/stream", [&](const Request &req, Response &res) {
 svr.Post("/content_receiver",
   [&](const Request &req, Response &res, const ContentReader &content_reader) {
     if (req.is_multipart_form_data()) {
-      MultipartFiles files;
+      MultipartFormDataItems files;
       content_reader(
-        [&](const std::string &name, const MultipartFile &file) {
-          files.emplace(name, file);
+        [&](const MultipartFormData &file) {
+          files.push_back(file);
           return true;
         },
-        [&](const std::string &name, const char *data, size_t data_length) {
-          auto &file = files.find(name)->second;
-          file.content.append(data, data_length);
+        [&](const char *data, size_t data_length) {
+          files.back().content.append(data, data_length);
           return true;
         });
     } else {
@@ -324,16 +323,23 @@ std::shared_ptr<httplib::Response> res =
 
 This feature was contributed by [underscorediscovery](https://github.com/yhirose/cpp-httplib/pull/23).
 
-### Basic Authentication
+### Authentication
+
+NOTE: OpenSSL is required for Digest Authentication, since cpp-httplib uses message digest functions in OpenSSL.
 
 ```cpp
 httplib::Client cli("httplib.org");
+cli.set_auth("user", "pass");
 
-auto res = cli.Get("/basic-auth/hello/world", {
-  httplib::make_basic_authentication_header("hello", "world")
-});
+// Basic
+auto res = cli.Get("/basic-auth/user/pass");
 // res->status should be 200
-// res->body should be "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n".
+// res->body should be "{\n  \"authenticated\": true, \n  \"user\": \"user\"\n}\n".
+
+// Digest
+res = cli.Get("/digest-auth/auth/user/pass/SHA-256");
+// res->status should be 200
+// res->body should be "{\n  \"authenticated\": true, \n  \"user\": \"user\"\n}\n".
 ```
 
 ### Range

example/Makefile

@@ -33,4 +33,4 @@ pem:
 	openssl req -new -key key.pem | openssl x509 -days 3650 -req -signkey key.pem > cert.pem
 
 clean:
-	rm server client hello simplesvr upload redirect *.pem
+	rm server client hello simplesvr upload redirect benchmark *.pem

httplib.h

@@ -149,9 +149,13 @@ using socket_t = int;
 
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
 #include <openssl/err.h>
+#include <openssl/md5.h>
 #include <openssl/ssl.h>
 #include <openssl/x509v3.h>
 
+#include <iomanip>
+#include <sstream>
+
 // #if OPENSSL_VERSION_NUMBER < 0x1010100fL
 // #error Sorry, OpenSSL versions prior to 1.1.1 are not supported
 // #endif
@@ -207,13 +211,6 @@ using Progress = std::function<bool(uint64_t current, uint64_t total)>;
 struct Response;
 using ResponseHandler = std::function<bool(const Response &response)>;
 
-struct MultipartFile {
-  std::string filename;
-  std::string content_type;
-  std::string content;
-};
-using MultipartFiles = std::multimap<std::string, MultipartFile>;
-
 struct MultipartFormData {
   std::string name;
   std::string content;
@@ -221,25 +218,23 @@ struct MultipartFormData {
   std::string content_type;
 };
 using MultipartFormDataItems = std::vector<MultipartFormData>;
+using MultipartFormDataMap = std::multimap<std::string, MultipartFormData>;
 
 using ContentReceiver =
     std::function<bool(const char *data, size_t data_length)>;
 
 using MultipartContentHeader =
-    std::function<bool(const std::string &name, const MultipartFile &file)>;
-
-using MultipartContentReceiver =
-    std::function<bool(const std::string& name, const char *data, size_t data_length)>;
+    std::function<bool(const MultipartFormData &file)>;
 
 class ContentReader {
   public:
     using Reader = std::function<bool(ContentReceiver receiver)>;
-    using MultipartReader = std::function<bool(MultipartContentHeader header, MultipartContentReceiver receiver)>;
+    using MultipartReader = std::function<bool(MultipartContentHeader header, ContentReceiver receiver)>;
 
     ContentReader(Reader reader, MultipartReader muitlpart_reader)
       : reader_(reader), muitlpart_reader_(muitlpart_reader) {}
 
-    bool operator()(MultipartContentHeader header, MultipartContentReceiver receiver) const {
+    bool operator()(MultipartContentHeader header, ContentReceiver receiver) const {
       return muitlpart_reader_(header, receiver);
     }
 
@@ -264,7 +259,7 @@ struct Request {
   std::string version;
   std::string target;
   Params params;
-  MultipartFiles files;
+  MultipartFormDataMap files;
   Ranges ranges;
   Match matches;
 
@@ -291,7 +286,7 @@ struct Request {
   bool is_multipart_form_data() const;
 
   bool has_file(const char *key) const;
-  MultipartFile get_file_value(const char *key) const;
+  MultipartFormData get_file_value(const char *key) const;
 
   // private members...
   size_t content_length;
@@ -592,12 +587,12 @@ private:
                                           Request &req, Response &res,
                                           ContentReceiver receiver,
                                           MultipartContentHeader multipart_header,
-                                          MultipartContentReceiver multipart_receiver);
+                                          ContentReceiver multipart_receiver);
   bool read_content_core(Stream &strm, bool last_connection,
                          Request &req, Response &res,
                          ContentReceiver receiver,
                          MultipartContentHeader mulitpart_header,
-                         MultipartContentReceiver multipart_receiver);
+                         ContentReceiver multipart_receiver);
 
   virtual bool process_and_close_socket(socket_t sock);
 
@@ -756,10 +751,13 @@ public:
             std::vector<Response> &responses);
 
   void set_keep_alive_max_count(size_t count);
+
   void set_read_timeout(time_t sec, time_t usec);
 
   void follow_location(bool on);
 
+  void set_auth(const char *username, const char *password);
+
 protected:
   bool process_request(Stream &strm, const Request &req, Response &res,
                        bool last_connection, bool &connection_close);
@@ -772,6 +770,8 @@ protected:
   time_t read_timeout_sec_;
   time_t read_timeout_usec_;
   size_t follow_location_;
+  std::string username_;
+  std::string password_;
 
 private:
   socket_t create_client_socket() const;
@@ -903,6 +903,8 @@ private:
 };
 #endif
 
+// ----------------------------------------------------------------------------
+
 /*
  * Implementation
  */
@@ -1439,6 +1441,7 @@ inline const char *status_message(int status) {
   case 303: return "See Other";
   case 304: return "Not Modified";
   case 400: return "Bad Request";
+  case 401: return "Unauthorized";
   case 403: return "Forbidden";
   case 404: return "Not Found";
   case 413: return "Payload Too Large";
@@ -2003,8 +2006,9 @@ public:
       case 2: { // Headers
         auto pos = buf_.find(crlf_);
         while (pos != std::string::npos) {
+          // Empty line
           if (pos == 0) {
-            if (!header_callback(name_, file_)) {
+            if (!header_callback(file_)) {
               is_valid_ = false;
               is_done_ = false;
               return false;
@@ -2021,7 +2025,7 @@ public:
             if (std::regex_match(header, m, re_content_type)) {
               file_.content_type = m[1];
             } else if (std::regex_match(header, m, re_content_disposition)) {
-              name_ = m[1];
+              file_.name = m[1];
               file_.filename = m[2];
             }
           }
@@ -2039,7 +2043,7 @@ public:
           if (pos == std::string::npos) {
             pos = buf_.size();
           }
-          if (!content_callback(name_, buf_.data(), pos)) {
+          if (!content_callback(buf_.data(), pos)) {
             is_valid_ = false;
             is_done_ = false;
             return false;
@@ -2055,7 +2059,7 @@ public:
 
           auto pos = buf_.find(pattern);
           if (pos != std::string::npos) {
-            if (!content_callback(name_, buf_.data(), pos)) {
+            if (!content_callback(buf_.data(), pos)) {
               is_valid_ = false;
               is_done_ = false;
               return false;
@@ -2065,7 +2069,7 @@ public:
             buf_.erase(0, pos + pattern.size());
             state_ = 4;
           } else {
-            if (!content_callback(name_, buf_.data(), pattern.size())) {
+            if (!content_callback(buf_.data(), pattern.size())) {
               is_valid_ = false;
               is_done_ = false;
               return false;
@@ -2110,7 +2114,7 @@ public:
 
 private:
   void clear_file_info() {
-    name_.clear();
+    file_.name.clear();
     file_.filename.clear();
     file_.content_type.clear();
   }
@@ -2124,8 +2128,7 @@ private:
   size_t is_valid_ = false;
   size_t is_done_ = false;
   size_t off_ = 0;
-  std::string name_;
-  MultipartFile file_;
+  MultipartFormData file_;
 };
 
 inline std::string to_lower(const char *beg, const char *end) {
@@ -2287,6 +2290,43 @@ inline bool expect_content(const Request &req) {
   return false;
 }
 
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+template <typename CTX, typename Init, typename Update, typename Final>
+inline std::string message_digest(const std::string &s, Init init,
+                                  Update update, Final final,
+                                  size_t digest_length) {
+  using namespace std;
+
+  unsigned char md[digest_length];
+  CTX ctx;
+  init(&ctx);
+  update(&ctx, s.data(), s.size());
+  final(md, &ctx);
+
+  stringstream ss;
+  for (auto c : md) {
+    ss << setfill('0') << setw(2) << hex << (unsigned int)c;
+  }
+  return ss.str();
+}
+
+inline std::string MD5(const std::string &s) {
+  using namespace detail;
+  return message_digest<MD5_CTX>(s, MD5_Init, MD5_Update, MD5_Final,
+                                 MD5_DIGEST_LENGTH);
+}
+
+inline std::string SHA_256(const std::string &s) {
+  return message_digest<SHA256_CTX>(s, SHA256_Init, SHA256_Update, SHA256_Final,
+                                    SHA256_DIGEST_LENGTH);
+}
+
+inline std::string SHA_512(const std::string &s) {
+  return message_digest<SHA512_CTX>(s, SHA512_Init, SHA512_Update, SHA512_Final,
+                                    SHA512_DIGEST_LENGTH);
+}
+#endif
+
 #ifdef _WIN32
 class WSInit {
 public:
@@ -2324,6 +2364,98 @@ make_basic_authentication_header(const std::string &username,
   return std::make_pair("Authorization", field);
 }
 
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+inline std::pair<std::string, std::string> make_digest_authentication_header(
+    const Request &req,
+    const std::map<std::string, std::string> &auth,
+    size_t cnonce_count, const std::string &cnonce,
+    const std::string &username, const std::string &password) {
+  using namespace std;
+
+  string nc;
+  {
+    stringstream ss;
+    ss << setfill('0') << setw(8) << hex << cnonce_count;
+    nc = ss.str();
+  }
+
+  auto qop = auth.at("qop");
+  if (qop.find("auth-int") != std::string::npos) {
+    qop = "auth-int";
+  } else {
+    qop = "auth";
+  }
+
+  string response;
+  {
+    auto algo = auth.at("algorithm");
+
+    auto H = algo == "SHA-256"
+                 ? detail::SHA_256
+                 : algo == "SHA-512" ? detail::SHA_512 : detail::MD5;
+
+    auto A1 = username + ":" + auth.at("realm") + ":" + password;
+
+    auto A2 = req.method + ":" + req.path;
+    if (qop == "auth-int") {
+      A2 += ":" + H(req.body);
+    }
+
+    response = H(H(A1) + ":" + auth.at("nonce") + ":" + nc + ":" + cnonce +
+                 ":" + qop + ":" + H(A2));
+  }
+
+  auto field = "Digest username=\"hello\", realm=\"" + auth.at("realm") +
+               "\", nonce=\"" + auth.at("nonce") + "\", uri=\"" + req.path +
+               "\", algorithm=" + auth.at("algorithm") + ", qop=" + qop + ", nc=\"" +
+               nc + "\", cnonce=\"" + cnonce + "\", response=\"" + response +
+               "\"";
+
+  return make_pair("Authorization", field);
+}
+#endif
+
+inline int parse_www_authenticate(const httplib::Response &res,
+                            std::map<std::string, std::string> &digest_auth) {
+  if (res.has_header("WWW-Authenticate")) {
+    static auto re = std::regex(R"~((?:(?:,\s*)?(.+?)=(?:"(.*?)"|([^,]*))))~");
+    auto s = res.get_header_value("WWW-Authenticate");
+    auto pos = s.find(' ');
+    if (pos != std::string::npos) {
+      auto type = s.substr(0, pos);
+      if (type == "Basic") {
+        return 1;
+      } else if (type == "Digest") {
+        s = s.substr(pos + 1);
+        auto beg = std::sregex_iterator(s.begin(), s.end(), re);
+        for (auto i = beg; i != std::sregex_iterator(); ++i) {
+          auto m = *i;
+          auto key = s.substr(m.position(1), m.length(1));
+          auto val = m.length(2) > 0 ? s.substr(m.position(2), m.length(2))
+                                     : s.substr(m.position(3), m.length(3));
+          digest_auth[key] = val;
+        }
+        return 2;
+      }
+    }
+  }
+  return 0;
+}
+
+// https://stackoverflow.com/questions/440133/how-do-i-create-a-random-alpha-numeric-string-in-c/440240#answer-440240
+inline std::string random_string(size_t length) {
+  auto randchar = []() -> char {
+    const char charset[] = "0123456789"
+                           "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                           "abcdefghijklmnopqrstuvwxyz";
+    const size_t max_index = (sizeof(charset) - 1);
+    return charset[rand() % max_index];
+  };
+  std::string str(length, 0);
+  std::generate_n(str.begin(), length, randchar);
+  return str;
+}
+
 // Request implementation
 inline bool Request::has_header(const char *key) const {
   return detail::has_header(headers, key);
@@ -2371,10 +2503,10 @@ inline bool Request::has_file(const char *key) const {
   return files.find(key) != files.end();
 }
 
-inline MultipartFile Request::get_file_value(const char *key) const {
+inline MultipartFormData Request::get_file_value(const char *key) const {
   auto it = files.find(key);
   if (it != files.end()) { return it->second; }
-  return MultipartFile();
+  return MultipartFormData();
 }
 
 // Response implementation
@@ -2836,6 +2968,7 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
 
 inline bool Server::read_content(Stream &strm, bool last_connection,
                                  Request &req, Response &res) {
+  MultipartFormDataMap::iterator cur;
   auto ret = read_content_core(strm, last_connection, req, res,
     // Regular
     [&](const char *buf, size_t n) {
@@ -2844,14 +2977,12 @@ inline bool Server::read_content(Stream &strm, bool last_connection,
       return true;
     },
     // Multipart
-    [&](const std::string &name, const MultipartFile &file) {
-      req.files.emplace(name, file);
+    [&](const MultipartFormData &file) {
+      cur = req.files.emplace(file.name, file);
       return true;
     },
-    [&](const std::string &name, const char *buf, size_t n) {
-      // TODO: handle elements with a same key
-      auto it = req.files.find(name);
-      auto &content = it->second.content;
+    [&](const char *buf, size_t n) {
+      auto &content = cur->second.content;
       if (content.size() + n > content.max_size()) { return false; }
       content.append(buf, n);
       return true;
@@ -2871,7 +3002,7 @@ Server::read_content_with_content_receiver(Stream &strm, bool last_connection,
                                            Request &req, Response &res,
                                            ContentReceiver receiver,
                                            MultipartContentHeader multipart_header,
-                                           MultipartContentReceiver multipart_receiver) {
+                                           ContentReceiver multipart_receiver) {
   return read_content_core(strm, last_connection, req, res,
       receiver, multipart_header, multipart_receiver);
 }
@@ -2881,7 +3012,7 @@ Server::read_content_core(Stream &strm, bool last_connection,
                           Request &req, Response &res,
                           ContentReceiver receiver,
                           MultipartContentHeader mulitpart_header,
-                          MultipartContentReceiver multipart_receiver) {
+                          ContentReceiver multipart_receiver) {
   detail::MultipartFormDataParser multipart_form_data_parser;
   ContentReceiver out;
 
@@ -3044,7 +3175,7 @@ inline bool Server::routing(Request &req, Response &res, Stream &strm,
           return read_content_with_content_receiver(strm, last_connection, req, res,
                                                     receiver, nullptr, nullptr);
         },
-        [&](MultipartContentHeader header, MultipartContentReceiver receiver) {
+        [&](MultipartContentHeader header, ContentReceiver receiver) {
           return read_content_with_content_receiver(strm, last_connection, req, res,
                                                     nullptr, header, receiver);
         }
@@ -3244,6 +3375,43 @@ inline bool Client::send(const Request &req, Response &res) {
     ret = redirect(req, res);
   }
 
+  if (ret && !username_.empty() && !password_.empty() && res.status == 401) {
+    int type;
+    std::map<std::string, std::string> digest_auth;
+
+    if ((type = parse_www_authenticate(res, digest_auth)) > 0) {
+      std::pair<std::string, std::string> header;
+
+      if (type == 1) {
+        header = make_basic_authentication_header(username_, password_);
+      } else if (type == 2) {
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+        size_t cnonce_count = 1;
+        auto cnonce = random_string(10);
+
+        header = make_digest_authentication_header(
+            req, digest_auth, cnonce_count, cnonce, username_, password_);
+#endif
+      }
+
+      Request new_req;
+      new_req.method = req.method;
+      new_req.path = req.path;
+      new_req.headers = req.headers;
+      new_req.body = req.body;
+      new_req.response_handler = req.response_handler;
+      new_req.content_receiver = req.content_receiver;
+      new_req.progress = req.progress;
+
+      new_req.headers.insert(header);
+
+      Response new_res;
+      auto ret = send(new_req, new_res);
+      if (ret) { res = new_res; }
+      return ret;
+    }
+  }
+
   return ret;
 }
 
@@ -3810,6 +3978,11 @@ inline void Client::set_read_timeout(time_t sec, time_t usec) {
 
 inline void Client::follow_location(bool on) { follow_location_ = on; }
 
+inline void Client::set_auth(const char *username, const char *password) {
+  username_ = username;
+  password_ = password;
+}
+
 /*
  * SSL Implementation
  */
@@ -4254,6 +4427,8 @@ inline bool SSLClient::check_host_name(const char *pattern,
 }
 #endif
 
+// ----------------------------------------------------------------------------
+
 } // namespace httplib
 
 #endif // CPPHTTPLIB_HTTPLIB_H

split.py

@@ -0,0 +1,24 @@
+import os
+
+border = '// ----------------------------------------------------------------------------'
+
+with open('httplib.h') as f:
+    lines = f.readlines()
+    inImplementation = False
+    os.makedirs('out', exist_ok=True)
+    with open('out/httplib.h', 'w') as fh:
+        with open('out/httplib.cc', 'w') as fc:
+            fc.write('#include "httplib.h"\n')
+            fc.write('namespace httplib {\n')
+            for line in lines:
+                isBorderLine = border in line
+                if isBorderLine:
+                    inImplementation = not inImplementation
+                else:
+                    if inImplementation:
+                        fc.write(line.replace('inline ', ''))
+                        pass
+                    else:
+                        fh.write(line)
+                        pass
+            fc.write('} // namespace httplib\n')

test/test.cc

@@ -30,9 +30,11 @@ const std::string JSON_DATA = "{\"hello\":\"world\"}";
 
 const string LARGE_DATA = string(1024 * 1024 * 100, '@'); // 100MB
 
-MultipartFile& get_file_value(MultipartFiles &files, const char *key) {
-  auto it = files.find(key);
-  if (it != files.end()) { return it->second; }
+MultipartFormData& get_file_value(MultipartFormDataItems &files, const char *key) {
+  auto it = std::find_if(files.begin(), files.end(), [&](const MultipartFormData &file) {
+    return file.name == key;
+  });
+  if (it != files.end()) { return *it; }
   throw std::runtime_error("invalid mulitpart form data name error");
 }
 
@@ -469,8 +471,50 @@ TEST(BaseAuthTest, FromHTTPWatch) {
               "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
     EXPECT_EQ(200, res->status);
   }
+
+  {
+    cli.set_auth("hello", "world");
+    auto res = cli.Get("/basic-auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(res->body,
+              "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+    EXPECT_EQ(200, res->status);
+  }
 }
 
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+TEST(DigestAuthTest, FromHTTPWatch) {
+  auto host = "httpbin.org";
+  auto port = 443;
+  httplib::SSLClient cli(host, port);
+
+  {
+    auto res = cli.Get("/digest-auth/auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(401, res->status);
+  }
+
+  {
+    std::vector<std::string> paths = {
+      "/digest-auth/auth/hello/world/MD5",
+      "/digest-auth/auth/hello/world/SHA-256",
+      "/digest-auth/auth/hello/world/SHA-512",
+      "/digest-auth/auth-init/hello/world/MD5",
+      "/digest-auth/auth-int/hello/world/MD5",
+    };
+
+    cli.set_auth("hello", "world");
+    for (auto path: paths) {
+      auto res = cli.Get(path.c_str());
+      ASSERT_TRUE(res != nullptr);
+      EXPECT_EQ(res->body,
+                "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+      EXPECT_EQ(200, res->status);
+    }
+  }
+}
+#endif
+
 TEST(AbsoluteRedirectTest, Redirect) {
   auto host = "httpbin.org";
 
@@ -759,15 +803,14 @@ protected:
         .Post("/content_receiver",
               [&](const Request & req, Response &res, const ContentReader &content_reader) {
                 if (req.is_multipart_form_data()) {
-                  MultipartFiles files;
+                  MultipartFormDataItems files;
                   content_reader(
-                    [&](const std::string &name, const MultipartFile &file) {
-                      files.emplace(name, file);
+                    [&](const MultipartFormData &file) {
+                      files.push_back(file);
                       return true;
                     },
-                    [&](const std::string &name, const char *data, size_t data_length) {
-                      auto &file = files.find(name)->second;
-                      file.content.append(data, data_length);
+                    [&](const char *data, size_t data_length) {
+                      files.back().content.append(data, data_length);
                       return true;
                     });
 
@@ -1357,7 +1400,7 @@ TEST_F(ServerTest, GetStreamedWithRangeMultipart) {
 }
 
 TEST_F(ServerTest, GetStreamedEndless) {
-  size_t offset = 0;
+  uint64_t offset = 0;
   auto res = cli_.Get("/streamed-cancel",
                       [&](const char * /*data*/, uint64_t data_length) {
                         if (offset < 100) {