diff --git a/benchmark/ab.sh b/benchmark/ab.sh index 79ef0ef..6014b7b 100755 --- a/benchmark/ab.sh +++ b/benchmark/ab.sh @@ -3,7 +3,25 @@ # A/B throughput comparison between two git refs. # # Usage: ./ab.sh [--base REF] [--head REF] [--rounds N] [--duration S] -# [--connections N] [--threads N] +# [--connections N] [--threads N] [--path PATH] [--tls] +# [--large-mib N] [--timeout S] +# +# --path selects the workload. The harness serves: +# / small body via set_content(); the response line, the +# headers and the body already share a single write(), so +# this is the least sensitive case +# /large large body via set_content() +# /static/small.js 1 KiB file from a mount point, where the headers and the +# body are two separate writes +# /static/large.bin same, with the body large enough to dominate +# +# --large-mib sizes the two large workloads (default 1). +# +# --tls runs the same workload over HTTPS, which writes through +# SSLSocketStream instead of SocketStream. +# +# --timeout is bombardier's per-request timeout. Its 2s default aborts large +# TLS responses, and the run then fails on the non-2xx check. # # Absolute numbers from a single run are meaningless: on a quiet 8-core laptop # the same binary varies by +/-20% run to run, and shared CI runners are worse. @@ -21,6 +39,10 @@ DURATION="5s" CONNECTIONS=10 THREADS="" PORT=8080 +REQ_PATH="/" +TLS=0 +LARGE_MIB=1 +TIMEOUT="30s" while [ $# -gt 0 ]; do case "$1" in @@ -30,6 +52,10 @@ while [ $# -gt 0 ]; do --duration) DURATION="$2"; shift 2 ;; --connections) CONNECTIONS="$2"; shift 2 ;; --threads) THREADS="$2"; shift 2 ;; + --path) REQ_PATH="$2"; shift 2 ;; + --large-mib) LARGE_MIB="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --tls) TLS=1; shift ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac done @@ -62,6 +88,7 @@ HEAD_SHA=$(git -C "$REPO_ROOT" rev-parse --short "$HEAD_REF") echo "==> base: $BASE_REF ($BASE_SHA)" echo "==> head: $HEAD_REF ($HEAD_SHA)" echo "==> rounds=$ROUNDS duration=$DURATION connections=$CONNECTIONS threads=$THREADS" +echo "==> path=$REQ_PATH tls=$TLS large=${LARGE_MIB}MiB" echo "" if [ "$BASE_SHA" = "$HEAD_SHA" ]; then @@ -69,18 +96,49 @@ if [ "$BASE_SHA" = "$HEAD_SHA" ]; then echo "" fi +# --- Toolchain bits that depend on --tls --- +SCHEME="http" +INSECURE="" +TLS_CXXFLAGS="" +TLS_LDFLAGS="" +TLS_ARGS="" +if [ "$TLS" = "1" ]; then + SCHEME="https" + INSECURE="-k" + TLS_CXXFLAGS="-DCPPHTTPLIB_OPENSSL_SUPPORT" + TLS_LDFLAGS="-lssl -lcrypto" + if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists openssl; then + TLS_CXXFLAGS="$TLS_CXXFLAGS $(pkg-config --cflags openssl)" + TLS_LDFLAGS="$(pkg-config --libs openssl)" + elif command -v brew >/dev/null 2>&1 && brew --prefix openssl >/dev/null 2>&1; then + OPENSSL_PREFIX=$(brew --prefix openssl) + TLS_CXXFLAGS="$TLS_CXXFLAGS -I$OPENSSL_PREFIX/include" + TLS_LDFLAGS="-L$OPENSSL_PREFIX/lib -lssl -lcrypto" + fi + if [ "$(uname -s)" = "Darwin" ]; then + TLS_LDFLAGS="$TLS_LDFLAGS -framework CoreFoundation -framework Security" + fi + TLS_ARGS="--cert $REPO_ROOT/test/cert.pem --key $REPO_ROOT/test/key.pem" + for f in "$REPO_ROOT/test/cert.pem" "$REPO_ROOT/test/key.pem"; do + [ -f "$f" ] || { echo "Error: $f not found" >&2; exit 1; } + done +fi + # --- Build both refs --- +# The harness source always comes from the invoking worktree, so both refs run +# an identical workload and a ref that predates a harness change stays +# measurable. Only httplib.h varies, through -I. +HARNESS="$REPO_ROOT/benchmark/cpp-httplib/main.cpp" +[ -f "$HARNESS" ] || { echo "Error: $HARNESS not found" >&2; exit 1; } + build() { local name=$1 ref=$2 git -C "$REPO_ROOT" worktree add --detach --quiet "$WORKDIR/$name" "$ref" - if [ ! -f "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" ]; then - echo "Error: benchmark/cpp-httplib/main.cpp missing in $ref" >&2 - exit 1 - fi "$CXX" -o "$WORKDIR/$name/server-ab" -O2 -std=c++11 \ -I"$WORKDIR/$name" \ -DCPPHTTPLIB_THREAD_POOL_COUNT="$THREADS" \ - "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" -lpthread + $TLS_CXXFLAGS \ + "$HARNESS" -lpthread $TLS_LDFLAGS } echo "==> Building..." @@ -92,7 +150,8 @@ measure() { local name=$1 local json rc - "$WORKDIR/$name/server-ab" >/dev/null 2>&1 & + "$WORKDIR/$name/server-ab" --port "$PORT" --dir "$WORKDIR/$name-www" \ + --large-mib "$LARGE_MIB" $TLS_ARGS >/dev/null 2>&1 & local pid=$! # Wait for the listener (no dependency on nc) @@ -103,8 +162,8 @@ measure() { done set +e - json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -o json -p r \ - "http://127.0.0.1:$PORT/" 2>/dev/null) + json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -t "$TIMEOUT" -o json -p r $INSECURE \ + "$SCHEME://127.0.0.1:$PORT$REQ_PATH" 2>/dev/null) rc=$? set -e diff --git a/benchmark/cpp-httplib/main.cpp b/benchmark/cpp-httplib/main.cpp index ab2e757..56987e6 100644 --- a/benchmark/cpp-httplib/main.cpp +++ b/benchmark/cpp-httplib/main.cpp @@ -1,12 +1,139 @@ #include "httplib.h" + +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#endif + using namespace httplib; -int main() { - Server svr; +namespace { +// The workloads differ in which part of the write path they exercise: +// +// / small body set through set_content(); the response line, +// the headers and the body already share a single write() +// /large large body set through set_content(); the body is copied +// into the header buffer before that single write(). +// --large-mib sets its size (and large.bin's). +// /static/small.js small file served from a mount point, where the headers +// and the body are two separate writes +// /static/large.bin same, with the body large enough to dominate +// +// Bodies are generated at startup so the repository carries no fixtures. + +const size_t SMALL_SIZE = 1024; +const size_t LARGE_SIZE_DEFAULT_MIB = 1; + +std::string filler(size_t n) { + std::string s; + s.reserve(n); + while (s.size() < n) { + s += "0123456789abcdef"; + } + s.resize(n); + return s; +} + +bool write_file(const std::string &path, const std::string &content) { + std::ofstream f(path.c_str(), std::ios::binary); + f.write(content.data(), static_cast(content.size())); + return f.good(); +} + +std::string default_dir() { + const char *tmp = std::getenv("TMPDIR"); + std::string base = tmp && *tmp ? tmp : "/tmp"; + if (!base.empty() && base[base.size() - 1] == '/') { + base.erase(base.size() - 1); + } + return base + "/cpp-httplib-bench"; +} + +bool make_dir(const std::string &path) { +#ifdef _WIN32 + return _mkdir(path.c_str()) == 0 || errno == EEXIST; +#else + return ::mkdir(path.c_str(), 0755) == 0 || errno == EEXIST; +#endif +} + +void setup(Server &svr, const std::string &large, const std::string &dir) { svr.Get("/", [](const Request &, Response &res) { res.set_content("Hello World!", "text/plain"); }); - svr.listen("0.0.0.0", 8080); + svr.Get("/large", [&large](const Request &, Response &res) { + res.set_content(large, "application/octet-stream"); + }); + + svr.set_mount_point("/static", dir); +} + +} // namespace + +int main(int argc, char *argv[]) { + int port = 8080; + std::string dir = default_dir(); + std::string cert; + std::string key; + size_t large_mib = LARGE_SIZE_DEFAULT_MIB; + + for (int i = 1; i < argc; i++) { + auto last = i + 1 < argc; + if (!std::strcmp(argv[i], "--port") && last) { + port = std::atoi(argv[++i]); + } else if (!std::strcmp(argv[i], "--large-mib") && last) { + large_mib = static_cast(std::atoi(argv[++i])); + } else if (!std::strcmp(argv[i], "--dir") && last) { + dir = argv[++i]; + } else if (!std::strcmp(argv[i], "--cert") && last) { + cert = argv[++i]; + } else if (!std::strcmp(argv[i], "--key") && last) { + key = argv[++i]; + } else { + std::fprintf(stderr, + "usage: %s [--port N] [--dir PATH] [--large-mib N]" + " [--cert PATH --key PATH]\n", + argv[0]); + return 2; + } + } + + if (!make_dir(dir)) { + std::fprintf(stderr, "cannot create %s\n", dir.c_str()); + return 1; + } + + auto large = filler(large_mib * 1024 * 1024); + if (!write_file(dir + "/small.js", filler(SMALL_SIZE)) || + !write_file(dir + "/large.bin", large)) { + std::fprintf(stderr, "cannot write fixtures under %s\n", dir.c_str()); + return 1; + } + + if (!cert.empty()) { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + SSLServer svr(cert.c_str(), key.c_str()); + if (!svr.is_valid()) { + std::fprintf(stderr, "cannot load %s / %s\n", cert.c_str(), key.c_str()); + return 1; + } + setup(svr, large, dir); + svr.listen("0.0.0.0", port); + return 0; +#else + std::fprintf(stderr, "built without CPPHTTPLIB_OPENSSL_SUPPORT\n"); + return 1; +#endif + } + + Server svr; + setup(svr, large, dir); + svr.listen("0.0.0.0", port); }