mirror of
https://github.com/yhirose/cpp-httplib.git
synced 2026-08-24 10:57:15 +00:00
Add static-file, large-body and TLS workloads to the A/B benchmark
The harness had a single endpoint returning a 12-byte set_content() body. That is the one case where the response line, the headers and the body already share a single write(), so any change to the write path measured as noise. Comparing a gather-write branch against its merge base reported 0.993x at p = 1.000 while the same branch moved static-file throughput by a quarter and TLS throughput by nearly half in both directions. The server now also serves a large set_content() body and small and large files from a mount point, over HTTPS when a certificate is given, with --path, --large-mib and --tls selecting the combination. ab.sh now compiles the harness from the invoking worktree instead of each ref's own copy, so both refs run an identical workload and a ref that predates a harness change stays measurable. Only httplib.h varies, through -I. --timeout is exposed because bombardier's 2s default aborts large TLS responses, which then fails the non-2xx check.
This commit is contained in:
@@ -3,7 +3,25 @@
|
|||||||
# A/B throughput comparison between two git refs.
|
# A/B throughput comparison between two git refs.
|
||||||
#
|
#
|
||||||
# Usage: ./ab.sh [--base REF] [--head REF] [--rounds N] [--duration S]
|
# Usage: ./ab.sh [--base REF] [--head REF] [--rounds N] [--duration S]
|
||||||
# [--connections N] [--threads N]
|
# [--connections N] [--threads N] [--path PATH] [--tls]
|
||||||
|
# [--large-mib N] [--timeout S]
|
||||||
|
#
|
||||||
|
# --path selects the workload. The harness serves:
|
||||||
|
# / small body via set_content(); the response line, the
|
||||||
|
# headers and the body already share a single write(), so
|
||||||
|
# this is the least sensitive case
|
||||||
|
# /large large body via set_content()
|
||||||
|
# /static/small.js 1 KiB file from a mount point, where the headers and the
|
||||||
|
# body are two separate writes
|
||||||
|
# /static/large.bin same, with the body large enough to dominate
|
||||||
|
#
|
||||||
|
# --large-mib sizes the two large workloads (default 1).
|
||||||
|
#
|
||||||
|
# --tls runs the same workload over HTTPS, which writes through
|
||||||
|
# SSLSocketStream instead of SocketStream.
|
||||||
|
#
|
||||||
|
# --timeout is bombardier's per-request timeout. Its 2s default aborts large
|
||||||
|
# TLS responses, and the run then fails on the non-2xx check.
|
||||||
#
|
#
|
||||||
# Absolute numbers from a single run are meaningless: on a quiet 8-core laptop
|
# Absolute numbers from a single run are meaningless: on a quiet 8-core laptop
|
||||||
# the same binary varies by +/-20% run to run, and shared CI runners are worse.
|
# the same binary varies by +/-20% run to run, and shared CI runners are worse.
|
||||||
@@ -21,6 +39,10 @@ DURATION="5s"
|
|||||||
CONNECTIONS=10
|
CONNECTIONS=10
|
||||||
THREADS=""
|
THREADS=""
|
||||||
PORT=8080
|
PORT=8080
|
||||||
|
REQ_PATH="/"
|
||||||
|
TLS=0
|
||||||
|
LARGE_MIB=1
|
||||||
|
TIMEOUT="30s"
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
@@ -30,6 +52,10 @@ while [ $# -gt 0 ]; do
|
|||||||
--duration) DURATION="$2"; shift 2 ;;
|
--duration) DURATION="$2"; shift 2 ;;
|
||||||
--connections) CONNECTIONS="$2"; shift 2 ;;
|
--connections) CONNECTIONS="$2"; shift 2 ;;
|
||||||
--threads) THREADS="$2"; shift 2 ;;
|
--threads) THREADS="$2"; shift 2 ;;
|
||||||
|
--path) REQ_PATH="$2"; shift 2 ;;
|
||||||
|
--large-mib) LARGE_MIB="$2"; shift 2 ;;
|
||||||
|
--timeout) TIMEOUT="$2"; shift 2 ;;
|
||||||
|
--tls) TLS=1; shift ;;
|
||||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
@@ -62,6 +88,7 @@ HEAD_SHA=$(git -C "$REPO_ROOT" rev-parse --short "$HEAD_REF")
|
|||||||
echo "==> base: $BASE_REF ($BASE_SHA)"
|
echo "==> base: $BASE_REF ($BASE_SHA)"
|
||||||
echo "==> head: $HEAD_REF ($HEAD_SHA)"
|
echo "==> head: $HEAD_REF ($HEAD_SHA)"
|
||||||
echo "==> rounds=$ROUNDS duration=$DURATION connections=$CONNECTIONS threads=$THREADS"
|
echo "==> rounds=$ROUNDS duration=$DURATION connections=$CONNECTIONS threads=$THREADS"
|
||||||
|
echo "==> path=$REQ_PATH tls=$TLS large=${LARGE_MIB}MiB"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [ "$BASE_SHA" = "$HEAD_SHA" ]; then
|
if [ "$BASE_SHA" = "$HEAD_SHA" ]; then
|
||||||
@@ -69,18 +96,49 @@ if [ "$BASE_SHA" = "$HEAD_SHA" ]; then
|
|||||||
echo ""
|
echo ""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# --- Toolchain bits that depend on --tls ---
|
||||||
|
SCHEME="http"
|
||||||
|
INSECURE=""
|
||||||
|
TLS_CXXFLAGS=""
|
||||||
|
TLS_LDFLAGS=""
|
||||||
|
TLS_ARGS=""
|
||||||
|
if [ "$TLS" = "1" ]; then
|
||||||
|
SCHEME="https"
|
||||||
|
INSECURE="-k"
|
||||||
|
TLS_CXXFLAGS="-DCPPHTTPLIB_OPENSSL_SUPPORT"
|
||||||
|
TLS_LDFLAGS="-lssl -lcrypto"
|
||||||
|
if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists openssl; then
|
||||||
|
TLS_CXXFLAGS="$TLS_CXXFLAGS $(pkg-config --cflags openssl)"
|
||||||
|
TLS_LDFLAGS="$(pkg-config --libs openssl)"
|
||||||
|
elif command -v brew >/dev/null 2>&1 && brew --prefix openssl >/dev/null 2>&1; then
|
||||||
|
OPENSSL_PREFIX=$(brew --prefix openssl)
|
||||||
|
TLS_CXXFLAGS="$TLS_CXXFLAGS -I$OPENSSL_PREFIX/include"
|
||||||
|
TLS_LDFLAGS="-L$OPENSSL_PREFIX/lib -lssl -lcrypto"
|
||||||
|
fi
|
||||||
|
if [ "$(uname -s)" = "Darwin" ]; then
|
||||||
|
TLS_LDFLAGS="$TLS_LDFLAGS -framework CoreFoundation -framework Security"
|
||||||
|
fi
|
||||||
|
TLS_ARGS="--cert $REPO_ROOT/test/cert.pem --key $REPO_ROOT/test/key.pem"
|
||||||
|
for f in "$REPO_ROOT/test/cert.pem" "$REPO_ROOT/test/key.pem"; do
|
||||||
|
[ -f "$f" ] || { echo "Error: $f not found" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
# --- Build both refs ---
|
# --- Build both refs ---
|
||||||
|
# The harness source always comes from the invoking worktree, so both refs run
|
||||||
|
# an identical workload and a ref that predates a harness change stays
|
||||||
|
# measurable. Only httplib.h varies, through -I.
|
||||||
|
HARNESS="$REPO_ROOT/benchmark/cpp-httplib/main.cpp"
|
||||||
|
[ -f "$HARNESS" ] || { echo "Error: $HARNESS not found" >&2; exit 1; }
|
||||||
|
|
||||||
build() {
|
build() {
|
||||||
local name=$1 ref=$2
|
local name=$1 ref=$2
|
||||||
git -C "$REPO_ROOT" worktree add --detach --quiet "$WORKDIR/$name" "$ref"
|
git -C "$REPO_ROOT" worktree add --detach --quiet "$WORKDIR/$name" "$ref"
|
||||||
if [ ! -f "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" ]; then
|
|
||||||
echo "Error: benchmark/cpp-httplib/main.cpp missing in $ref" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
"$CXX" -o "$WORKDIR/$name/server-ab" -O2 -std=c++11 \
|
"$CXX" -o "$WORKDIR/$name/server-ab" -O2 -std=c++11 \
|
||||||
-I"$WORKDIR/$name" \
|
-I"$WORKDIR/$name" \
|
||||||
-DCPPHTTPLIB_THREAD_POOL_COUNT="$THREADS" \
|
-DCPPHTTPLIB_THREAD_POOL_COUNT="$THREADS" \
|
||||||
"$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" -lpthread
|
$TLS_CXXFLAGS \
|
||||||
|
"$HARNESS" -lpthread $TLS_LDFLAGS
|
||||||
}
|
}
|
||||||
|
|
||||||
echo "==> Building..."
|
echo "==> Building..."
|
||||||
@@ -92,7 +150,8 @@ measure() {
|
|||||||
local name=$1
|
local name=$1
|
||||||
local json rc
|
local json rc
|
||||||
|
|
||||||
"$WORKDIR/$name/server-ab" >/dev/null 2>&1 &
|
"$WORKDIR/$name/server-ab" --port "$PORT" --dir "$WORKDIR/$name-www" \
|
||||||
|
--large-mib "$LARGE_MIB" $TLS_ARGS >/dev/null 2>&1 &
|
||||||
local pid=$!
|
local pid=$!
|
||||||
|
|
||||||
# Wait for the listener (no dependency on nc)
|
# Wait for the listener (no dependency on nc)
|
||||||
@@ -103,8 +162,8 @@ measure() {
|
|||||||
done
|
done
|
||||||
|
|
||||||
set +e
|
set +e
|
||||||
json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -o json -p r \
|
json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -t "$TIMEOUT" -o json -p r $INSECURE \
|
||||||
"http://127.0.0.1:$PORT/" 2>/dev/null)
|
"$SCHEME://127.0.0.1:$PORT$REQ_PATH" 2>/dev/null)
|
||||||
rc=$?
|
rc=$?
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,139 @@
|
|||||||
#include "httplib.h"
|
#include "httplib.h"
|
||||||
|
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <cstring>
|
||||||
|
#include <fstream>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#ifndef _WIN32
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
using namespace httplib;
|
using namespace httplib;
|
||||||
|
|
||||||
int main() {
|
namespace {
|
||||||
Server svr;
|
|
||||||
|
|
||||||
|
// The workloads differ in which part of the write path they exercise:
|
||||||
|
//
|
||||||
|
// / small body set through set_content(); the response line,
|
||||||
|
// the headers and the body already share a single write()
|
||||||
|
// /large large body set through set_content(); the body is copied
|
||||||
|
// into the header buffer before that single write().
|
||||||
|
// --large-mib sets its size (and large.bin's).
|
||||||
|
// /static/small.js small file served from a mount point, where the headers
|
||||||
|
// and the body are two separate writes
|
||||||
|
// /static/large.bin same, with the body large enough to dominate
|
||||||
|
//
|
||||||
|
// Bodies are generated at startup so the repository carries no fixtures.
|
||||||
|
|
||||||
|
const size_t SMALL_SIZE = 1024;
|
||||||
|
const size_t LARGE_SIZE_DEFAULT_MIB = 1;
|
||||||
|
|
||||||
|
std::string filler(size_t n) {
|
||||||
|
std::string s;
|
||||||
|
s.reserve(n);
|
||||||
|
while (s.size() < n) {
|
||||||
|
s += "0123456789abcdef";
|
||||||
|
}
|
||||||
|
s.resize(n);
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool write_file(const std::string &path, const std::string &content) {
|
||||||
|
std::ofstream f(path.c_str(), std::ios::binary);
|
||||||
|
f.write(content.data(), static_cast<std::streamsize>(content.size()));
|
||||||
|
return f.good();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string default_dir() {
|
||||||
|
const char *tmp = std::getenv("TMPDIR");
|
||||||
|
std::string base = tmp && *tmp ? tmp : "/tmp";
|
||||||
|
if (!base.empty() && base[base.size() - 1] == '/') {
|
||||||
|
base.erase(base.size() - 1);
|
||||||
|
}
|
||||||
|
return base + "/cpp-httplib-bench";
|
||||||
|
}
|
||||||
|
|
||||||
|
bool make_dir(const std::string &path) {
|
||||||
|
#ifdef _WIN32
|
||||||
|
return _mkdir(path.c_str()) == 0 || errno == EEXIST;
|
||||||
|
#else
|
||||||
|
return ::mkdir(path.c_str(), 0755) == 0 || errno == EEXIST;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void setup(Server &svr, const std::string &large, const std::string &dir) {
|
||||||
svr.Get("/", [](const Request &, Response &res) {
|
svr.Get("/", [](const Request &, Response &res) {
|
||||||
res.set_content("Hello World!", "text/plain");
|
res.set_content("Hello World!", "text/plain");
|
||||||
});
|
});
|
||||||
|
|
||||||
svr.listen("0.0.0.0", 8080);
|
svr.Get("/large", [&large](const Request &, Response &res) {
|
||||||
|
res.set_content(large, "application/octet-stream");
|
||||||
|
});
|
||||||
|
|
||||||
|
svr.set_mount_point("/static", dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
int port = 8080;
|
||||||
|
std::string dir = default_dir();
|
||||||
|
std::string cert;
|
||||||
|
std::string key;
|
||||||
|
size_t large_mib = LARGE_SIZE_DEFAULT_MIB;
|
||||||
|
|
||||||
|
for (int i = 1; i < argc; i++) {
|
||||||
|
auto last = i + 1 < argc;
|
||||||
|
if (!std::strcmp(argv[i], "--port") && last) {
|
||||||
|
port = std::atoi(argv[++i]);
|
||||||
|
} else if (!std::strcmp(argv[i], "--large-mib") && last) {
|
||||||
|
large_mib = static_cast<size_t>(std::atoi(argv[++i]));
|
||||||
|
} else if (!std::strcmp(argv[i], "--dir") && last) {
|
||||||
|
dir = argv[++i];
|
||||||
|
} else if (!std::strcmp(argv[i], "--cert") && last) {
|
||||||
|
cert = argv[++i];
|
||||||
|
} else if (!std::strcmp(argv[i], "--key") && last) {
|
||||||
|
key = argv[++i];
|
||||||
|
} else {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"usage: %s [--port N] [--dir PATH] [--large-mib N]"
|
||||||
|
" [--cert PATH --key PATH]\n",
|
||||||
|
argv[0]);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!make_dir(dir)) {
|
||||||
|
std::fprintf(stderr, "cannot create %s\n", dir.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto large = filler(large_mib * 1024 * 1024);
|
||||||
|
if (!write_file(dir + "/small.js", filler(SMALL_SIZE)) ||
|
||||||
|
!write_file(dir + "/large.bin", large)) {
|
||||||
|
std::fprintf(stderr, "cannot write fixtures under %s\n", dir.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!cert.empty()) {
|
||||||
|
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
|
||||||
|
SSLServer svr(cert.c_str(), key.c_str());
|
||||||
|
if (!svr.is_valid()) {
|
||||||
|
std::fprintf(stderr, "cannot load %s / %s\n", cert.c_str(), key.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
setup(svr, large, dir);
|
||||||
|
svr.listen("0.0.0.0", port);
|
||||||
|
return 0;
|
||||||
|
#else
|
||||||
|
std::fprintf(stderr, "built without CPPHTTPLIB_OPENSSL_SUPPORT\n");
|
||||||
|
return 1;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
Server svr;
|
||||||
|
setup(svr, large, dir);
|
||||||
|
svr.listen("0.0.0.0", port);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user