Fix benchmark-run reporting success without measuring anything

The first run of this workflow exposed three problems.

Crow's amalgamated header includes <asio.hpp>, which no runner provides,
so the build died immediately. It compiled locally only because CPATH
happened to point at Homebrew's include directory. Install asio
explicitly, and on macOS pass its include path through CROW_CXXFLAGS.

The macos runner image has no Go, so bombardier could not be installed.
Add actions/setup-go, which also pins a known toolchain on Linux.

Worst of all, the ubuntu job reported success. `make ... | tee` returns
tee's status, so the failed build was invisible. Enable pipefail. That
alone is not enough: every recipe in benchmark/Makefile ends in `kill`,
so make still exits 0 when bombardier itself fails to run. Assert that
the expected number of "Reqs/sec" lines came out.

.github/workflows/benchmark_ab.yaml

@@ -0,0 +1,62 @@
+name: benchmark-ab
+
+# Manual A/B throughput comparison between two refs.
+#
+# This is a measurement, not a test: it never fails the build on a slow result.
+# Absolute req/s from a shared runner is meaningless on its own, so both refs
+# are built and measured alternately in the same job and only the ratio of the
+# medians is reported, with a permutation test to say whether the difference
+# stands out from the run-to-run noise.
+#
+# Non-SSL and Linux only for now.
+
+on:
+  workflow_dispatch:
+    inputs:
+      base:
+        description: "Baseline ref"
+        required: false
+        default: "origin/master"
+      head:
+        description: "Ref to compare (defaults to the ref this run was started on)"
+        required: false
+        default: ""
+      rounds:
+        description: "Measurement rounds per ref (9+ recommended; below 4 the test can never reach significance)"
+        required: false
+        default: "9"
+      duration:
+        description: "Load duration per measurement"
+        required: false
+        default: "5s"
+      connections:
+        description: "Concurrent connections"
+        required: false
+        default: "10"
+
+permissions:
+  contents: read
+
+jobs:
+  ubuntu:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: install bombardier
+        run: go install github.com/codesenberg/bombardier@latest
+
+      - name: run A/B benchmark
+        run: |
+          export PATH="$(go env GOPATH)/bin:$PATH"
+          HEAD_REF="${{ inputs.head }}"
+          if [ -z "$HEAD_REF" ]; then HEAD_REF="${{ github.sha }}"; fi
+          ./benchmark/ab.sh \
+            --base "${{ inputs.base }}" \
+            --head "$HEAD_REF" \
+            --rounds "${{ inputs.rounds }}" \
+            --duration "${{ inputs.duration }}" \
+            --connections "${{ inputs.connections }}"

.github/workflows/benchmark_run.yaml

@@ -0,0 +1,106 @@
+name: benchmark-run
+
+# Runs the committed benchmark (`just bench`) and records the numbers.
+#
+# This is a measurement, not a test: nothing here fails the build. Unlike
+# benchmark-ab, which compares two refs inside one job, this just reports the
+# absolute throughput of the current ref alongside Crow for reference.
+#
+# Absolute req/s is only meaningful against other runs on the same runner type,
+# so compare like with like when reading the history.
+#
+# Non-SSL only. Windows is excluded: benchmark/Makefile depends on `nc`, `&`
+# and `kill`, so it would need a PowerShell rewrite first.
+
+on:
+  workflow_dispatch:
+    inputs:
+      duration:
+        description: "Load duration per server"
+        required: false
+        default: "5s"
+      connections:
+        description: "Concurrent connections"
+        required: false
+        default: "10"
+      crow:
+        description: "Also benchmark Crow v1.3.1 for reference"
+        type: boolean
+        required: false
+        default: true
+
+permissions:
+  contents: read
+
+jobs:
+  bench:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      # macos runners ship without Go.
+      - name: setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      - name: install bombardier
+        run: go install github.com/codesenberg/bombardier@latest
+
+      # crow_all.h includes <asio.hpp>, which no runner has out of the box.
+      - name: install asio
+        if: ${{ inputs.crow }}
+        run: |
+          if [ "$RUNNER_OS" = "Linux" ]; then
+            sudo apt-get update && sudo apt-get install -y libasio-dev
+          else
+            brew install asio
+          fi
+
+      - name: run benchmark
+        run: |
+          # Without pipefail the `tee` below swallows a build failure and the
+          # job reports success having measured nothing.
+          set -o pipefail
+          export PATH="$(go env GOPATH)/bin:$PATH"
+          CROW_FLAGS="-std=c++17"
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            CROW_FLAGS="$CROW_FLAGS -I$(brew --prefix asio)/include"
+          fi
+          if [ "${{ inputs.crow }}" = "true" ]; then TARGET=bench-all; else TARGET=bench; fi
+          make -C benchmark "$TARGET" \
+            CROW_CXXFLAGS="$CROW_FLAGS" \
+            BENCH="bombardier -c ${{ inputs.connections }} -d ${{ inputs.duration }} localhost:8080" \
+            2>&1 | tee /tmp/bench.txt
+
+      # pipefail only catches a failed build. Each Makefile recipe ends in
+      # `kill`, so a bombardier that never ran still leaves make happy — check
+      # that the measurements are actually there.
+      - name: check results were produced
+        run: |
+          expected=1
+          if [ "${{ inputs.crow }}" = "true" ]; then expected=2; fi
+          got=$(grep -c "Reqs/sec" /tmp/bench.txt || true)
+          if [ "$got" -lt "$expected" ]; then
+            echo "::error::expected $expected benchmark result(s), found $got"
+            exit 1
+          fi
+
+      - name: record results
+        if: always()
+        run: |
+          {
+            echo "## Benchmark (${{ matrix.os }})"
+            echo ""
+            echo "- ref: \`${{ github.ref_name }}\` (${{ github.sha }})"
+            echo "- connections=${{ inputs.connections }} duration=${{ inputs.duration }}"
+            echo ""
+            echo '```'
+            cat /tmp/bench.txt
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"

benchmark/ab.sh

@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+#
+# A/B throughput comparison between two git refs.
+#
+# Usage: ./ab.sh [--base REF] [--head REF] [--rounds N] [--duration S]
+#                [--connections N] [--threads N]
+#
+# Absolute numbers from a single run are meaningless: on a quiet 8-core laptop
+# the same binary varies by +/-20% run to run, and shared CI runners are worse.
+# So both refs are built and then measured alternately in the same session, and
+# only the ratio of the medians is reported.
+#
+# Requires: bombardier, python3, g++ (or $CXX), git.
+
+set -euo pipefail
+
+BASE_REF="master"
+HEAD_REF="HEAD"
+ROUNDS=5
+DURATION="5s"
+CONNECTIONS=10
+THREADS=""
+PORT=8080
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --base) BASE_REF="$2"; shift 2 ;;
+    --head) HEAD_REF="$2"; shift 2 ;;
+    --rounds) ROUNDS="$2"; shift 2 ;;
+    --duration) DURATION="$2"; shift 2 ;;
+    --connections) CONNECTIONS="$2"; shift 2 ;;
+    --threads) THREADS="$2"; shift 2 ;;
+    *) echo "Unknown option: $1" >&2; exit 1 ;;
+  esac
+done
+
+command -v bombardier >/dev/null || { echo "Error: bombardier not found" >&2; exit 1; }
+command -v python3 >/dev/null || { echo "Error: python3 not found" >&2; exit 1; }
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+CXX=${CXX:-g++}
+
+# Default the thread pool to the core count. The committed benchmark Makefile
+# hardcodes 16, which heavily oversubscribes a 2-4 vCPU CI runner and inflates
+# the variance we are trying to see through.
+if [ -z "$THREADS" ]; then
+  THREADS=$(python3 -c 'import os; print(os.cpu_count() or 4)')
+fi
+
+WORKDIR=$(mktemp -d)
+cleanup() {
+  pkill -f "$WORKDIR/.*/server-ab" 2>/dev/null || true
+  git -C "$REPO_ROOT" worktree remove --force "$WORKDIR/base" 2>/dev/null || true
+  git -C "$REPO_ROOT" worktree remove --force "$WORKDIR/head" 2>/dev/null || true
+  rm -rf "$WORKDIR"
+}
+trap cleanup EXIT
+
+BASE_SHA=$(git -C "$REPO_ROOT" rev-parse --short "$BASE_REF")
+HEAD_SHA=$(git -C "$REPO_ROOT" rev-parse --short "$HEAD_REF")
+
+echo "==> base: $BASE_REF ($BASE_SHA)"
+echo "==> head: $HEAD_REF ($HEAD_SHA)"
+echo "==> rounds=$ROUNDS duration=$DURATION connections=$CONNECTIONS threads=$THREADS"
+echo ""
+
+if [ "$BASE_SHA" = "$HEAD_SHA" ]; then
+  echo "Note: base and head are the same commit; this measures harness noise."
+  echo ""
+fi
+
+# --- Build both refs ---
+build() {
+  local name=$1 ref=$2
+  git -C "$REPO_ROOT" worktree add --detach --quiet "$WORKDIR/$name" "$ref"
+  if [ ! -f "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" ]; then
+    echo "Error: benchmark/cpp-httplib/main.cpp missing in $ref" >&2
+    exit 1
+  fi
+  "$CXX" -o "$WORKDIR/$name/server-ab" -O2 -std=c++11 \
+    -I"$WORKDIR/$name" \
+    -DCPPHTTPLIB_THREAD_POOL_COUNT="$THREADS" \
+    "$WORKDIR/$name/benchmark/cpp-httplib/main.cpp" -lpthread
+}
+
+echo "==> Building..."
+build base "$BASE_REF"
+build head "$HEAD_REF"
+
+# --- Measure one ref once, echo rps ---
+measure() {
+  local name=$1
+  local json rc
+
+  "$WORKDIR/$name/server-ab" >/dev/null 2>&1 &
+  local pid=$!
+
+  # Wait for the listener (no dependency on nc)
+  local i
+  for i in $(seq 1 200); do
+    if (exec 3<>/dev/tcp/127.0.0.1/$PORT) 2>/dev/null; then exec 3>&- 3<&-; break; fi
+    sleep 0.05
+  done
+
+  set +e
+  json=$(bombardier -c "$CONNECTIONS" -d "$DURATION" -o json -p r \
+    "http://127.0.0.1:$PORT/" 2>/dev/null)
+  rc=$?
+  set -e
+
+  kill "$pid" 2>/dev/null || true
+  wait "$pid" 2>/dev/null || true
+
+  # Wait for the port to be released before the next run
+  for i in $(seq 1 200); do
+    if ! (exec 3<>/dev/tcp/127.0.0.1/$PORT) 2>/dev/null; then break; fi
+    exec 3>&- 3<&-
+    sleep 0.05
+  done
+
+  if [ $rc -ne 0 ] || [ -z "$json" ]; then
+    echo "Error: bombardier failed for $name" >&2
+    exit 1
+  fi
+
+  python3 -c '
+import json, sys
+r = json.load(sys.stdin)["result"]
+total = sum(r[k] for k in ("req1xx","req2xx","req3xx","req4xx","req5xx","others"))
+bad = total - r["req2xx"]
+if bad:
+    sys.stderr.write("Error: %d non-2xx/error responses\n" % bad)
+    sys.exit(1)
+print("%.1f" % (total / r["timeTakenSeconds"]))
+' <<<"$json"
+}
+
+# --- Alternate, flipping the order each round to cancel ordering bias ---
+BASE_RESULTS=()
+HEAD_RESULTS=()
+
+echo ""
+echo "==> Measuring..."
+for ((r = 1; r <= ROUNDS; r++)); do
+  if (( r % 2 == 1 )); then order=("base" "head"); else order=("head" "base"); fi
+  line="    round $r:"
+  for name in "${order[@]}"; do
+    rps=$(measure "$name")
+    if [ "$name" = "base" ]; then BASE_RESULTS+=("$rps"); else HEAD_RESULTS+=("$rps"); fi
+    line="$line  $name=$rps"
+  done
+  echo "$line"
+done
+
+# --- Report ---
+SUMMARY=$(python3 -c '
+import statistics, sys
+from itertools import combinations
+
+base = [float(x) for x in sys.argv[1].split()]
+head = [float(x) for x in sys.argv[2].split()]
+bm, hm = statistics.median(base), statistics.median(head)
+
+def spread(v):
+    return (max(v) - min(v)) / statistics.median(v) * 100
+
+def u_stat(a, b):
+    """Mann-Whitney U: number of (a, b) pairs where a > b, ties count a half."""
+    return sum((x > y) + 0.5 * (x == y) for x in a for y in b)
+
+def exact_p(a, b):
+    """Two-sided permutation p-value. A single slow round cannot swing this
+    the way a min/max spread check can."""
+    n1, n2 = len(a), len(b)
+    pooled = a + b
+    observed = abs(u_stat(a, b) - n1 * n2 / 2)
+    total = extreme = 0
+    for idx in combinations(range(n1 + n2), n1):
+        s = set(idx)
+        ga = [pooled[i] for i in idx]
+        gb = [pooled[i] for i in range(n1 + n2) if i not in s]
+        total += 1
+        if abs(u_stat(ga, gb) - n1 * n2 / 2) >= observed:
+            extreme += 1
+    return extreme / total
+
+print("| | median req/s | min | max | spread |")
+print("|---|---|---|---|---|")
+print("| base | %.0f | %.0f | %.0f | %.1f%% |" % (bm, min(base), max(base), spread(base)))
+print("| head | %.0f | %.0f | %.0f | %.1f%% |" % (hm, min(head), max(head), spread(head)))
+print("")
+print("**ratio: %.3fx** (%+.1f%%)" % (hm / bm, (hm / bm - 1) * 100))
+print("")
+
+if len(base) + len(head) > 20:
+    print("> %d rounds: skipping the permutation test (too many combinations)."
+          % len(base))
+else:
+    p = exact_p(base, head)
+    if p <= 0.05:
+        print("> Separation is consistent across rounds (permutation p = %.3f)." % p)
+    else:
+        print("> Not separated from noise (permutation p = %.3f). Inconclusive;" % p)
+        print("> raise --rounds or --duration, or run on a quieter machine.")
+    min_p = exact_p(list(range(len(base))),
+                    list(range(len(base), len(base) + len(head))))
+    if min_p > 0.05:
+        print(">")
+        print("> With %d rounds even perfect separation only reaches p = %.3f,"
+              % (len(base), min_p))
+        print("> so this test can never call a win. Use --rounds 4 or more.")
+' "${BASE_RESULTS[*]}" "${HEAD_RESULTS[*]}")
+
+echo ""
+echo "$SUMMARY"
+
+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+  {
+    echo "## Benchmark A/B"
+    echo ""
+    echo "- base: \`$BASE_REF\` ($BASE_SHA)"
+    echo "- head: \`$HEAD_REF\` ($HEAD_SHA)"
+    echo "- rounds=$ROUNDS duration=$DURATION connections=$CONNECTIONS threads=$THREADS"
+    echo ""
+    echo "$SUMMARY"
+  } >> "$GITHUB_STEP_SUMMARY"
+fi

justfile

@@ -46,6 +46,10 @@ build:
 bench:
     @(cd benchmark && make bench-all)
 
+# A/B throughput comparison between two refs (see benchmark/ab.sh --help)
+bench-ab *args:
+    @./benchmark/ab.sh {{args}}
+
 docs-serve:
     -@docs-gen serve docs-src --open