From f24e79aab94537055609c2bffcd09900aee7bcd2 Mon Sep 17 00:00:00 2001
From: yhirose <yuji.hirose.bug@gmail.com>
Date: Thu, 30 Jul 2026 20:59:12 -0400
Subject: [PATCH] Fix benchmark-run reporting success without measuring
 anything

The first run of this workflow exposed three problems.

Crow's amalgamated header includes <asio.hpp>, which no runner provides,
so the build died immediately. It compiled locally only because CPATH
happened to point at Homebrew's include directory. Install asio
explicitly, and on macOS pass its include path through CROW_CXXFLAGS.

The macos runner image has no Go, so bombardier could not be installed.
Add actions/setup-go, which also pins a known toolchain on Linux.

Worst of all, the ubuntu job reported success. `make ... | tee` returns
tee's status, so the failed build was invisible. Enable pipefail. That
alone is not enough: every recipe in benchmark/Makefile ends in `kill`,
so make still exits 0 when bombardier itself fails to run. Assert that
the expected number of "Reqs/sec" lines came out.
---
 .github/workflows/benchmark_run.yaml | 37 ++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/.github/workflows/benchmark_run.yaml b/.github/workflows/benchmark_run.yaml
index 4842e19..326f4ca 100644
--- a/.github/workflows/benchmark_run.yaml
+++ b/.github/workflows/benchmark_run.yaml
@@ -43,17 +43,54 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4
 
+      # macos runners ship without Go.
+      - name: setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
       - name: install bombardier
         run: go install github.com/codesenberg/bombardier@latest
 
+      # crow_all.h includes <asio.hpp>, which no runner has out of the box.
+      - name: install asio
+        if: ${{ inputs.crow }}
+        run: |
+          if [ "$RUNNER_OS" = "Linux" ]; then
+            sudo apt-get update && sudo apt-get install -y libasio-dev
+          else
+            brew install asio
+          fi
+
       - name: run benchmark
         run: |
+          # Without pipefail the `tee` below swallows a build failure and the
+          # job reports success having measured nothing.
+          set -o pipefail
           export PATH="$(go env GOPATH)/bin:$PATH"
+          CROW_FLAGS="-std=c++17"
+          if [ "$RUNNER_OS" = "macOS" ]; then
+            CROW_FLAGS="$CROW_FLAGS -I$(brew --prefix asio)/include"
+          fi
           if [ "${{ inputs.crow }}" = "true" ]; then TARGET=bench-all; else TARGET=bench; fi
           make -C benchmark "$TARGET" \
+            CROW_CXXFLAGS="$CROW_FLAGS" \
             BENCH="bombardier -c ${{ inputs.connections }} -d ${{ inputs.duration }} localhost:8080" \
             2>&1 | tee /tmp/bench.txt
 
+      # pipefail only catches a failed build. Each Makefile recipe ends in
+      # `kill`, so a bombardier that never ran still leaves make happy — check
+      # that the measurements are actually there.
+      - name: check results were produced
+        run: |
+          expected=1
+          if [ "${{ inputs.crow }}" = "true" ]; then expected=2; fi
+          got=$(grep -c "Reqs/sec" /tmp/bench.txt || true)
+          if [ "$got" -lt "$expected" ]; then
+            echo "::error::expected $expected benchmark result(s), found $got"
+            exit 1
+          fi
+
       - name: record results
         if: always()
         run: |