diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9bac545..69cc8d7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -228,7 +228,7 @@ jobs: for ($i = 0; $i -lt $shards; $i++) { $log = "shard_${i}.log" $procs += Start-Process -FilePath ./Release/httplib-test.exe ` - -ArgumentList "--gtest_color=yes","--gtest_filter=${{ github.event.inputs.gtest_filter || '*' }}-*BenchmarkTest*" ` + -ArgumentList "--gtest_color=yes","--gtest_filter=${{ github.event.inputs.gtest_filter || '*' }}" ` -NoNewWindow -PassThru -RedirectStandardOutput $log -RedirectStandardError "${log}.err" ` -Environment @{ GTEST_TOTAL_SHARDS="$shards"; GTEST_SHARD_INDEX="$i" } } @@ -248,9 +248,6 @@ jobs: } if ($failed) { exit 1 } Write-Host "All shards passed." - - name: Run benchmark tests with retry ${{ matrix.config.name }} - if: ${{ matrix.config.run_tests }} - run: ctest --output-on-failure --test-dir build -C Release -R "BenchmarkTest" --repeat until-pass:5 env: VCPKG_ROOT: "C:/vcpkg" diff --git a/.github/workflows/test_benchmark.yaml b/.github/workflows/test_benchmark.yaml new file mode 100644 index 0000000..e2c0b01 --- /dev/null +++ b/.github/workflows/test_benchmark.yaml @@ -0,0 +1,79 @@ +name: benchmark + +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + ubuntu: + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + steps: + - name: checkout + uses: actions/checkout@v4 + - name: build and run + run: cd test && make test_benchmark && ./test_benchmark + + macos: + runs-on: macos-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + steps: + - name: checkout + uses: actions/checkout@v4 + - name: build and run + run: cd test && make test_benchmark && ./test_benchmark + + windows: + runs-on: windows-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + steps: + - name: Prepare Git for Checkout on Windows + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: checkout + uses: actions/checkout@v4 + - name: Export GitHub Actions cache environment variables + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + - name: Cache vcpkg packages + id: vcpkg-cache + uses: actions/cache@v4 + with: + path: C:/vcpkg/installed + key: vcpkg-installed-windows-gtest + - name: Install vcpkg dependencies + if: steps.vcpkg-cache.outputs.cache-hit != 'true' + run: vcpkg install gtest + - name: Configure and build + shell: pwsh + run: | + $cmake_content = @" + cmake_minimum_required(VERSION 3.14) + project(httplib-benchmark CXX) + find_package(GTest REQUIRED) + add_executable(httplib-benchmark test/test_benchmark.cc) + target_include_directories(httplib-benchmark PRIVATE .) + target_link_libraries(httplib-benchmark PRIVATE GTest::gtest_main) + target_compile_options(httplib-benchmark PRIVATE "$<$:/utf-8>") + "@ + New-Item -ItemType Directory -Force -Path build_bench/test | Out-Null + Set-Content -Path build_bench/CMakeLists.txt -Value $cmake_content + Copy-Item -Path httplib.h -Destination build_bench/ + Copy-Item -Path test/test_benchmark.cc -Destination build_bench/test/ + cmake -B build_bench/build -S build_bench ` + -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" + cmake --build build_bench/build --config Release + - name: Run with retry + run: ctest --output-on-failure --test-dir build_bench/build -C Release --repeat until-pass:5 + env: + VCPKG_ROOT: "C:/vcpkg" + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" diff --git a/.gitignore b/.gitignore index 98e3153..9dca1da 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ test/test_split_wolfssl test/test_split_no_tls test/test_websocket_heartbeat test/test_thread_pool +test/test_benchmark test/test.xcodeproj/xcuser* test/test.xcodeproj/*/xcuser* test/*.o diff --git a/justfile b/justfile index 61f0692..6a2a7b4 100644 --- a/justfile +++ b/justfile @@ -36,6 +36,7 @@ others: @(cd test && make fuzz_test) @(cd test && make test_websocket_heartbeat && ./test_websocket_heartbeat) @(cd test && make test_thread_pool && ./test_thread_pool) + @(cd test && make test_benchmark && ./test_benchmark) build: @(cd test && make test_split) diff --git a/test/Makefile b/test/Makefile index 9021a9c..40b2088 100644 --- a/test/Makefile +++ b/test/Makefile @@ -219,6 +219,16 @@ style_check: $(STYLE_CHECK_FILES) echo "All files are properly formatted."; \ fi +BENCHMARK_LIBS = -lpthread +ifneq ($(OS), Windows_NT) + ifeq ($(shell uname -s), Darwin) + BENCHMARK_LIBS += -framework CoreFoundation -framework CFNetwork + endif +endif + +test_benchmark : test_benchmark.cc ../httplib.h Makefile + $(CXX) -o $@ -I.. $(CXXFLAGS) test_benchmark.cc gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(BENCHMARK_LIBS) + test_websocket_heartbeat : test_websocket_heartbeat.cc ../httplib.h Makefile $(CXX) -o $@ -I.. $(CXXFLAGS) test_websocket_heartbeat.cc $(TEST_ARGS) @file $@ @@ -254,5 +264,5 @@ cert.pem: ./gen-certs.sh clean: - rm -rf test test_split test_mbedtls test_split_mbedtls test_wolfssl test_split_wolfssl test_no_tls, test_split_no_tls test_proxy test_proxy_mbedtls test_proxy_wolfssl server_fuzzer *.pem *.0 *.o *.1 *.srl httplib.h httplib.cc _build* *.dSYM *_shard_*.log cpp-httplib + rm -rf test test_split test_mbedtls test_split_mbedtls test_wolfssl test_split_wolfssl test_no_tls, test_split_no_tls test_proxy test_proxy_mbedtls test_proxy_wolfssl test_benchmark server_fuzzer *.pem *.0 *.o *.1 *.srl httplib.h httplib.cc _build* *.dSYM *_shard_*.log cpp-httplib diff --git a/test/test.cc b/test/test.cc index 1c15bc1..776d704 100644 --- a/test/test.cc +++ b/test/test.cc @@ -102,71 +102,6 @@ static void read_file(const std::string &path, std::string &out) { fs.read(&out[0], static_cast(size)); } -void performance_test(const char *host) { - Server svr; - - svr.Get("/benchmark", [&](const Request & /*req*/, Response &res) { - res.set_content("Benchmark Response", "text/plain"); - }); - - auto listen_thread = std::thread([&]() { svr.listen(host, PORT); }); - auto se = detail::scope_exit([&] { - svr.stop(); - listen_thread.join(); - ASSERT_FALSE(svr.is_running()); - }); - - svr.wait_until_ready(); - - Client cli(host, PORT); - - // Warm-up request to establish connection and resolve DNS - auto warmup_res = cli.Get("/benchmark"); - ASSERT_TRUE(warmup_res); // Ensure server is responding correctly - - // Run multiple trials and collect timings - const int num_trials = 20; - std::vector timings; - timings.reserve(num_trials); - - for (int i = 0; i < num_trials; i++) { - auto start = std::chrono::high_resolution_clock::now(); - auto res = cli.Get("/benchmark"); - auto end = std::chrono::high_resolution_clock::now(); - - auto elapsed = - std::chrono::duration_cast(end - start) - .count(); - - // Assertions after timing measurement to avoid overhead - ASSERT_TRUE(res); - EXPECT_EQ(StatusCode::OK_200, res->status); - - timings.push_back(elapsed); - } - - // Calculate 25th percentile (lower quartile) - std::sort(timings.begin(), timings.end()); - auto p25 = timings[num_trials / 4]; - - // Format timings for output - std::ostringstream timings_str; - timings_str << "["; - for (size_t i = 0; i < timings.size(); i++) { - if (i > 0) timings_str << ", "; - timings_str << timings[i]; - } - timings_str << "]"; - - // Localhost HTTP GET should be fast even in CI environments - EXPECT_LE(p25, 5) << "25th percentile performance is too slow: " << p25 - << "ms (Issue #1777). Timings: " << timings_str.str(); -} - -TEST(BenchmarkTest, localhost) { performance_test("localhost"); } - -TEST(BenchmarkTest, v6) { performance_test("::1"); } - class UnixSocketTest : public ::testing::Test { protected: void TearDown() override { std::remove(pathname_.c_str()); } diff --git a/test/test_benchmark.cc b/test/test_benchmark.cc new file mode 100644 index 0000000..fb7d754 --- /dev/null +++ b/test/test_benchmark.cc @@ -0,0 +1,78 @@ +#include + +#include + +#include +#include +#include +#include +#include + +using namespace httplib; + +static const int PORT = 11134; + +static void performance_test(const char *host) { + Server svr; + + svr.Get("/benchmark", [&](const Request & /*req*/, Response &res) { + res.set_content("Benchmark Response", "text/plain"); + }); + + auto listen_thread = std::thread([&]() { svr.listen(host, PORT); }); + auto se = detail::scope_exit([&] { + svr.stop(); + listen_thread.join(); + ASSERT_FALSE(svr.is_running()); + }); + + svr.wait_until_ready(); + + Client cli(host, PORT); + + // Warm-up request to establish connection and resolve DNS + auto warmup_res = cli.Get("/benchmark"); + ASSERT_TRUE(warmup_res); // Ensure server is responding correctly + + // Run multiple trials and collect timings + const int num_trials = 20; + std::vector timings; + timings.reserve(num_trials); + + for (int i = 0; i < num_trials; i++) { + auto start = std::chrono::high_resolution_clock::now(); + auto res = cli.Get("/benchmark"); + auto end = std::chrono::high_resolution_clock::now(); + + auto elapsed = + std::chrono::duration_cast(end - start) + .count(); + + // Assertions after timing measurement to avoid overhead + ASSERT_TRUE(res); + EXPECT_EQ(StatusCode::OK_200, res->status); + + timings.push_back(elapsed); + } + + // Calculate 25th percentile (lower quartile) + std::sort(timings.begin(), timings.end()); + auto p25 = timings[num_trials / 4]; + + // Format timings for output + std::ostringstream timings_str; + timings_str << "["; + for (size_t i = 0; i < timings.size(); i++) { + if (i > 0) timings_str << ", "; + timings_str << timings[i]; + } + timings_str << "]"; + + // Localhost HTTP GET should be fast even in CI environments + EXPECT_LE(p25, 5) << "25th percentile performance is too slow: " << p25 + << "ms (Issue #1777). Timings: " << timings_str.str(); +} + +TEST(BenchmarkTest, localhost) { performance_test("localhost"); } + +TEST(BenchmarkTest, v6) { performance_test("::1"); }