Stop publishing runs whose responses were mostly errors

.gitignore

@@ -28,3 +28,7 @@ node_modules/
 
 # macOS Finder metadata
 .DS_Store
+
+# Python bytecode (generated when the site-data scripts are imported/run)
+__pycache__/
+*.py[cod]

scripts/benchmark.sh

@@ -448,6 +448,22 @@ save_result() {
 EOF
     info "saved results/$profile/$CONNS/${FRAMEWORK}.json"
 
+    # rebuild_site_data.py refuses to publish a run whose responses were mostly
+    # errors — rps counts a 500 exactly like a 200. The row and the container
+    # log are still written above, since they're the post-mortem evidence, but
+    # say so here: otherwise a clean-looking --save implies a result that will
+    # never reach the board. Threshold shared via HTTPARENA_MAX_ERROR_PCT.
+    local ok_count err_count
+    ok_count=$((  ${BEST_M[status_2xx]:-0} + ${BEST_M[status_3xx]:-0} ))
+    err_count=$(( ${BEST_M[status_4xx]:-0} + ${BEST_M[status_5xx]:-0} ))
+    if [ "$ok_count" -eq 0 ]; then
+        warn "$profile/$CONNS: no successful responses ($err_count errors) — will not be published"
+    elif awk -v e="$err_count" -v n="$((ok_count + err_count))" \
+             -v thr="${HTTPARENA_MAX_ERROR_PCT:-5}" 'BEGIN{exit !(e/n*100 > thr)}'; then
+        warn "$profile/$CONNS: $(awk -v e="$err_count" -v n="$((ok_count + err_count))" \
+             'BEGIN{printf "%.1f", e/n*100}')% non-2xx/3xx ($err_count of $((ok_count + err_count))) — will not be published"
+    fi
+
     # Persist container logs alongside results for post-mortem.
     local log_dir="$ROOT_DIR/site/static/logs/$profile/$CONNS"
     mkdir -p "$log_dir"

scripts/rebuild_site_data.py

@@ -22,6 +22,32 @@
 from pathlib import Path
 
 
+# A run whose responses were overwhelmingly errors is not a measurement. rps
+# counts a 500 exactly like a 200, so a framework that collapses under load
+# can outrank one that serves every request — php-fpm published 294k rps on
+# baseline-h2 with 99.2% 5xx. Rows above this error rate are dropped rather
+# than ranked. Override for a one-off rebuild with HTTPARENA_MAX_ERROR_PCT.
+MAX_ERROR_PCT = float(os.environ.get("HTTPARENA_MAX_ERROR_PCT", "5"))
+
+
+def failure_reason(entry: dict) -> str | None:
+    """Why this result row isn't publishable, or None if it is.
+
+    Every load generator in scripts/lib/tools/ reports status counts, so a row
+    whose 2xx+3xx is zero got nothing back at all — a crash, a hang, or a
+    refused connection. Publishing its rps=0 states "we measured zero
+    throughput", which ranks the framework last on a test that never ran.
+    """
+    ok = entry.get("status_2xx", 0) + entry.get("status_3xx", 0)
+    err = entry.get("status_4xx", 0) + entry.get("status_5xx", 0)
+    if ok == 0:
+        return "no successful responses"
+    pct = err / (ok + err) * 100
+    if pct > MAX_ERROR_PCT:
+        return f"{pct:.1f}% non-2xx/3xx"
+    return None
+
+
 def rebuild_frameworks_json(root: Path, site_data: Path) -> None:
     """Aggregate every frameworks/*/meta.json into site/data/frameworks.json.
 
@@ -135,6 +161,17 @@ def merge_results(results_dir: Path, site_data: Path) -> None:
                 if stale:
                     print(f"[purged stale] {data_file.name}: {stale}", file=sys.stderr)
                 final = [e for e in final if e.get("framework", "") in valid_names]
+
+            # Applied to the merged list, not just this run's batch, so a bad
+            # row already sitting in site/data is dropped on the next rebuild
+            # rather than living on because its framework wasn't re-run.
+            dropped = [(e.get("framework", ""), failure_reason(e)) for e in final]
+            dropped = [(name, why) for name, why in dropped if why]
+            if dropped:
+                detail = ", ".join(f"{name} ({why})" for name, why in dropped)
+                print(f"[dropped failed] {data_file.name}: {detail}", file=sys.stderr)
+                final = [e for e in final if not failure_reason(e)]
+
             data_file.write_text(json.dumps(final, indent=2))
             print(f"[updated] {data_file}")
 

site/content/docs/scoring/composite-score.md

@@ -34,6 +34,12 @@ Summing instead of averaging means the composite scales with the number of score
 
 Frameworks that don't participate in a scored profile receive 0 for that profile, which lowers their composite by the full 100-point ceiling of that profile.
 
+### Failed runs are not scored
+
+A run whose responses were mostly errors is not a measurement: rps counts a 500 exactly like a 200, so a framework that collapses under load would otherwise outrank one that serves every request. When the site data is built, a result is dropped rather than ranked if it has **no successful responses at all** (a crash, a hang, or a refused connection — its rps of 0 would read as a measured zero) or if **more than 5% of its responses were non-2xx/3xx**.
+
+A dropped result is treated exactly like not participating in that profile: the framework scores 0 for it. The raw result and the container log are still kept under `results/` and `site/static/logs/` for post-mortem.
+
 ## Scored vs reference-only profiles
 
 Not all profiles count toward the composite score. Profiles marked as **scored** contribute to the composite. Reference-only profiles (marked with **\***) are displayed for comparison but do not affect the ranking.

site/data/api-16-1024.json

@@ -1011,32 +1011,6 @@
     "tpl_static": 0,
     "tpl_async_db": 388837
   },
-  {
-    "framework": "rage",
-    "language": "Ruby",
-    "rps": 9825,
-    "avg_latency": "84.56ms",
-    "p99_latency": "1.78s",
-    "cpu": "749.7%",
-    "memory": "6.0GiB",
-    "connections": 1024,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 1,
-    "bandwidth": "47.17MB/s",
-    "input_bw": "566.09KB/s",
-    "reconnects": 34413,
-    "status_2xx": 147382,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 26433,
-    "tpl_baseline": 62443,
-    "tpl_json": 68252,
-    "tpl_db": 0,
-    "tpl_upload": 0,
-    "tpl_static": 0,
-    "tpl_async_db": 16687
-  },
   {
     "framework": "rails",
     "language": "Ruby",

site/data/api-4-256.json

@@ -1011,32 +1011,6 @@
     "tpl_static": 0,
     "tpl_async_db": 164332
   },
-  {
-    "framework": "rage",
-    "language": "Ruby",
-    "rps": 13999,
-    "avg_latency": "16.30ms",
-    "p99_latency": "266.00ms",
-    "cpu": "407.6%",
-    "memory": "5.9GiB",
-    "connections": 256,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 1,
-    "bandwidth": "68.87MB/s",
-    "input_bw": "806.58KB/s",
-    "reconnects": 45990,
-    "status_2xx": 209997,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 20194,
-    "tpl_baseline": 85974,
-    "tpl_json": 86523,
-    "tpl_db": 0,
-    "tpl_upload": 0,
-    "tpl_static": 0,
-    "tpl_async_db": 37500
-  },
   {
     "framework": "rails",
     "language": "Ruby",

site/data/baseline-h2-1024.json

@@ -404,26 +404,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "php-fpm",
-    "language": "PHP",
-    "rps": 294864,
-    "avg_latency": "143.45ms",
-    "p99_latency": "143.45ms",
-    "cpu": "2169.8%",
-    "memory": "4.8GiB",
-    "connections": 1024,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 1,
-    "bandwidth": "60.80MB/s",
-    "input_bw": "",
-    "reconnects": 0,
-    "status_2xx": 11817,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 1462506
-  },
   {
     "framework": "pyronova",
     "language": "Python",
@@ -558,25 +538,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "symfony-spawn-franken",
-    "language": "PHP",
-    "rps": 97817,
-    "avg_latency": "465.99ms",
-    "p99_latency": "465.99ms",
-    "cpu": "4817.0%",
-    "memory": "7.1GiB",
-    "connections": 1024,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 1,
-    "bandwidth": "57.95MB/s",
-    "reconnects": 0,
-    "status_2xx": 494959,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 284226
-  },
   {
     "framework": "symfony-spawn-tas",
     "language": "PHP",

site/data/baseline-h2-256.json

@@ -558,25 +558,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "symfony-spawn-franken",
-    "language": "PHP",
-    "rps": 28243,
-    "avg_latency": "229.68ms",
-    "p99_latency": "229.68ms",
-    "cpu": "5180.8%",
-    "memory": "4.3GiB",
-    "connections": 256,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 1,
-    "bandwidth": "76.21MB/s",
-    "reconnects": 0,
-    "status_2xx": 142064,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 381252
-  },
   {
     "framework": "symfony-spawn-tas",
     "language": "PHP",

site/data/crud-4096.json

@@ -119,25 +119,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "fasthttp",
-    "language": "V",
-    "rps": 0,
-    "avg_latency": "0us",
-    "p99_latency": "0us",
-    "cpu": "0.0%",
-    "memory": "0MiB",
-    "connections": 4096,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 1,
-    "bandwidth": "4B/s",
-    "reconnects": 14506257,
-    "status_2xx": 0,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 0
-  },
   {
     "framework": "fishcake",
     "language": "Kotlin",

site/data/pipelined-4096.json

@@ -18,25 +18,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "apache",
-    "language": "C",
-    "rps": 0,
-    "avg_latency": "26.50ms",
-    "p99_latency": "250.40ms",
-    "cpu": "2043.6%",
-    "memory": "136MiB",
-    "connections": 4096,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 16,
-    "bandwidth": "136.05MB/s",
-    "reconnects": 14,
-    "status_2xx": 0,
-    "status_3xx": 0,
-    "status_4xx": 1858306,
-    "status_5xx": 0
-  },
   {
     "framework": "aspnet-minimal",
     "language": "C#",
@@ -171,25 +152,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "caddy",
-    "language": "Go",
-    "rps": 0,
-    "avg_latency": "47.40ms",
-    "p99_latency": "1.56s",
-    "cpu": "4491.3%",
-    "memory": "362MiB",
-    "connections": 4096,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 16,
-    "bandwidth": "105.93MB/s",
-    "reconnects": 0,
-    "status_2xx": 0,
-    "status_3xx": 0,
-    "status_4xx": 3779515,
-    "status_5xx": 0
-  },
   {
     "framework": "dart-io",
     "language": "Dart",
@@ -1059,25 +1021,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "pingora",
-    "language": "Rust",
-    "rps": 0,
-    "avg_latency": "54.31ms",
-    "p99_latency": "73.60ms",
-    "cpu": "561.1%",
-    "memory": "75MiB",
-    "connections": 4096,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 16,
-    "bandwidth": "10.01MB/s",
-    "reconnects": 372335,
-    "status_2xx": 0,
-    "status_3xx": 0,
-    "status_4xx": 372355,
-    "status_5xx": 0
-  },
   {
     "framework": "pyronova",
     "language": "Python",
@@ -1556,25 +1499,6 @@
     "status_4xx": 0,
     "status_5xx": 0
   },
-  {
-    "framework": "traefik",
-    "language": "Go",
-    "rps": 0,
-    "avg_latency": "573.13ms",
-    "p99_latency": "1.29s",
-    "cpu": "5316.4%",
-    "memory": "517MiB",
-    "connections": 4096,
-    "threads": 64,
-    "duration": "5s",
-    "pipeline": 16,
-    "bandwidth": "9.78MB/s",
-    "reconnects": 0,
-    "status_2xx": 0,
-    "status_3xx": 0,
-    "status_4xx": 0,
-    "status_5xx": 534152
-  },
   {
     "framework": "trillium",
     "language": "Rust",