fix(reports): exact Student-t p-values and a paired comparison section

src/coder_eval/reports_experiment.py

@@ -15,13 +15,14 @@
 from coder_eval.path_utils import replicate_subdir_name
 from coder_eval.reports import resolve_agent_settings
 from coder_eval.reports_stats import (
+    VariantSeries,
     bootstrap_mean_ci,
-    cohens_d,
+    collect_variant_series,
     describe_prompt_config,
     fmt_mean_sd,
     fmt_p,
     load_variant_eval_results,
-    paired_bootstrap_diff_ci,
+    paired_comparison,
     stddev,
     welch_t_test,
     wilson_interval,
@@ -216,27 +217,6 @@ def _prompt_config_lines(result: ExperimentResult, experiment: ExperimentDefinit
             lines.append(f"- **{vid}**: {desc}")
         return lines
 
-    @staticmethod
-    def _collect_variant_series(
-        result: ExperimentResult,
-    ) -> tuple[dict[str, list[float]], dict[str, list[float]], dict[str, list[float]], dict[str, list[float]]]:
-        """Per-variant (scores, durations, tokens, assistant-turns) series collected
-        across all task summaries — the raw inputs to the Aggregate Metrics stat rows."""
-        variant_scores: dict[str, list[float]] = {vid: [] for vid in result.variant_ids}
-        variant_durations: dict[str, list[float]] = {vid: [] for vid in result.variant_ids}
-        variant_tokens: dict[str, list[float]] = {vid: [] for vid in result.variant_ids}
-        variant_asst_turns: dict[str, list[float]] = {vid: [] for vid in result.variant_ids}
-
-        for ts in result.task_summaries:
-            for vr in ts.variant_results:
-                variant_scores[vr.variant_id].append(vr.weighted_score)
-                variant_durations[vr.variant_id].append(vr.duration_seconds / vr.replicate_count)
-                if vr.total_tokens is not None:
-                    variant_tokens[vr.variant_id].append(float(vr.total_tokens))
-                if vr.total_assistant_turns is not None:
-                    variant_asst_turns[vr.variant_id].append(float(vr.total_assistant_turns))
-        return variant_scores, variant_durations, variant_tokens, variant_asst_turns
-
     @staticmethod
     def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list[str]:
         """The integer-aggregate rows of the Aggregate Metrics table: Tasks Run,
@@ -312,10 +292,7 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list
     @staticmethod
     def _aggregate_stat_rows(
         result: ExperimentResult,
-        variant_scores: dict[str, list[float]],
-        variant_durations: dict[str, list[float]],
-        variant_tokens: dict[str, list[float]],
-        variant_asst_turns: dict[str, list[float]],
+        series: dict[str, VariantSeries],
         show_p_values: bool,
         vid_a: str,
         vid_b: str,
@@ -328,38 +305,38 @@ def _aggregate_stat_rows(
         # Row: Score (mean ± stddev, p-value)
         row = "| Score"
         for vid in result.variant_ids:
-            row += f" | {fmt_mean_sd(variant_scores[vid])}"
+            row += f" | {fmt_mean_sd(series[vid].scores)}"
         if show_p_values:
-            p = welch_t_test(variant_scores[vid_a], variant_scores[vid_b])
+            p = welch_t_test(series[vid_a].scores, series[vid_b].scores)
             row += f" | {fmt_p(p)}"
         lines.append(row + " |")
 
         # Row: Duration
         row = "| Avg Duration (s)"
         for vid in result.variant_ids:
-            row += f" | {fmt_mean_sd(variant_durations[vid], '.1f')}"
+            row += f" | {fmt_mean_sd(series[vid].durations, '.1f')}"
         if show_p_values:
-            p = welch_t_test(variant_durations[vid_a], variant_durations[vid_b])
+            p = welch_t_test(series[vid_a].durations, series[vid_b].durations)
             row += f" | {fmt_p(p)}"
         lines.append(row + " |")
 
         # Row: Assistant Turns (if data available)
-        if any(variant_asst_turns[vid] for vid in result.variant_ids):
+        if any(series[vid].asst_turns for vid in result.variant_ids):
             row = "| Assistant Turns"
             for vid in result.variant_ids:
-                row += f" | {fmt_mean_sd(variant_asst_turns[vid], '.1f')}"
+                row += f" | {fmt_mean_sd(series[vid].asst_turns, '.1f')}"
             if show_p_values:
-                p = welch_t_test(variant_asst_turns[vid_a], variant_asst_turns[vid_b])
+                p = welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)
                 row += f" | {fmt_p(p)}"
             lines.append(row + " |")
 
         # Row: Tokens (if data available)
-        if any(variant_tokens[vid] for vid in result.variant_ids):
+        if any(series[vid].tokens for vid in result.variant_ids):
             row = "| Tokens"
             for vid in result.variant_ids:
-                row += f" | {fmt_mean_sd(variant_tokens[vid], ',.0f')}"
+                row += f" | {fmt_mean_sd(series[vid].tokens, ',.0f')}"
             if show_p_values:
-                p = welch_t_test(variant_tokens[vid_a], variant_tokens[vid_b])
+                p = welch_t_test(series[vid_a].tokens, series[vid_b].tokens)
                 row += f" | {fmt_p(p)}"
             lines.append(row + " |")
 
@@ -371,9 +348,7 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]:
         columns). The p-value column + Welch t-tests appear only for exactly 2 variants;
         ``vid_a``/``vid_b`` stay local so the 3+-variant path never indexes them."""
         # ── Aggregate Metrics (vertical: metrics as rows, variants as columns) ──
-        variant_scores, variant_durations, variant_tokens, variant_asst_turns = (
-            ExperimentReportGenerator._collect_variant_series(result)
-        )
+        series = collect_variant_series(result)
 
         show_p_values = len(result.variant_ids) == 2
         vid_a, vid_b = (result.variant_ids[0], result.variant_ids[1]) if show_p_values else ("", "")
@@ -389,9 +364,7 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]:
 
         lines = ["", "## Aggregate Metrics", "", header, sep]
         lines += ExperimentReportGenerator._aggregate_count_rows(result, show_p_values)
-        lines += ExperimentReportGenerator._aggregate_stat_rows(
-            result, variant_scores, variant_durations, variant_tokens, variant_asst_turns, show_p_values, vid_a, vid_b
-        )
+        lines += ExperimentReportGenerator._aggregate_stat_rows(result, series, show_p_values, vid_a, vid_b)
 
         # Row: Replicates/task (if any variant ran >1 replicate)
         if any(result.variant_aggregates[vid].replicate_count > 1 for vid in result.variant_ids):
@@ -465,8 +438,7 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]:
     @staticmethod
     def _replicate_stats_lines(result: ExperimentResult) -> list[str]:
         """The ``## Replicate Statistics`` block: per-variant bootstrap-CI / Wilson
-        pass-rate table + the 2-variant paired-bootstrap comparison. Returns ``[]``
-        when no variant ran more than one replicate."""
+        pass-rate table. Returns ``[]`` when no variant ran more than one replicate."""
         # ── Replicate Statistics (only when any variant ran >1 replicate) ──
         if not any(ts.replicate_count > 1 for ts in result.task_summaries):
             return []
@@ -488,45 +460,41 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]:
                 + f" | {passes}/{len(all_scores)} [{wlo:.2f}, {whi:.2f}] |"
             )
 
-        # Paired comparison for 2-variant experiments
-        if len(result.variant_ids) == 2:
-            vid_a, vid_b = result.variant_ids[0], result.variant_ids[1]
-            per_rep_a = result.per_replicate_scores.get(vid_a, {})
-            per_rep_b = result.per_replicate_scores.get(vid_b, {})
-            common_tasks = sorted(set(per_rep_a) & set(per_rep_b))
-            a_scores: list[float] = []
-            b_scores: list[float] = []
-            skipped_tasks: list[str] = []
-            for task_id in common_tasks:
-                rep_a = per_rep_a[task_id]
-                rep_b = per_rep_b[task_id]
-                if len(rep_a) == len(rep_b):
-                    a_scores.extend(rep_a)
-                    b_scores.extend(rep_b)
-                else:
-                    skipped_tasks.append(task_id)
-            diff = paired_bootstrap_diff_ci(a_scores, b_scores)
-            if diff is not None:
-                mean_diff, d_lo, d_hi = diff
-                d_val = cohens_d(a_scores, b_scores)
-                d_str = f"{d_val:.2f}" if d_val is not None else "n/a"
-                suffix = f" ({len(skipped_tasks)} task(s) excluded: unequal replicate counts)" if skipped_tasks else ""
-                lines.extend(
-                    [
-                        "",
-                        f"**Paired mean diff ({vid_a} - {vid_b})**: {mean_diff:+.3f}"
-                        + f" [95% CI {d_lo:+.3f}, {d_hi:+.3f}], Cohen's d = {d_str}{suffix}",
-                    ]
-                )
-            else:
-                lines.extend(
-                    [
-                        "",
-                        f"*Paired statistics skipped — unequal replicate counts between {vid_a} and {vid_b}.*",
-                    ]
-                )
         return lines
 
+    @staticmethod
+    def _paired_comparison_lines(result: ExperimentResult) -> list[str]:
+        """The ``## Paired Comparison`` block for 2-variant experiments.
+
+        Renders :func:`coder_eval.reports_stats.paired_comparison`, which the HTML
+        reporter renders too. Returns ``[]`` only when the two variants have no
+        scored task in common; when they have exactly one, the section explains why
+        no paired result is shown.
+        """
+        pc = paired_comparison(result)
+        if pc is None:
+            return []
+
+        header = ["", "## Paired Comparison", ""]
+        if pc.mean_diff is None or pc.ci_low is None or pc.ci_high is None:
+            return [
+                *header,
+                f"*A paired comparison needs at least 2 tasks common to {pc.vid_a} and {pc.vid_b};"
+                + f" found {pc.task_count}.*",
+            ]
+
+        d_str = f"{pc.effect_size:.2f}" if pc.effect_size is not None else "n/a"
+        excluded = f" ({pc.excluded_count} task(s) excluded — not scored by both variants)" if pc.excluded_count else ""
+        return [
+            *header,
+            f"*Paired over the per-task mean score of {pc.task_count} task(s) common to both variants"
+            + excluded
+            + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot.*",
+            f"**Paired mean diff ({pc.vid_a} - {pc.vid_b})**: {pc.mean_diff:+.3f}"
+            + f" [95% CI {pc.ci_low:+.3f}, {pc.ci_high:+.3f}], Cohen's d = {d_str}"
+            + f", p = {fmt_p(pc.p_value)}",
+        ]
+
     @staticmethod
     def generate_experiment_report(
         result: ExperimentResult,
@@ -549,6 +517,7 @@ def generate_experiment_report(
         lines += ExperimentReportGenerator._aggregate_metrics_lines(result)
         lines += ExperimentReportGenerator._win_loss_lines(result)
         lines += ExperimentReportGenerator._replicate_stats_lines(result)
+        lines += ExperimentReportGenerator._paired_comparison_lines(result)
         return "\n".join(lines)
 
     @staticmethod

src/coder_eval/reports_html.py

@@ -1245,28 +1245,52 @@ def _experiment_prompt_config(experiment: ExperimentDefinition | None, variant_i
 """
 
 
-def _collect_variant_series(result: ExperimentResult) -> dict[str, dict[str, list[float]]]:
-    """Group per-variant numeric series (scores, durations, etc.)."""
-    series: dict[str, dict[str, list[float]]] = {
-        vid: {"scores": [], "durations": [], "asst_turns": [], "tokens": []} for vid in result.variant_ids
-    }
-    for ts in result.task_summaries:
-        for vr in ts.variant_results:
-            s = series.get(vr.variant_id)
-            if s is None:
-                continue
-            s["scores"].append(vr.weighted_score)
-            s["durations"].append(vr.duration_seconds)
-            if vr.total_tokens is not None:
-                s["tokens"].append(float(vr.total_tokens))
-            if vr.total_assistant_turns is not None:
-                s["asst_turns"].append(float(vr.total_assistant_turns))
-    return series
+def _experiment_paired_comparison(result: ExperimentResult) -> str:
+    """Render the Paired Comparison section — the HTML twin of the markdown one.
+
+    Both render the same ``reports_stats.paired_comparison`` result, so the two
+    reports can never disagree about the paired numbers.
+    """
+    from .reports_stats import fmt_p, paired_comparison
+
+    pc = paired_comparison(result)
+    if pc is None:
+        return ""
+
+    if pc.mean_diff is None or pc.ci_low is None or pc.ci_high is None:
+        body = _esc(
+            f"A paired comparison needs at least 2 tasks common to {pc.vid_a} and {pc.vid_b}; found {pc.task_count}."
+        )
+        return f"""
+<h2>Paired Comparison</h2>
+<div class="card">
+  <p class="muted">{body}</p>
+</div>
+"""
+
+    d_str = f"{pc.effect_size:.2f}" if pc.effect_size is not None else "n/a"
+    excluded = f" ({pc.excluded_count} task(s) excluded — not scored by both variants)" if pc.excluded_count else ""
+    note = _esc(
+        f"Paired over the per-task mean score of {pc.task_count} task(s) common to both variants"
+        + excluded
+        + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot."
+    )
+    headline = _esc(
+        f"Paired mean diff ({pc.vid_a} - {pc.vid_b}): {pc.mean_diff:+.3f}"
+        + f" [95% CI {pc.ci_low:+.3f}, {pc.ci_high:+.3f}], Cohen's d = {d_str}, p = {fmt_p(pc.p_value)}"
+    )
+    return f"""
+<h2>Paired Comparison</h2>
+<div class="card">
+  <p class="muted">{note}</p>
+  <p><strong>{headline}</strong></p>
+</div>
+"""
 
 
 def _experiment_aggregate_metrics(result: ExperimentResult) -> str:
     """Render the Aggregate Metrics table (with p-values when exactly 2 variants)."""
-    from .reports_stats import fmt_mean_sd, fmt_p, welch_t_test
+    from .reports_stats import collect_variant_series, fmt_mean_sd, fmt_p, welch_t_test
 
     show_p = len(result.variant_ids) == 2
     vid_a, vid_b = (result.variant_ids[0], result.variant_ids[1]) if show_p else ("", "")
@@ -1276,7 +1300,7 @@ def _experiment_aggregate_metrics(result: ExperimentResult) -> str:
         header_cells.append("<th>p-value</th>")
     header_html = "<tr>" + "".join(header_cells) + "</tr>"
 
-    series = _collect_variant_series(result)
+    series = collect_variant_series(result)
 
     def _row(label: str, values: list[str], p: str | None) -> str:
         cells = [f"<td>{_esc(label)}</td>"] + [f"<td>{_esc(v)}</td>" for v in values]
@@ -1307,31 +1331,31 @@ def _success_rate(vid: str) -> str:
     rows.append(
         _row(
             "Score",
-            [fmt_mean_sd(series[vid]["scores"]) for vid in result.variant_ids],
-            fmt_p(welch_t_test(series[vid_a]["scores"], series[vid_b]["scores"])) if show_p else None,
+            [fmt_mean_sd(series[vid].scores) for vid in result.variant_ids],
+            fmt_p(welch_t_test(series[vid_a].scores, series[vid_b].scores)) if show_p else None,
         )
     )
     rows.append(
         _row(
             "Duration (s)",
-            [fmt_mean_sd(series[vid]["durations"], ".1f") for vid in result.variant_ids],
-            fmt_p(welch_t_test(series[vid_a]["durations"], series[vid_b]["durations"])) if show_p else None,
+            [fmt_mean_sd(series[vid].durations, ".1f") for vid in result.variant_ids],
+            fmt_p(welch_t_test(series[vid_a].durations, series[vid_b].durations)) if show_p else None,
         )
     )
-    if any(series[vid]["asst_turns"] for vid in result.variant_ids):
+    if any(series[vid].asst_turns for vid in result.variant_ids):
         rows.append(
             _row(
                 "Assistant Turns",
-                [fmt_mean_sd(series[vid]["asst_turns"], ".1f") for vid in result.variant_ids],
-                fmt_p(welch_t_test(series[vid_a]["asst_turns"], series[vid_b]["asst_turns"])) if show_p else None,
+                [fmt_mean_sd(series[vid].asst_turns, ".1f") for vid in result.variant_ids],
+                fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)) if show_p else None,
             )
         )
-    if any(series[vid]["tokens"] for vid in result.variant_ids):
+    if any(series[vid].tokens for vid in result.variant_ids):
         rows.append(
             _row(
                 "Tokens",
-                [fmt_mean_sd(series[vid]["tokens"], ",.0f") for vid in result.variant_ids],
-                fmt_p(welch_t_test(series[vid_a]["tokens"], series[vid_b]["tokens"])) if show_p else None,
+                [fmt_mean_sd(series[vid].tokens, ",.0f") for vid in result.variant_ids],
+                fmt_p(welch_t_test(series[vid_a].tokens, series[vid_b].tokens)) if show_p else None,
             )
         )
 
@@ -1607,6 +1631,7 @@ def generate_experiment_html(
 """
             + _experiment_prompt_config(experiment, result.variant_ids)
             + _experiment_aggregate_metrics(result)
+            + _experiment_paired_comparison(result)
             + _experiment_win_rates(result)
             + _experiment_per_task_comparison(result)
             + _experiment_most_divergent(result)