From 652e09810693c027e0fd1e6a971c39418db42e3e Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 17:47:01 -0700 Subject: [PATCH 1/9] =?UTF-8?q?feat(reports):=201/3=20=E2=80=94=20add=20re?= =?UTF-8?q?ports=5Fjunit.py=20disk-driven=20JUnit=20XML=20writer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New src/coder_eval/reports_junit.py: generate_junit_xml(run_dir) reads the run.json spine (RunSummary), optional */*/suite.json gates (SuiteRollup), and per-failed-row task.json (best-effort plain-dict) to build a JUnit XML string; write_junit_xml is the thin persist wrapper. Counts always equal emitted children; status classification goes through FinalStatus.category; all agent-derived text is scrubbed of illegal XML 1.0 chars. Production code never parses XML. Adds defusedxml (dev-only) for test-side round-trip parsing and a shared write_run_json conftest fixture reused by later phases. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 1 + src/coder_eval/reports_junit.py | 310 +++++++++++++++++++++ tests/conftest.py | 52 ++++ tests/test_reports_junit.py | 458 ++++++++++++++++++++++++++++++++ 4 files changed, 821 insertions(+) create mode 100644 src/coder_eval/reports_junit.py create mode 100644 tests/test_reports_junit.py diff --git a/pyproject.toml b/pyproject.toml index 8b074c06..03c1de2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ dev = [ "pip-audit>=2.10.0", "bandit[toml]>=1.9.4", "pre-commit>=4.5.1", + "defusedxml>=0.7.1", # test-side JUnit XML round-trip parsing (defense-in-depth; production code never parses XML) ] # Optional extra that enables UiPath-specific features: # - the `uipath` Python SDK on the host (handy for local sandbox parity with diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py new file mode 100644 index 00000000..194a6f5e --- /dev/null +++ b/src/coder_eval/reports_junit.py @@ -0,0 +1,310 @@ +"""Disk-driven JUnit XML report generation from a finalized run directory. + +Reads a run directory's on-disk contracts — ``run.json`` (required, the +``RunSummary`` spine), ``*/*/suite.json`` (optional suite gates), and per-failed +row ``task.json`` (optional, best-effort failure detail) — and produces a JUnit +XML string that CI test-report ingesters (GitHub ``mikepenz/action-junit-report``, +Azure DevOps ``PublishTestResults@2``, …) understand. + +Single code path: ``coder-eval run --junit-xml`` and ``coder-eval report -f +junit`` both call :func:`generate_junit_xml`, so the two entry points can never +drift. + +SECURITY: this module only *builds* an element tree from JSON-derived data and +serializes it — it never *parses* XML, so stdlib ``xml.etree.ElementTree`` is +safe here (XXE / billion-laughs are parser-side attacks on untrusted XML input, +which has no surface). Every string entering the tree is scrubbed of characters +outside the XML 1.0 legal set via :func:`_xml_safe`. +""" + +from __future__ import annotations + +import json +import logging +import re +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any, Literal + +from .evaluation.judge_context import truncate +from .models import FinalStatus, RunSummary, SuiteRollup + + +logger = logging.getLogger(__name__) + +# Characters outside XML 1.0's legal set. Kept as a plain (non-raw) ASCII-only +# string with doubled backslashes so this source file carries no literal astral +# or control characters. ``ElementTree`` will happily serialize control chars +# into *invalid* XML, so every agent-derived string is scrubbed before it enters +# the tree. +_ILLEGAL_XML = re.compile("[^\\x09\\x0A\\x0D\\x20-\\uD7FF\\uE000-\\uFFFD\\U00010000-\\U0010FFFF]") + +# Per-testcase failure/error body cap (chars). Agent detail dumps can be huge. +_BODY_LIMIT = 10_000 + + +def _xml_safe(text: str) -> str: + """Strip characters outside XML 1.0's legal set (ET handles ``<>&`` itself).""" + return _ILLEGAL_XML.sub("", text) + + +def _category_of(status: str) -> Literal["succeeded", "failed", "error"]: + """Map a serialized status string to a reporting category via the SSOT. + + Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); + an unknown status value (older/newer schema) falls to the error bucket + explicitly rather than via a denylist. + """ + try: + return FinalStatus(status).category + except ValueError: + return "error" + + +def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None: + """Set ``tests``/``failures``/``errors``/``skipped`` from the actual children. + + The invariant the whole writer is built around: count attributes always + equal the emitted child elements (no separately-tracked counters that can + drift). + """ + elem.set("tests", str(len(cases))) + elem.set("failures", str(sum(1 for c in cases if c.find("failure") is not None))) + elem.set("errors", str(sum(1 for c in cases if c.find("error") is not None))) + elem.set("skipped", str(sum(1 for c in cases if c.find("skipped") is not None))) + + +def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None: + """Best-effort load of a failed row's ``task.json`` as a plain dict. + + Plain-dict access (not ``EvaluationResult.model_validate``) keeps the writer + tolerant of schema skew in blob-pulled/older runs and avoids materializing + the large ``turns`` array. Any error (missing dir/file, bad JSON) → ``None``. + """ + task_id = str(row.get("task_id", "")) + replicate_index = row.get("replicate_index") + task_dir = run_dir / variant / task_id + if isinstance(replicate_index, int): + candidate = task_dir / f"{replicate_index:02d}" / "task.json" + else: + matches = sorted(task_dir.glob("*/task.json")) + candidate = matches[0] if matches else task_dir / "task.json" + try: + return json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: + """Build the failure/error body for a non-succeeded row. + + Prefers per-criterion lines from the row's ``task.json``; falls back to a + status + weighted-score line when task.json is missing/corrupt. Capped via + the shared ``truncate``. + """ + status = str(row.get("status")) + data = _load_task_json(run_dir, row, variant) + criteria = data.get("success_criteria_results") if isinstance(data, dict) else None + + lines: list[str] = [] + if isinstance(criteria, list) and criteria: + for crit in criteria: + if not isinstance(crit, dict): + continue + ctype = str(crit.get("criterion_type", "unknown")) + description = str(crit.get("description", "")) + score = crit.get("score") + threshold = crit.get("pass_threshold") + passed = isinstance(score, int | float) and isinstance(threshold, int | float) and score >= threshold + if passed: + lines.append(f"[PASS] {ctype}: {description}") + continue + score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score) + thr_str = f"{threshold:.2f}" if isinstance(threshold, int | float) else str(threshold) + lines.append(f"[FAIL] {ctype}: score {score_str} < threshold {thr_str} — {description}") + detail = crit.get("details") or crit.get("error") + if detail: + lines.append(str(detail)) + else: + weighted = row.get("weighted_score") + weighted_str = f"{weighted:.2f}" if isinstance(weighted, int | float) else str(weighted) + lines.append(f"status={status} weighted_score={weighted_str}") + + return _xml_safe(truncate("\n".join(lines), _BODY_LIMIT)) + + +def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: + """Build one ```` for a task row.""" + variant = row.get("variant_id") or "default" + task_id = str(row.get("task_id", "")) + replicate_index = row.get("replicate_index") + name = f"{task_id}[{replicate_index:02d}]" if isinstance(replicate_index, int) else task_id + + task_path = row.get("task_path") + classname = Path(task_path).stem if task_path else variant + + duration = row.get("duration") + time_str = f"{duration:.3f}" if isinstance(duration, int | float) else "0.000" + + case = ET.Element( + "testcase", + {"name": _xml_safe(name), "classname": _xml_safe(classname), "time": time_str}, + ) + + # Properties — emit only non-None values (no "None" strings in the tree). + prop_specs = [ + ("model_used", row.get("model_used")), + ("weighted_score", row.get("weighted_score")), + ("total_cost_usd", row.get("total_cost_usd")), + ("total_tokens", row.get("total_tokens")), + ("visible_turns", row.get("visible_turns")), + ] + props = [(k, v) for k, v in prop_specs if v is not None] + if props: + properties = ET.SubElement(case, "properties") + for key, value in props: + ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))}) + + category = _category_of(str(row.get("status"))) + if category == "succeeded": + return case + + status = str(row.get("status")) + try: + FinalStatus(status) + message = status + except ValueError: + message = f"unknown status: {status}" + tag = "failure" if category == "failed" else "error" + child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) + child.text = _criteria_body(row, run_dir, variant) + return case + + +def _skipped_suite(summary: RunSummary) -> ET.Element | None: + """Build the synthetic ``skipped`` testsuite from ``RunSummary.skipped_tasks``.""" + if not summary.skipped_tasks: + return None + suite = ET.Element("testsuite", {"name": "skipped"}) + cases: list[ET.Element] = [] + for entry in summary.skipped_tasks: + case = ET.SubElement( + suite, + "testcase", + {"name": _xml_safe(Path(entry.path).stem), "classname": "skipped"}, + ) + ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) + cases.append(case) + _set_counts(suite, cases) + return suite + + +def _suite_gate_suite(run_dir: Path) -> ET.Element | None: + """Build the synthetic ``suite-gates`` testsuite from ``*/*/suite.json``. + + A suite.json that fails ``SuiteRollup`` validation is skipped with a warning + (schema skew must not kill report generation). Returns ``None`` when no valid + rollup is found. + """ + rollups: list[SuiteRollup] = [] + for path in sorted(run_dir.glob("*/*/suite.json")): + try: + rollups.append(SuiteRollup.model_validate_json(path.read_text(encoding="utf-8"))) + except (OSError, ValueError) as e: + logger.warning("Skipping unparseable suite rollup %s: %s", path, e) + if not rollups: + return None + + suite = ET.Element("testsuite", {"name": "suite-gates"}) + cases: list[ET.Element] = [] + for rollup in rollups: + case = ET.SubElement( + suite, + "testcase", + {"name": _xml_safe(f"{rollup.variant_id}/{rollup.suite_id}"), "classname": "suite-gates"}, + ) + cases.append(case) + if rollup.passed: + continue + lines: list[str] = [] + for agg in rollup.criterion_aggregates: + for check in agg.threshold_checks: + if check.passed: + continue + if check.actual_value is None: + lines.append(f"{check.metric}: metric not emitted (min {check.min_value})") + else: + lines.append(f"{check.metric}: {check.actual_value} < {check.min_value}") + if agg.error: + lines.append(f"{agg.criterion_type}: {agg.error}") + failure = ET.SubElement(case, "failure", {"message": "suite thresholds not met"}) + failure.text = _xml_safe(truncate("\n".join(lines), _BODY_LIMIT)) + _set_counts(suite, cases) + return suite + + +def generate_junit_xml(run_dir: Path) -> str: + """Read ``run_dir`` and return a JUnit XML string. + + Args: + run_dir: A finalized run directory containing ``run.json``. + + Returns: + JUnit XML with an ```` declaration. + + Raises: + FileNotFoundError: When ``run.json`` is absent (not a finalized run dir). + pydantic.ValidationError: When ``run.json`` fails ``RunSummary`` validation + (the spine contract is broken; a fabricated report would be worse). + """ + run_json = run_dir / "run.json" + if not run_json.is_file(): + raise FileNotFoundError(f"No run.json in {run_dir} — not a finalized run directory (see 'coder-eval run')") + summary = RunSummary.model_validate_json(run_json.read_text(encoding="utf-8")) + + root = ET.Element("testsuites", {"name": _xml_safe(summary.run_id)}) + root.set("time", f"{summary.total_duration_seconds:.3f}") + + # Group task rows by variant, preserving first-seen order. + grouped: dict[str, list[dict[str, Any]]] = {} + for row in summary.task_results: + variant = row.get("variant_id") or "default" + grouped.setdefault(variant, []).append(row) + + all_cases: list[ET.Element] = [] + for variant, rows in grouped.items(): + suite = ET.SubElement(root, "testsuite", {"name": _xml_safe(variant)}) + cases = [_task_case(row, run_dir) for row in rows] + for case in cases: + suite.append(case) + _set_counts(suite, cases) + all_cases.extend(cases) + + skipped_suite = _skipped_suite(summary) + if skipped_suite is not None: + root.append(skipped_suite) + all_cases.extend(skipped_suite.findall("testcase")) + + gate_suite = _suite_gate_suite(run_dir) + if gate_suite is not None: + root.append(gate_suite) + all_cases.extend(gate_suite.findall("testcase")) + + _set_counts(root, all_cases) + + return ET.tostring(root, encoding="utf-8", xml_declaration=True).decode("utf-8") + + +def write_junit_xml(run_dir: Path, output_path: Path) -> Path: + """Generate JUnit XML for ``run_dir`` and write it to ``output_path``. + + Thin persist wrapper mirroring the ``build_run_summary``/``write_run_summary`` + seam. Creates parent directories as needed. + + Returns: + The path written. + """ + xml = generate_junit_xml(run_dir) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(xml, encoding="utf-8") + return output_path diff --git a/tests/conftest.py b/tests/conftest.py index 61f9b118..023745d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,10 @@ """Pytest configuration and shared fixtures.""" import logging +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import Any import pytest @@ -54,3 +58,51 @@ def reset_logging_after_test(): # Reset to default level app_logger.setLevel(logging.NOTSET) + + +@pytest.fixture +def write_run_json() -> Callable[..., Path]: + """Factory that writes a valid ``run.json`` (a real ``RunSummary``) into a run dir. + + Shared by the JUnit writer tests (Phase 1) and the CLI report/run tests + (Phase 2) so both track the ``RunSummary`` model instead of hand-typed JSON. + Task-status counts (``tasks_succeeded``/``failed``/``error``) are computed + from each row's ``status`` via ``FinalStatus.category`` so the model's + task-count invariant always holds. Callers pass row dicts shaped like + ``eval_result_to_task_dict`` output; only ``task_id`` and ``status`` are + required per row. + """ + from coder_eval.models import FinalStatus, RunSummary, SkippedTask + + def _build( + run_dir: Path, + rows: list[dict[str, Any]], + *, + skipped: list[tuple[str, str]] | None = None, + run_id: str = "2026-07-21_12-00-00", + ) -> Path: + counts = {"succeeded": 0, "failed": 0, "error": 0} + for row in rows: + try: + category = FinalStatus(row["status"]).category + except ValueError: + category = "error" # unknown status → error bucket (mirrors the writer) + counts[category] += 1 + summary = RunSummary( + run_id=run_id, + start_time=datetime(2026, 7, 21, 12, 0, 0), + end_time=datetime(2026, 7, 21, 12, 5, 0), + total_duration_seconds=300.0, + tasks_run=len(rows), + tasks_succeeded=counts["succeeded"], + tasks_failed=counts["failed"], + tasks_error=counts["error"], + skipped_tasks=[SkippedTask(path=p, reason=r) for p, r in (skipped or [])], + task_results=rows, + framework_version="test", + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run.json").write_text(summary.model_dump_json(indent=2), encoding="utf-8") + return run_dir / "run.json" + + return _build diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py new file mode 100644 index 00000000..a84c42e0 --- /dev/null +++ b/tests/test_reports_junit.py @@ -0,0 +1,458 @@ +"""Unit tests for the disk-driven JUnit XML writer (``reports_junit``). + +All tests are hermetic: the run directory is built by hand under ``tmp_path`` +(no agents, no API). Test-side XML parsing uses ``defusedxml`` as +defense-in-depth even though the parsed strings are self-generated. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from defusedxml.ElementTree import fromstring + +from coder_eval.models import SuiteRollup, ThresholdCheck +from coder_eval.reports_junit import generate_junit_xml, write_junit_xml + + +def _write_task_json( + run_dir: Path, + variant: str, + task_id: str, + replicate_index: int, + criteria: list[dict[str, Any]], +) -> None: + """Write a minimal task.json (plain dict) at the run-layout location.""" + task_dir = run_dir / variant / task_id / f"{replicate_index:02d}" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.json").write_text(json.dumps({"success_criteria_results": criteria}), encoding="utf-8") + + +def _row( + task_id: str, + status: str, + *, + variant_id: str | None = "default", + replicate_index: int | None = 0, + duration: float | None = 1.5, + task_path: str | None = "tasks/sample.yaml", + weighted_score: float | None = 0.5, + model_used: str | None = "claude-haiku-4-5-20251001", + total_tokens: int | None = 1234, + total_cost_usd: float | None = 0.01, + visible_turns: int | None = 3, +) -> dict[str, Any]: + return { + "task_id": task_id, + "status": status, + "variant_id": variant_id, + "replicate_index": replicate_index, + "duration": duration, + "task_path": task_path, + "weighted_score": weighted_score, + "model_used": model_used, + "total_tokens": total_tokens, + "total_cost_usd": total_cost_usd, + "visible_turns": visible_turns, + } + + +def _find_testsuite(root: Any, name: str) -> Any: + for ts in root.findall("testsuite"): + if ts.get("name") == name: + return ts + return None + + +def test_happy_path_grouping_and_counts(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("t_pass", "SUCCESS", variant_id="v1"), + _row("t_fail", "FAILURE", variant_id="v1"), + _row("t_err", "ERROR", variant_id="v1"), + _row("t_pass", "SUCCESS", variant_id="v2"), + _row("t_fail", "FAILURE", variant_id="v2"), + _row("t_err", "ERROR", variant_id="v2"), + ] + write_run_json(run_dir, rows) + + root = fromstring(generate_junit_xml(run_dir)) + assert root.tag == "testsuites" + # Root counts match the emitted children. + assert int(root.get("tests")) == 6 + assert int(root.get("failures")) == 2 + assert int(root.get("errors")) == 2 + assert int(root.get("skipped")) == 0 + + for variant in ("v1", "v2"): + ts = _find_testsuite(root, variant) + assert ts is not None + cases = ts.findall("testcase") + assert int(ts.get("tests")) == len(cases) == 3 + assert int(ts.get("failures")) == 1 + assert int(ts.get("errors")) == 1 + # classname derives from task_path stem. + assert all(c.get("classname") == "sample" for c in cases) + # time is a float string. + assert all(float(c.get("time")) == 1.5 for c in cases) + + +def test_root_counts_equal_summed_children(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("a", "SUCCESS"), _row("b", "FAILURE"), _row("c", "MAX_TURNS_EXHAUSTED")] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + + total_cases = 0 + total_failures = 0 + total_errors = 0 + for ts in root.findall("testsuite"): + cases = ts.findall("testcase") + total_cases += len(cases) + total_failures += sum(1 for c in cases if c.find("failure") is not None) + total_errors += sum(1 for c in cases if c.find("error") is not None) + assert int(root.get("tests")) == total_cases + assert int(root.get("failures")) == total_failures + assert int(root.get("errors")) == total_errors + # MAX_TURNS_EXHAUSTED is category "failed". + assert total_failures == 2 + assert total_errors == 0 + + +def test_failure_body_from_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "file_exists", + "description": "output.txt must exist", + "score": 0.0, + "pass_threshold": 0.9, + "details": "file not found on disk", + "error": None, + }, + { + "criterion_type": "file_contains", + "description": "greeting present", + "score": 1.0, + "pass_threshold": 0.9, + "details": None, + "error": None, + }, + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + failure = ts.find("testcase").find("failure") + body = failure.text or "" + assert "file_exists" in body + assert "0.00" in body and "0.90" in body # score-vs-threshold line + assert "file not found on disk" in body + assert "output.txt must exist" in body + # passed criterion appears as a one-liner. + assert "file_contains" in body + + +def test_fallback_body_without_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", weighted_score=0.42)] + write_run_json(run_dir, rows) + # No task.json written. + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + body = ts.find("testcase").find("failure").text or "" + assert "FAILURE" in body + assert "0.42" in body + + +def test_replicates_naming_and_per_replicate_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("task", "FAILURE", variant_id="v1", replicate_index=0), + _row("task", "FAILURE", variant_id="v1", replicate_index=1), + ] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "task", + 0, + [ + { + "criterion_type": "c", + "description": "rep0", + "score": 0.0, + "pass_threshold": 0.9, + "details": "REP0DETAIL", + "error": None, + } + ], + ) + _write_task_json( + run_dir, + "v1", + "task", + 1, + [ + { + "criterion_type": "c", + "description": "rep1", + "score": 0.0, + "pass_threshold": 0.9, + "details": "REP1DETAIL", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + names = {c.get("name") for c in ts.findall("testcase")} + assert names == {"task[00]", "task[01]"} + bodies = {c.get("name"): (c.find("failure").text or "") for c in ts.findall("testcase")} + assert "REP0DETAIL" in bodies["task[00]"] + assert "REP1DETAIL" in bodies["task[01]"] + + +def test_replicate_index_none_globs_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("task", "FAILURE", variant_id="v1", replicate_index=None)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "task", + 0, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": "GLOBBED", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "v1") + case = ts.find("testcase") + assert case.get("name") == "task" # no [NN] suffix + assert "GLOBBED" in (case.find("failure").text or "") + + +def test_variant_none_grouped_as_default(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS", variant_id=None, task_path=None)] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "default") + assert ts is not None + # task_path None → classname falls back to variant. + assert ts.find("testcase").get("classname") == "default" + + +def test_duration_none_renders_zero(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS", duration=None)] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + case = root.find("testsuite").find("testcase") + assert float(case.get("time")) == 0.0 + + +def test_properties_omit_none_values(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [ + _row("t", "SUCCESS", total_cost_usd=None, total_tokens=None, weighted_score=None), + ] + write_run_json(run_dir, rows) + xml = generate_junit_xml(run_dir) + root = fromstring(xml) + props = root.find("testsuite").find("testcase").find("properties") + names = {p.get("name") for p in props.findall("property")} if props is not None else set() + assert "total_cost_usd" not in names + assert "total_tokens" not in names + assert "weighted_score" not in names + assert "model_used" in names + # No literal "None" strings leaked into the tree. + assert "None" not in xml + + +def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SUCCESS")] + write_run_json( + run_dir, + rows, + skipped=[("tasks/broken.yaml", "ValueError: bad schema"), ("tasks/opt.yaml", "skip: true")], + ) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("skipped")) == 2 + ts = _find_testsuite(root, "skipped") + assert ts is not None + skipped_cases = ts.findall("testcase") + assert len(skipped_cases) == 2 + names = {c.get("name") for c in skipped_cases} + assert names == {"broken", "opt"} + assert all(c.find("skipped") is not None for c in skipped_cases) + + +def test_suite_gates_failing_and_none_metric(tmp_path: Path, write_run_json: Callable[..., Path]) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + rollup = SuiteRollup( + suite_id="s1", + variant_id="v1", + rows_total=10, + rows_passed=5, + rows_failed=5, + rows_error=0, + pass_rate=0.5, + passed=False, + ) + # Inject two failed threshold checks via a criterion aggregate. + from coder_eval.models import CriterionAggregate + + rollup.criterion_aggregates = [ + CriterionAggregate( + criterion_type="classification_match", + passed=False, + threshold_checks=[ + ThresholdCheck(metric="accuracy", min_value=0.9, actual_value=0.5, passed=False), + ThresholdCheck(metric="f1.macro", min_value=0.8, actual_value=None, passed=False), + ], + ), + # A criterion that declared suite_thresholds but aggregate() returned nothing. + CriterionAggregate( + criterion_type="skill_triggered", + passed=False, + error="aggregate() returned no metrics", + ), + ] + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text(rollup.model_dump_json(indent=2), encoding="utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "suite-gates") + assert ts is not None + case = ts.find("testcase") + assert case.get("name") == "v1/s1" + body = case.find("failure").text or "" + assert "accuracy" in body and "0.5" in body and "0.9" in body + assert "f1.macro" in body + assert "metric not emitted" in body + # CriterionAggregate.error is surfaced in the gate body. + assert "skill_triggered" in body and "aggregate() returned no metrics" in body + + +def test_suite_gates_passing_is_plain_case(tmp_path: Path, write_run_json: Callable[..., Path]) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + rollup = SuiteRollup( + suite_id="s1", + variant_id="v1", + rows_total=2, + rows_passed=2, + rows_failed=0, + rows_error=0, + pass_rate=1.0, + passed=True, + ) + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text(rollup.model_dump_json(indent=2), encoding="utf-8") + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "suite-gates") + case = ts.find("testcase") + assert case.find("failure") is None + + +def test_corrupt_suite_json_skipped_with_report_still_produced( + tmp_path: Path, write_run_json: Callable[..., Path], caplog: pytest.LogCaptureFixture +) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS", variant_id="v1")]) + suite_dir = run_dir / "v1" / "s1" + suite_dir.mkdir(parents=True, exist_ok=True) + (suite_dir / "suite.json").write_text("{not valid json", encoding="utf-8") + # Report still generated; no suite-gates suite (the only rollup was corrupt). + root = fromstring(generate_junit_xml(run_dir)) + assert _find_testsuite(root, "suite-gates") is None + + +def test_xml_safety_control_chars_and_markup(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + nasty = "\x1b[31mboom\x00 & 'quote'" + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": nasty, + "error": None, + } + ], + ) + xml = generate_junit_xml(run_dir) + root = fromstring(xml) # must re-parse + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "boom" in body + assert "\x00" not in body # scrubbed + assert "\x1b" not in body + + +def test_unknown_status_lands_in_errors(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + rows = [_row("t", "SOME_FUTURE_STATUS", variant_id="v1")] + write_run_json(run_dir, rows) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("errors")) == 1 + err = _find_testsuite(root, "v1").find("testcase").find("error") + assert err is not None + assert "unknown status" in (err.get("message") or "") + + +def test_empty_run_all_skipped(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[("tasks/a.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + assert int(root.get("tests")) == 1 # only the skipped case + assert int(root.get("skipped")) == 1 + assert int(root.get("failures")) == 0 + assert int(root.get("errors")) == 0 + + +def test_missing_run_json_raises(tmp_path: Path) -> None: + run_dir = tmp_path / "empty" + run_dir.mkdir() + with pytest.raises(FileNotFoundError, match=r"run\.json"): + generate_junit_xml(run_dir) + + +def test_write_junit_xml_creates_parents_and_returns_path(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t", "SUCCESS")]) + out = tmp_path / "nested" / "deeper" / "junit.xml" + written = write_junit_xml(run_dir, out) + assert written == out + assert out.is_file() + fromstring(out.read_text(encoding="utf-8")) # parses From 602c850fc1fe700a192a79740f71b4c0fe1d10ff Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 17:55:56 -0700 Subject: [PATCH 2/9] =?UTF-8?q?feat(cli):=202/3=20=E2=80=94=20wire=20run?= =?UTF-8?q?=20--junit-xml=20and=20report=20-f=20junit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --junit-xml to `coder-eval run` (writes the JUnit report after the run summary is persisted and before the failure exit-code gate, so a red run still produces a report; write errors propagate). Add 'junit' to `coder-eval report --format` (regenerates from any run dir; defaults to /junit.xml, -o overrides; missing run.json → clean red error). Output-only — does not enter the config merge. Docs: CI-pipeline tutorial JUnit section + CLAUDE.md tree entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 1 + docs/tutorials/02-ci-pipeline.md | 46 +++++++++++++++++ src/coder_eval/cli/report_command.py | 24 +++++++-- src/coder_eval/cli/run_command.py | 19 +++++++ tests/test_report_command.py | 52 +++++++++++++++++++ tests/test_run_command_junit.py | 77 ++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 tests/test_run_command_junit.py diff --git a/CLAUDE.md b/CLAUDE.md index c20bde4f..947ad4f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ coder_eval/ ├── orchestrator.py # Main evaluation loop ├── reports.py # Markdown/JSON report generation (run-level + per-suite rollup via write_suite_rollups) ├── reports_experiment.py # Experiment/cross-variant report generation +├── reports_junit.py # JUnit XML report from a finalized run dir (run.json spine; for CI test-report ingestion) ├── analysis.py # Command statistics aggregation ├── logging_config.py # Structured logging setup ├── path_utils.py # Run ID generation, path utilities diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 7dc04eea..63143087 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -158,6 +158,52 @@ jobs: - **`if: always()`** on the verdict and upload steps means you still get results even when a task fails. +## Publishing test results (JUnit) + +The verdict step above parses `task.json` by hand. If your CI already ingests +JUnit XML — for the per-test annotations, history, and flake tracking most CI +systems render from it — let `coder-eval` emit it directly instead. Add +`--junit-xml` to the run: + +```yaml + - name: Run evaluations + run: coder-eval run $TASK_GLOBS -j 4 --junit-xml coder-eval-junit.xml +``` + +The file is written **before** the exit-code decision, so a failing run still +produces a report. You can also regenerate it from any existing run directory +without re-running the suite: + +```bash +coder-eval report runs/latest -f junit # writes runs/latest/junit.xml +coder-eval report runs/latest -f junit -o out.xml # custom path +``` + +The mapping: one `` per task row (grouped into a `` per +variant), failed/errored rows carry a ``/`` with the per-criterion +breakdown, skipped tasks and failing suite gates get their own synthetic suites. + +Feed the file to whatever your platform uses to render test results. On GitHub +Actions, [`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): + +```yaml + - name: Publish test report + if: always() + uses: mikepenz/action-junit-report@v5 + with: + report_paths: coder-eval-junit.xml +``` + +On Azure DevOps, `PublishTestResults@2`: + +```yaml + - task: PublishTestResults@2 + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'coder-eval-junit.xml' +``` + ## Secrets Add these under **Settings → Secrets and variables → Actions**: diff --git a/src/coder_eval/cli/report_command.py b/src/coder_eval/cli/report_command.py index 4d801c99..6c5fcfc4 100644 --- a/src/coder_eval/cli/report_command.py +++ b/src/coder_eval/cli/report_command.py @@ -27,7 +27,10 @@ def report_command( "md", "--format", "-f", - help="Output format: 'md' (default markdown), 'html' (render task.json files as HTML).", + help=( + "Output format: 'md' (default markdown), 'html' (render task.json files as HTML), " + "'junit' (JUnit XML from run.json)." + ), ), ) -> None: """Display or export a run report. @@ -44,16 +47,31 @@ def report_command( # Re-generate HTML reports for every task.json under a run dir coder-eval report runs/latest --format html + + # Write a JUnit XML report (defaults to /junit.xml) + coder-eval report runs/latest --format junit """ fmt = report_format.lower() - if fmt not in ("md", "html"): - console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md' or 'html')[/red]") + if fmt not in ("md", "html", "junit"): + console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md', 'html', or 'junit')[/red]") raise typer.Exit(1) if fmt == "html": _regenerate_html_reports(run_dir, output_file) return + if fmt == "junit": + from ..reports_junit import write_junit_xml + + target = output_file if output_file is not None else run_dir / "junit.xml" + try: + written = write_junit_xml(run_dir, target) + except FileNotFoundError as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) from e + console.print(f"[green][OK]JUnit report written to {written}[/green]") + return + try: report_md, source_path = ReportGenerator.load_from_run_dir(run_dir) console.print(f"[dim]Reading report from {source_path}[/dim]\n") diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index b35366ca..dff081da 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -173,6 +173,11 @@ def run_command( "--log-file", help="Log to file in addition to console", ), + junit_xml: Path | None = typer.Option( # noqa: B008 + None, + "--junit-xml", + help="Write a JUnit XML report of task results to this path (for CI test-report ingestion).", + ), tags: str | None = typer.Option( None, "--tags", @@ -365,6 +370,7 @@ def run_command( verbose=verbose, resume=resume, include_skipped=include_skipped, + junit_xml=junit_xml, ) ) except KeyboardInterrupt: @@ -389,6 +395,7 @@ async def _run_all_tasks( verbose: bool = False, resume: bool = False, include_skipped: bool = False, + junit_xml: Path | None = None, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -407,6 +414,8 @@ async def _run_all_tasks( from -D/--set and the bespoke flag aliases stream_mode: Optional stream mode ('full' or 'minimal') for real-time output experiment_path: Optional path to experiment YAML (default: experiments/default.yaml) + junit_xml: Optional path to write a JUnit XML report to, after the run + summary is persisted and before the failure exit-code gate. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -464,6 +473,16 @@ async def _run_all_tasks( # Print execution summary print_execution_summary(run_dir, summary) + + # Write the JUnit report (if requested) BEFORE the exit-code gate below, + # so a failing run still produces the report. suite.json + run.json are + # already on disk (written inside _run_with_experiment). A write error + # propagates (loud failure, exit != 0) rather than being swallowed. + if junit_xml is not None: + from ..reports_junit import write_junit_xml + + written = write_junit_xml(run_dir, junit_xml) + console.print(f"[green][OK]JUnit report written to {written}[/green]") finally: # Explicit flush before process exit (belt-and-suspenders with atexit). # In a `finally` so it runs on the success path and on any raised diff --git a/tests/test_report_command.py b/tests/test_report_command.py index 3c7a7fe9..060e7f12 100644 --- a/tests/test_report_command.py +++ b/tests/test_report_command.py @@ -144,3 +144,55 @@ def test_report_html_malformed_task_json_skipped_and_fails(tmp_path: Path) -> No assert "Skipping" in res.stdout assert "1 task(s) failed" in res.stdout assert (run_dir / "task-good" / "task.html").exists() + + +# -------------------------------------------------------------------------- +# JUnit path (-f junit) +# -------------------------------------------------------------------------- + + +def test_report_junit_default_output(write_run_json, tmp_path: Path) -> None: + from defusedxml.ElementTree import fromstring + + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS"}]) + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit"]) + assert res.exit_code == 0 + out = run_dir / "junit.xml" + assert out.is_file() + root = fromstring(out.read_text(encoding="utf-8")) + assert root.tag == "testsuites" + + +def test_report_junit_custom_output(write_run_json, tmp_path: Path) -> None: + from defusedxml.ElementTree import fromstring + + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS"}]) + out = tmp_path / "out" / "custom.xml" + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit", "-o", str(out)]) + assert res.exit_code == 0 + assert out.is_file() + fromstring(out.read_text(encoding="utf-8")) + assert not (run_dir / "junit.xml").exists() + + +def test_report_junit_missing_run_json_exits_1(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() # no run.json + + res = runner.invoke(app, ["report", str(run_dir), "-f", "junit"]) + assert res.exit_code == 1 + assert "run.json" in res.stdout + + +def test_report_unknown_format_message_lists_junit(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + + res = runner.invoke(app, ["report", str(run_dir), "--format", "bogus"]) + assert res.exit_code == 1 + assert "unknown --format" in res.stdout + assert "junit" in res.stdout diff --git a/tests/test_run_command_junit.py b/tests/test_run_command_junit.py new file mode 100644 index 00000000..b5ac1de0 --- /dev/null +++ b/tests/test_run_command_junit.py @@ -0,0 +1,77 @@ +"""`coder-eval run --junit-xml` wiring at the ``_run_all_tasks`` seam. + +Fully hermetic: every side-effecting collaborator is patched (see the pattern in +``test_cli_telemetry.py``). The ``_run_with_experiment`` mock writes a minimal +``run.json`` to disk, which is exactly what the JUnit writer reads. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +import pytest +import typer +from defusedxml.ElementTree import fromstring + +from coder_eval.cli.run_command import _run_all_tasks + + +async def _invoke( + run_dir: Path, + junit_xml: Path | None, + write_run_json: Callable[..., Path], + *, + status: str, + failed: bool, +) -> None: + summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0) + + async def _fake(*_args, **_kwargs): + # Mirror production: run.json is persisted inside _run_with_experiment. + write_run_json(run_dir, [{"task_id": "t", "status": status}]) + return summary, 0 + + with ( + patch("coder_eval.cli.run_command.prepare_run_directory", return_value=run_dir), + patch("coder_eval.cli.run_command.expand_task_files", return_value=[Path("a.yaml")]), + patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(side_effect=_fake)), + patch("coder_eval.logging_config.aggregate_task_logs"), + patch("coder_eval.cli.run_command.print_execution_summary"), + patch("coder_eval.telemetry.track_event"), + patch("coder_eval.telemetry.flush_telemetry"), + ): + await _run_all_tasks( + task_files=[Path("a.yaml")], + preservation_mode=None, + run_dir=run_dir, + max_parallel=1, + junit_xml=junit_xml, + ) + + +async def test_junit_written_on_success(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + await _invoke(run_dir, junit, write_run_json, status="SUCCESS", failed=False) + assert junit.is_file() + fromstring(junit.read_text(encoding="utf-8")) + + +async def test_junit_written_even_when_run_fails(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + # Ordering guarantee: the report is written BEFORE the failure exit-code gate. + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + with pytest.raises(typer.Exit) as exc: + await _invoke(run_dir, junit, write_run_json, status="FAILURE", failed=True) + assert exc.value.exit_code == 1 + assert junit.is_file() + fromstring(junit.read_text(encoding="utf-8")) + + +async def test_no_junit_when_flag_absent(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + run_dir = tmp_path / "run" + junit = tmp_path / "j.xml" + await _invoke(run_dir, None, write_run_json, status="SUCCESS", failed=False) + assert not junit.exists() From cc18d4c2671e076b086c4ab0474d36b25b7a2694 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:09:27 -0700 Subject: [PATCH 3/9] =?UTF-8?q?feat(ci):=203/3=20=E2=80=94=20publish=20com?= =?UTF-8?q?posite=20action,=20release=20automation,=20PR=20dogfood?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add action.yml at the repo root: a composite action that installs a pinned coder-eval (or the local checkout via `version: local`), runs the suite, always emits run-dir/junit-path outputs and a run.md job summary, then exits with coder-eval's real exit code. Inputs cross into bash via env: only. release.yml now bumps the action's `version:` default inside the amended release commit (anchored on a `# <-- kept in sync` comment, with a loud grep guard) and force-moves the `v` tag after pushing the release tag. pr-checks.yml gains a fork-gated `action-dogfood` job that exercises the action via `uses: ./` on one Haiku task and verifies the JUnit + run.json output — the sensor for this surface, since it isn't covered by pytest. The action is deliberately agent-agnostic: it installs no coding-agent runtime, so callers provide the Claude CLI (as the dogfood job does). Documented in the README and the CI tutorial along with the pull_request_target security caveat. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/pr-checks.yml | 50 ++++++++++++++ .github/workflows/release.yml | 39 +++++++++-- CLAUDE.md | 1 + README.md | 47 +++++++++++++ action.yml | 110 +++++++++++++++++++++++++++++++ docs/tutorials/02-ci-pipeline.md | 25 +++++++ 6 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 action.yml diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 28578b09..f985560f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -831,3 +831,53 @@ jobs: name: byoa-live-output path: tmp/ retention-days: 7 + + action-dogfood: + name: Action Dogfood (composite action, real API) + runs-on: ubuntu-latest + timeout-minutes: 15 + # Skip on fork PRs where secrets aren't available (matches e2e-smoke). + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The composite action is agent-agnostic and does NOT install a coding-agent + # runtime. The dogfood task uses the default claude-code agent, so provide + # Node + the Claude CLI here (as e2e-smoke does), before invoking the action. + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + - name: Install Claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Run coder-eval via local action + id: dogfood + uses: ./ + with: + version: local + tasks: tasks/hello_date.yaml + model: claude-haiku-4-5-20251001 + run-dir: runs/ci-action-dogfood + junit-path: runs/ci-action-dogfood/junit.xml + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Verify outputs and JUnit file + env: + JUNIT: ${{ steps.dogfood.outputs.junit-path }} + RUNDIR: ${{ steps.dogfood.outputs.run-dir }} + run: | + set -euo pipefail + test -n "$JUNIT" && test -f "$JUNIT" || { echo "junit output missing"; exit 1; } + # Well-formedness check on a file this job just generated (trusted input; + # our writer emits no DTDs/entities) — stdlib ET is fine here. + python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" + test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; } + + - name: Upload dogfood run on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: action-dogfood-runs + path: runs/ci-action-dogfood/ + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff2a7ebb..23094069 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -182,23 +182,50 @@ jobs: echo "version=$V" >> "$GITHUB_OUTPUT" echo "Publishing version: $V" - - name: Regenerate uv.lock and amend release commit + - name: Regenerate uv.lock, bump action.yml pin, and amend release commit if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' + env: + # Passed via env (not interpolated into the script) per GitHub's + # injection guidance. + VERSION: ${{ steps.release.outputs.version }} run: | + set -euo pipefail + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + # Bump the composite action's default `version:` pin to the just-released + # version so `UiPath/coder_eval@vX.Y.Z` installs `coder-eval==X.Y.Z`. The + # anchor is indentation-tolerant and keyed on the unique trailing + # "# <-- kept in sync" comment; the grep guard fails the release loudly + # if a reformat ever detaches it (rather than shipping a stale pin). + sed -i -E 's/^([[:space:]]*default: ")[0-9]+\.[0-9]+\.[0-9]+(" # <-- kept in sync)/\1'"${VERSION}"'\2/' action.yml + grep -q "default: \"${VERSION}\"" action.yml || { echo "action.yml version bump failed"; exit 1; } + git add action.yml + # Regenerate the lock too; stage it (a no-op if unchanged). uv lock - if ! git diff --quiet uv.lock; then - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "github-actions[bot]" - git add uv.lock + git add uv.lock + # Amend only if action.yml/uv.lock actually changed the tree. + if ! git diff --cached --quiet; then git commit --amend --no-edit # Amend replaced the commit the tag points at; re-point it before pushing. - git tag -f "v${{ steps.release.outputs.version }}" + git tag -f "v${VERSION}" fi - name: Push release commit and tags if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' run: git push origin main "v${{ steps.release.outputs.version }}" + - name: Move major action tag (vN -> this release) + if: steps.release.outputs.version != '' + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + # Consumers pin `UiPath/coder_eval@v0` (becomes `@v1` at 1.0.0). Force-move + # the moving major tag to this release. Force on a missing tag creates it. + MAJOR="v${VERSION%%.*}" + git tag -f "$MAJOR" "v${VERSION}" + git push -f origin "$MAJOR" + - name: Build wheel + sdist if: steps.ver.outputs.version != '' run: uv build diff --git a/CLAUDE.md b/CLAUDE.md index 947ad4f2..87b531a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,6 +121,7 @@ tasks/ # Task definition YAML files tests/ # Test suite docs/ # Documentation templates/ # Sandbox template directories +action.yml # Published composite GitHub Action (coder-eval as a CI gate). release.yml maintains its `version:` default + the moving `v` tag. ``` ## Key Architectural Patterns diff --git a/README.md b/README.md index 733d1053..ccb79aff 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,53 @@ live in this repo — clone it or point the CLI at your own task files.) See [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for the full setup. +## Use as a GitHub Action + +A composite action at the repo root runs `coder-eval` as a CI gate — it installs +the pinned CLI, runs your tasks, writes a JUnit XML report, appends `run.md` to +the job summary, and fails the step on any task/gate failure: + +```yaml +- uses: UiPath/coder_eval@v0 # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly + with: + tasks: tests/tasks/**/*.yaml + model: claude-haiku-4-5-20251001 + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +| Input | Default | Purpose | +| --- | --- | --- | +| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob | +| `tags` | — | `--tags` filter | +| `model` | — | `--model` override | +| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) | +| `version` | pinned release | PyPI version, or `local` to install from the checkout | +| `run-dir` | `runs/ci` | Run directory | +| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report | +| `step-summary` | `true` | Append `run.md` to the job summary | +| `anthropic-api-key` | — | Exported as `ANTHROPIC_API_KEY` for the run | + +Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's +test-report renderer — e.g. on GitHub Actions with +[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report): + +```yaml +- uses: mikepenz/action-junit-report@v5 + if: always() + with: + report_paths: coder-eval-junit.xml +``` + +> **Agent runtime is the caller's responsibility.** The action is agent-agnostic — +> it installs `coder-eval` but no coding-agent runtime. Tasks using the default +> `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` + +> `npm install -g @anthropic-ai/claude-code`) in the job before the action runs. + +> **Security.** Evaluated tasks execute agent-generated code. Do **not** run this +> action under `pull_request_target` with secrets exposed to untrusted fork PRs — +> use `pull_request` and gate on the same-repo condition, as this repo's own +> dogfood job does. + ## Telemetry > 📊 **Usage telemetry is on by default.** `coder-eval` sends **anonymous** usage diff --git a/action.yml b/action.yml new file mode 100644 index 00000000..c9006644 --- /dev/null +++ b/action.yml @@ -0,0 +1,110 @@ +name: coder-eval +description: Run coder-eval evaluation tasks as a CI gate, with JUnit XML output and a job-summary report. +branding: + icon: check-circle + color: orange + +# This action installs and runs the `coder-eval` CLI. It is agent-agnostic: it +# does NOT install any coding-agent runtime. Tasks that use the default +# `claude-code` agent need the `claude` CLI on PATH (Node + the +# `@anthropic-ai/claude-code` npm package) provided by the calling job before +# this action runs. See the README "Use as a GitHub Action" section. +# +# SECURITY: evaluated tasks execute agent-generated code. Do NOT run this action +# under `pull_request_target` with secrets exposed to untrusted fork PRs. + +inputs: + tasks: + description: Task YAML path(s)/glob passed to `coder-eval run` (empty = all tasks/ recursively) + required: false + default: "" + tags: + description: Only run tasks matching any of these comma-separated tags (--tags) + required: false + default: "" + model: + description: Override agent model for all tasks (--model) + required: false + default: "" + extra-args: + description: Extra arguments appended verbatim to `coder-eval run` (trusted caller input; covers --experiment, -D overrides, --tags exclusions, etc.) + required: false + default: "" + version: + description: coder-eval version to install from PyPI, or "local" to install from the action checkout + required: false + default: "0.8.6" # <-- kept in sync with releases by release.yml + run-dir: + description: Run directory (--run-dir) + required: false + default: "runs/ci" + junit-path: + description: Where to write the JUnit XML report + required: false + default: "coder-eval-junit.xml" + step-summary: + description: Append run.md to the GitHub job summary ("true"/"false") + required: false + default: "true" + anthropic-api-key: + description: Anthropic API key (exported as ANTHROPIC_API_KEY for the run step) + required: false + default: "" + +outputs: + run-dir: + description: The run directory containing run.json/run.md + value: ${{ steps.run.outputs.run-dir }} + junit-path: + description: Path to the written JUnit XML report + value: ${{ steps.run.outputs.junit-path }} + +runs: + using: composite + steps: + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + - name: Install coder-eval + shell: bash + env: + CE_VERSION: ${{ inputs.version }} + CE_ACTION_PATH: ${{ github.action_path }} + run: | + set -euo pipefail + if [ "$CE_VERSION" = "local" ]; then + uv tool install "$CE_ACTION_PATH" + else + uv tool install "coder-eval==$CE_VERSION" + fi + - name: Run coder-eval + id: run + shell: bash + env: + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} + CE_TASKS: ${{ inputs.tasks }} + CE_TAGS: ${{ inputs.tags }} + CE_MODEL: ${{ inputs.model }} + CE_EXTRA_ARGS: ${{ inputs.extra-args }} + CE_RUN_DIR: ${{ inputs.run-dir }} + CE_JUNIT: ${{ inputs.junit-path }} + CE_SUMMARY: ${{ inputs.step-summary }} + run: | + set -uo pipefail + args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT") + [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS") + [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL") + # extra-args is a trusted caller input, split on whitespace intentionally + # shellcheck disable=SC2206 + [ -n "$CE_EXTRA_ARGS" ] && args+=($CE_EXTRA_ARGS) + # shellcheck disable=SC2206 + [ -n "$CE_TASKS" ] && args+=($CE_TASKS) + set +e + coder-eval "${args[@]}" + CODE=$? + set -e + echo "run-dir=$CE_RUN_DIR" >> "$GITHUB_OUTPUT" + echo "junit-path=$CE_JUNIT" >> "$GITHUB_OUTPUT" + if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then + head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY" + fi + exit $CODE diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 63143087..c19f3309 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -158,6 +158,31 @@ jobs: - **`if: always()`** on the verdict and upload steps means you still get results even when a task fails. +## Shortcut: the packaged action + +The five steps above spell out the mechanics, but coder_eval also ships a +composite action at the repo root that bundles install + run + JUnit report + +job-summary + fail-on-failure into one step: + +```yaml + - uses: actions/setup-node@v4 # the claude-code agent needs the Claude CLI… + with: { node-version: '20' } + - run: npm install -g @anthropic-ai/claude-code + + - uses: UiPath/coder_eval@v0 # …then run the gate (pin @vX.Y.Z in production) + with: + tasks: tests/tasks/**/*.yaml + model: claude-haiku-4-5-20251001 + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +The action is agent-agnostic — it installs `coder-eval` but not the agent +runtime, so provide the `claude` CLI (or your agent's runtime) in the job first. +See the [README "Use as a GitHub Action"](../../README.md#use-as-a-github-action) +section for the full inputs table and the fork/`pull_request_target` security +caveat. The rest of this tutorial's hand-rolled workflow is still useful when you +need finer control than the action's inputs expose. + ## Publishing test results (JUnit) The verdict step above parses `task.json` by hand. If your CI already ingests From 5c8374664e78d8fb98d4635b2d30f30e91e38256 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:11:40 -0700 Subject: [PATCH 4/9] chore(deps): lock defusedxml (dev-only, test-side XML parsing) Companion to the pyproject dev-group addition in 1/3. Co-Authored-By: Claude Opus 4.8 (1M context) --- uv.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uv.lock b/uv.lock index 5be0d013..8278218b 100644 --- a/uv.lock +++ b/uv.lock @@ -480,6 +480,7 @@ codex = [ ] dev = [ { name = "bandit" }, + { name = "defusedxml" }, { name = "mcp" }, { name = "pip-audit" }, { name = "pre-commit" }, @@ -503,6 +504,7 @@ requires-dist = [ { name = "bandit", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=1.9.4" }, { name = "claude-agent-sdk", specifier = ">=0.2.124" }, { name = "click", specifier = ">=8.3.3" }, + { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.7" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, From 2fa2292e4ddf90fe2952f26d74a09f58de58df70 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:19:33 -0700 Subject: [PATCH 5/9] fix(reports): code review fixes for the JUnit CI gate Harden reports_junit.py against schema-skewed / crafted run.json rows, which are untyped dicts (RunSummary.task_results is list[dict[str, Any]]) and may be blob-pulled from elsewhere: - Path containment: a crafted variant_id/task_id could steer the task.json lookup outside the run dir (absolute value discards run_dir, ".." walks up). Reject unsafe path components and verify containment after resolve(). - UnicodeDecodeError is a ValueError but NOT a json.JSONDecodeError, so a task.json with undecodable bytes aborted the whole report instead of falling back. Catch ValueError. - Ambiguous replicate: with replicate_index absent and several replicate dirs, the writer attributed replicate 00's failure detail to the row. Degrade to the status-only body instead of misattributing. - bool is an int subclass, so a bool replicate_index rendered as "[01]" and a bool duration as a time. Exclude bool explicitly. - Guard non-finite/negative durations (NaN would emit an invalid time="nan"). - Coerce non-str variant_id/task_path rather than crashing on dict keys/Path(). - Skipped testcases now use the suffix-stripped path, not the bare stem, so two skipped tasks sharing a basename keep distinct JUnit identities. - A row with no status reads as "" rather than "None". Found by the final multi-model review (gpt-5.6, gemini-3.1-pro, gpt-5.3-codex); each fix has a regression test that fails without it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/reports_junit.py | 92 ++++++++++++++++++----- tests/conftest.py | 2 +- tests/test_reports_junit.py | 127 +++++++++++++++++++++++++++++++- 3 files changed, 201 insertions(+), 20 deletions(-) diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index 194a6f5e..a6c3cb5f 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -21,6 +21,7 @@ import json import logging +import math import re import xml.etree.ElementTree as ET from pathlib import Path @@ -42,6 +43,11 @@ # Per-testcase failure/error body cap (chars). Agent detail dumps can be huge. _BODY_LIMIT = 10_000 +# Serialized status values we recognize, for distinguishing a known status from a +# schema-skewed one when labelling a failure/error (classification itself goes +# through FinalStatus.category — see _category_of). +_KNOWN_STATUSES = frozenset(s.value for s in FinalStatus) + def _xml_safe(text: str) -> str: """Strip characters outside XML 1.0's legal set (ET handles ``<>&`` itself).""" @@ -74,24 +80,65 @@ def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None: elem.set("skipped", str(sum(1 for c in cases if c.find("skipped") is not None))) +def _variant_of(row: dict[str, Any]) -> str: + """Variant bucket for a row — ``str``-coerced, ``None``/empty → ``"default"``. + + Rows are untyped dicts, so a non-string ``variant_id`` must not reach a dict + key or an XML attribute as a non-``str``. + """ + return str(row.get("variant_id") or "default") + + +def _status_of(row: dict[str, Any]) -> str: + """Row status as a string; an absent key reads as ``""``, not ``"None"``.""" + raw = row.get("status") + return str(raw) if raw is not None else "" + + +def _is_safe_component(value: str) -> bool: + """True when ``value`` is usable as a single, contained path segment. + + ``run.json`` rows are untyped and may be blob-pulled from elsewhere, so a + crafted ``variant_id``/``task_id`` must not steer the lookup outside the run + directory (an absolute value would discard ``run_dir`` entirely, and ``..`` + would walk up). + """ + return bool(value) and value not in {".", ".."} and "/" not in value and "\\" not in value + + def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None: """Best-effort load of a failed row's ``task.json`` as a plain dict. Plain-dict access (not ``EvaluationResult.model_validate``) keeps the writer tolerant of schema skew in blob-pulled/older runs and avoids materializing - the large ``turns`` array. Any error (missing dir/file, bad JSON) → ``None``. + the large ``turns`` array. Any problem — unsafe path component, missing + dir/file, undecodable bytes, bad JSON, or an ambiguous replicate — yields + ``None`` so the caller falls back to a status-only body. """ task_id = str(row.get("task_id", "")) + if not _is_safe_component(variant) or not _is_safe_component(task_id): + return None + replicate_index = row.get("replicate_index") task_dir = run_dir / variant / task_id - if isinstance(replicate_index, int): + if isinstance(replicate_index, int) and not isinstance(replicate_index, bool): candidate = task_dir / f"{replicate_index:02d}" / "task.json" else: matches = sorted(task_dir.glob("*/task.json")) + # With no replicate index, picking one of several would misattribute + # another replicate's failure detail to this row — degrade instead. + if len(matches) > 1: + return None candidate = matches[0] if matches else task_dir / "task.json" + try: + # Belt-and-braces containment check (catches symlink escapes too). + if not candidate.resolve().is_relative_to(run_dir.resolve()): + return None + # ValueError covers both json.JSONDecodeError and UnicodeDecodeError + # (undecodable bytes) — neither may abort report generation. return json.loads(candidate.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + except (OSError, ValueError): return None @@ -102,7 +149,7 @@ def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: status + weighted-score line when task.json is missing/corrupt. Capped via the shared ``truncate``. """ - status = str(row.get("status")) + status = _status_of(row) data = _load_task_json(run_dir, row, variant) criteria = data.get("success_criteria_results") if isinstance(data, dict) else None @@ -135,16 +182,27 @@ def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: """Build one ```` for a task row.""" - variant = row.get("variant_id") or "default" + variant = _variant_of(row) task_id = str(row.get("task_id", "")) + + # `bool` is an int subclass — exclude it so True never renders as "[01]". replicate_index = row.get("replicate_index") - name = f"{task_id}[{replicate_index:02d}]" if isinstance(replicate_index, int) else task_id + has_replicate = isinstance(replicate_index, int) and not isinstance(replicate_index, bool) + name = f"{task_id}[{replicate_index:02d}]" if has_replicate else task_id task_path = row.get("task_path") - classname = Path(task_path).stem if task_path else variant + classname = (Path(task_path).stem or variant) if isinstance(task_path, str) and task_path else variant + # time must be a finite, non-negative float: NaN/inf would serialize as + # "nan"/"inf" and make the report invalid for JUnit ingesters. duration = row.get("duration") - time_str = f"{duration:.3f}" if isinstance(duration, int | float) else "0.000" + valid_duration = ( + isinstance(duration, int | float) + and not isinstance(duration, bool) + and math.isfinite(duration) + and duration >= 0 + ) + time_str = f"{duration:.3f}" if valid_duration else "0.000" case = ET.Element( "testcase", @@ -165,16 +223,12 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: for key, value in props: ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))}) - category = _category_of(str(row.get("status"))) + status = _status_of(row) + category = _category_of(status) if category == "succeeded": return case - status = str(row.get("status")) - try: - FinalStatus(status) - message = status - except ValueError: - message = f"unknown status: {status}" + message = status if status in _KNOWN_STATUSES else f"unknown status: {status}" tag = "failure" if category == "failed" else "error" child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) child.text = _criteria_body(row, run_dir, variant) @@ -188,10 +242,13 @@ def _skipped_suite(summary: RunSummary) -> ET.Element | None: suite = ET.Element("testsuite", {"name": "skipped"}) cases: list[ET.Element] = [] for entry in summary.skipped_tasks: + # Use the suffix-stripped PATH, not just the stem: two skipped tasks + # sharing a basename (suiteA/task.yaml, suiteB/task.yaml) would otherwise + # collapse into one identity that some JUnit ingesters merge. case = ET.SubElement( suite, "testcase", - {"name": _xml_safe(Path(entry.path).stem), "classname": "skipped"}, + {"name": _xml_safe(str(Path(entry.path).with_suffix(""))), "classname": "skipped"}, ) ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) cases.append(case) @@ -268,8 +325,7 @@ def generate_junit_xml(run_dir: Path) -> str: # Group task rows by variant, preserving first-seen order. grouped: dict[str, list[dict[str, Any]]] = {} for row in summary.task_results: - variant = row.get("variant_id") or "default" - grouped.setdefault(variant, []).append(row) + grouped.setdefault(_variant_of(row), []).append(row) all_cases: list[ET.Element] = [] for variant, rows in grouped.items(): diff --git a/tests/conftest.py b/tests/conftest.py index 023745d0..07cf3c7d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,7 +84,7 @@ def _build( counts = {"succeeded": 0, "failed": 0, "error": 0} for row in rows: try: - category = FinalStatus(row["status"]).category + category = FinalStatus(row.get("status")).category except ValueError: category = "error" # unknown status → error bucket (mirrors the writer) counts[category] += 1 diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py index a84c42e0..0b041ffb 100644 --- a/tests/test_reports_junit.py +++ b/tests/test_reports_junit.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import math from collections.abc import Callable from pathlib import Path from typing import Any @@ -302,8 +303,9 @@ def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path assert ts is not None skipped_cases = ts.findall("testcase") assert len(skipped_cases) == 2 + # Suffix-stripped full path (not just the stem) so same-basename tasks stay distinct. names = {c.get("name") for c in skipped_cases} - assert names == {"broken", "opt"} + assert names == {"tasks/broken", "tasks/opt"} assert all(c.find("skipped") is not None for c in skipped_cases) @@ -456,3 +458,126 @@ def test_write_junit_xml_creates_parents_and_returns_path(write_run_json: Callab assert written == out assert out.is_file() fromstring(out.read_text(encoding="utf-8")) # parses + + +# -------------------------------------------------------------------------- +# Robustness against schema-skewed / crafted run.json rows (final-review findings) +# -------------------------------------------------------------------------- + + +def test_malformed_row_types_degrade_gracefully(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Loose `task_results` dicts are untyped; a skewed row must not abort the report. + + Covers: bool replicate_index (bool is an int subclass), non-finite duration + (would emit an invalid time="nan"), non-str variant_id / task_path, and a + row with no `status` key at all. + """ + run_dir = tmp_path / "run" + rows: list[dict[str, Any]] = [ + {"task_id": "t_bool", "status": "SUCCESS", "replicate_index": True, "duration": 1.0}, + {"task_id": "t_nan", "status": "SUCCESS", "duration": float("nan")}, + {"task_id": "t_inf", "status": "SUCCESS", "duration": float("inf")}, + {"task_id": "t_neg", "status": "SUCCESS", "duration": -5.0}, + {"task_id": "t_variant", "status": "SUCCESS", "variant_id": 17}, + {"task_id": "t_path", "status": "SUCCESS", "task_path": 42}, + {"task_id": "t_nostatus"}, + ] + write_run_json(run_dir, rows) + + xml = generate_junit_xml(run_dir) + root = fromstring(xml) # must still be well-formed + + cases = [c for ts in root.findall("testsuite") for c in ts.findall("testcase")] + assert len(cases) == len(rows) + # Every time attribute must be a finite, non-negative float (JUnit requirement). + for case in cases: + t = float(case.get("time")) + assert math.isfinite(t) and t >= 0.0 # finite (not NaN/inf), non-negative + # A bool replicate_index must NOT be formatted as a [NN] replicate suffix. + names = {c.get("name") for c in cases} + assert "t_bool" in names + # The missing-status row lands in the error bucket, not silently as a pass. + assert int(root.get("errors")) == 1 + + +def test_task_json_invalid_utf8_falls_back(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A task.json with undecodable bytes must degrade, not raise (UnicodeDecodeError + is a ValueError but NOT a json.JSONDecodeError).""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)]) + task_dir = run_dir / "v1" / "t_fail" / "00" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "task.json").write_bytes(b"\xff\xfe\x00 not utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "FAILURE" in body # status-only fallback body + + +def test_task_json_lookup_cannot_escape_run_dir(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A crafted variant_id/task_id must not make the writer read outside run_dir.""" + outside = tmp_path / "outside" + outside.mkdir() + (outside / "task.json").write_text( + json.dumps( + { + "success_criteria_results": [ + { + "criterion_type": "leaked", + "description": "SECRET", + "score": 0.0, + "pass_threshold": 0.9, + "details": "LEAKED_SECRET_DETAIL", + "error": None, + } + ] + } + ), + encoding="utf-8", + ) + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "..", "status": "FAILURE", "variant_id": "../outside"}]) + + root = fromstring(generate_junit_xml(run_dir)) + xml_text = generate_junit_xml(run_dir) + assert "LEAKED_SECRET_DETAIL" not in xml_text + body = root.find("testsuite").find("testcase").find("failure").text or "" + assert "FAILURE" in body # fell back, did not read the outside file + + +def test_ambiguous_replicate_does_not_misattribute(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """replicate_index=None with MULTIPLE replicate dirs must fall back rather than + attribute replicate 00's failure detail to the row.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [_row("task", "FAILURE", variant_id="v1", replicate_index=None)]) + for idx, marker in ((0, "REP0ONLY"), (1, "REP1ONLY")): + _write_task_json( + run_dir, + "v1", + "task", + idx, + [ + { + "criterion_type": "c", + "description": "d", + "score": 0.0, + "pass_threshold": 0.9, + "details": marker, + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "REP0ONLY" not in body and "REP1ONLY" not in body + assert "FAILURE" in body + + +def test_skipped_names_unique_for_same_stem(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Two skipped tasks sharing a basename must get distinct testcase identities.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[("suiteA/task.yaml", "skip: true"), ("suiteB/task.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + ts = _find_testsuite(root, "skipped") + names = {c.get("name") for c in ts.findall("testcase")} + assert len(names) == 2, f"skipped testcase names collided: {names}" From fb17eed9cbab245f8d6424308a73bfb300ad5b74 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:32:43 -0700 Subject: [PATCH 6/9] fix(reports): make skipped-task JUnit names platform-independent _skipped_suite built the testcase name with str(Path(path).with_suffix("")), which yields "tasks\opt" on Windows and "tasks/opt" on Linux. That made the JUnit testcase identity depend on the OS that generated the report, so the same logical run would split into two identities in CI history/flake tracking (and it broke the Windows smoke job). Normalize separators and parse with PurePosixPath so the emitted name is always "/"-separated. Adds a regression test feeding a Windows-style path. Also switch the action's model in the README/tutorial examples from Haiku to Sonnet (docs only; the dogfood job stays on Haiku for per-PR cost). Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- docs/tutorials/02-ci-pipeline.md | 2 +- src/coder_eval/reports_junit.py | 22 +++++++++++++++++----- tests/test_reports_junit.py | 15 ++++++++++++++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ccb79aff..a923d991 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ the job summary, and fails the step on any task/gate failure: - uses: UiPath/coder_eval@v0 # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly with: tasks: tests/tasks/**/*.yaml - model: claude-haiku-4-5-20251001 + model: claude-sonnet-5 anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} ``` diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index c19f3309..1e7fb200 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -172,7 +172,7 @@ job-summary + fail-on-failure into one step: - uses: UiPath/coder_eval@v0 # …then run the gate (pin @vX.Y.Z in production) with: tasks: tests/tasks/**/*.yaml - model: claude-haiku-4-5-20251001 + model: claude-sonnet-5 anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} ``` diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index a6c3cb5f..9786601e 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -24,7 +24,7 @@ import math import re import xml.etree.ElementTree as ET -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Literal from .evaluation.judge_context import truncate @@ -235,6 +235,21 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: return case +def _skipped_name(path: str) -> str: + """Stable testcase name for a skipped task: suffix-stripped, ``/``-separated. + + Uses the whole path rather than just the stem, because two skipped tasks + sharing a basename (``suiteA/task.yaml``, ``suiteB/task.yaml``) would + otherwise collapse into one identity that some JUnit ingesters merge. + + Separators are normalized to ``/`` and the path is parsed with + ``PurePosixPath`` so the emitted name does not depend on the OS that + generated the report — the same logical run must produce the same testcase + identity on Windows and Linux, or CI history/flake tracking splits in two. + """ + return str(PurePosixPath(path.replace("\\", "/")).with_suffix("")) + + def _skipped_suite(summary: RunSummary) -> ET.Element | None: """Build the synthetic ``skipped`` testsuite from ``RunSummary.skipped_tasks``.""" if not summary.skipped_tasks: @@ -242,13 +257,10 @@ def _skipped_suite(summary: RunSummary) -> ET.Element | None: suite = ET.Element("testsuite", {"name": "skipped"}) cases: list[ET.Element] = [] for entry in summary.skipped_tasks: - # Use the suffix-stripped PATH, not just the stem: two skipped tasks - # sharing a basename (suiteA/task.yaml, suiteB/task.yaml) would otherwise - # collapse into one identity that some JUnit ingesters merge. case = ET.SubElement( suite, "testcase", - {"name": _xml_safe(str(Path(entry.path).with_suffix(""))), "classname": "skipped"}, + {"name": _xml_safe(_skipped_name(entry.path)), "classname": "skipped"}, ) ET.SubElement(case, "skipped", {"message": _xml_safe(entry.reason)}) cases.append(case) diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py index 0b041ffb..73f7ef18 100644 --- a/tests/test_reports_junit.py +++ b/tests/test_reports_junit.py @@ -303,9 +303,12 @@ def test_skipped_tasks_suite(write_run_json: Callable[..., Path], tmp_path: Path assert ts is not None skipped_cases = ts.findall("testcase") assert len(skipped_cases) == 2 - # Suffix-stripped full path (not just the stem) so same-basename tasks stay distinct. + # Suffix-stripped full path (not just the stem) so same-basename tasks stay + # distinct. Always '/'-separated: the same logical run must yield the same + # testcase identity on Windows and Linux, or CI history splits in two. names = {c.get("name") for c in skipped_cases} assert names == {"tasks/broken", "tasks/opt"} + assert not any("\\" in n for n in names) assert all(c.find("skipped") is not None for c in skipped_cases) @@ -581,3 +584,13 @@ def test_skipped_names_unique_for_same_stem(write_run_json: Callable[..., Path], ts = _find_testsuite(root, "skipped") names = {c.get("name") for c in ts.findall("testcase")} assert len(names) == 2, f"skipped testcase names collided: {names}" + + +def test_skipped_name_is_platform_independent(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A Windows-style path recorded in run.json must still emit a '/'-separated + name, so a report generated on Windows matches one generated on Linux.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [], skipped=[(r"tasks\win\task.yaml", "skip: true")]) + root = fromstring(generate_junit_xml(run_dir)) + name = _find_testsuite(root, "skipped").find("testcase").get("name") + assert name == "tasks/win/task" From 2f27fc70c57cd3ed53e5112f4637c277a0a25042 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 21 Jul 2026 18:53:31 -0700 Subject: [PATCH 7/9] chore: re-trigger CI (GitHub dropped the force-push event) Co-Authored-By: Claude From b669a38fc12c35da9c207156889ce0432a740e98 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 22 Jul 2026 16:42:33 -0700 Subject: [PATCH 8/9] feat(action): generic env passthrough + minimum-task-score gate Replace the single `anthropic-api-key` input with a generic `env` NAME=VALUE passthrough (exported for the run step only, never $GITHUB_ENV) and add an optional `minimum-task-score` floor read from the always-written run.json spine. Both default off, so existing behavior is unchanged. README, tutorial, and the dogfood job route ANTHROPIC_API_KEY through `env`. Code-review hardening (gemini-3.1-pro + gpt-5.3-codex + opus): - score gate skips non-finite weighted_score (json.loads parses NaN; NaN makes min()/>= order-dependent and could mask a below-floor task) - minimum-task-score validated (numeric, finite, in [0.0, 1.0]) with a clean ::error:: instead of a raw traceback - malformed env entries reported by position, never echoing the raw line, so a mis-prefixed secret can't leak into logs Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/pr-checks.yml | 10 ++- README.md | 25 +++++++- action.yml | 107 +++++++++++++++++++++++++++++-- docs/tutorials/02-ci-pipeline.md | 3 +- 4 files changed, 137 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index f985560f..9f55e5b1 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -860,7 +860,15 @@ jobs: model: claude-haiku-4-5-20251001 run-dir: runs/ci-action-dogfood junit-path: runs/ci-action-dogfood/junit.xml - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + # Credentials go through the generic env passthrough (the only channel); + # ANTHROPIC_API_KEY reaching the run is proven by the API-backed task + # succeeding. A floor of 0.0 passes for any produced score (exercises + # the gate path green in CI without flakiness); the second line + # exercises multi-line env parsing. + minimum-task-score: "0.0" + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} + CE_DOGFOOD_MARKER=1 - name: Verify outputs and JUnit file env: diff --git a/README.md b/README.md index a923d991..3d5dd4e3 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,8 @@ the job summary, and fails the step on any task/gate failure: with: tasks: tests/tasks/**/*.yaml model: claude-sonnet-5 - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` | Input | Default | Purpose | @@ -120,7 +121,8 @@ the job summary, and fails the step on any task/gate failure: | `run-dir` | `runs/ci` | Run directory | | `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report | | `step-summary` | `true` | Append `run.md` to the job summary | -| `anthropic-api-key` | — | Exported as `ANTHROPIC_API_KEY` for the run | +| `env` | — | Credentials/backend passthrough: newline-separated `NAME=VALUE` pairs, exported for the run step only | +| `minimum-task-score` | *(off)* | Strict floor (0.0–1.0): fail the step if any task's `weighted_score` is below it | Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's test-report renderer — e.g. on GitHub Actions with @@ -133,6 +135,25 @@ test-report renderer — e.g. on GitHub Actions with report_paths: coder-eval-junit.xml ``` +**Credentials and backend config** are the sole responsibility of `env` — a +passthrough exported for the run step only (never written to `$GITHUB_ENV`, so +it can't leak into later steps). Set whatever the run needs, Anthropic or not: + +```yaml +- uses: UiPath/coder_eval@v0 + with: + tasks: tests/tasks/**/*.yaml + minimum-task-score: "0.8" # fail the build if any task scores below 0.8 + env: | + CODER_EVAL_API_BACKEND=bedrock + AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }} +``` + +`minimum-task-score` is a strict floor **on top of** coder-eval's own exit +code: the step fails if *either* coder-eval exits non-zero *or* any task's +`weighted_score` falls below the floor. Leave it unset to gate on the exit code +alone. + > **Agent runtime is the caller's responsibility.** The action is agent-agnostic — > it installs `coder-eval` but no coding-agent runtime. Tasks using the default > `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` + diff --git a/action.yml b/action.yml index c9006644..07b15081 100644 --- a/action.yml +++ b/action.yml @@ -46,8 +46,25 @@ inputs: description: Append run.md to the GitHub job summary ("true"/"false") required: false default: "true" - anthropic-api-key: - description: Anthropic API key (exported as ANTHROPIC_API_KEY for the run step) + env: + description: >- + Environment passthrough — newline-separated NAME=VALUE pairs exported for + the coder-eval process only (scoped to the run step; NOT written to + $GITHUB_ENV, so nothing leaks into later job steps). This is the sole + channel for credentials and backend config: set ANTHROPIC_API_KEY, + CODER_EVAL_API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names + must match ^[A-Za-z_][A-Za-z0-9_]*$; blank lines and `#` comments are + ignored. Wire values from repository secrets (secrets.MY_KEY) — never + inline a secret literal. + required: false + default: "" + minimum-task-score: + description: >- + Optional strict floor (0.0–1.0): EVERY scored task, in every variant, must + reach it or the step fails. A gate ON TOP OF coder-eval's own exit code — + the step fails if EITHER coder-eval exits non-zero OR any task's + weighted_score is below this. Empty (the default) disables the floor, + leaving coder-eval's exit code the sole gate. required: false default: "" @@ -80,7 +97,6 @@ runs: id: run shell: bash env: - ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} CE_TASKS: ${{ inputs.tasks }} CE_TAGS: ${{ inputs.tags }} CE_MODEL: ${{ inputs.model }} @@ -88,8 +104,36 @@ runs: CE_RUN_DIR: ${{ inputs.run-dir }} CE_JUNIT: ${{ inputs.junit-path }} CE_SUMMARY: ${{ inputs.step-summary }} + CE_ENV: ${{ inputs.env }} + CE_MIN_SCORE: ${{ inputs.minimum-task-score }} run: | set -uo pipefail + + # Generic env passthrough. Each NAME=VALUE line is exported for the + # coder-eval child of THIS step only — deliberately not written to + # $GITHUB_ENV, so a forwarded secret never bleeds into later job steps. + # Only NAME is validated; VALUE is treated as opaque data (never eval'd), + # so no crafted value can inject shell. + n=0 + while IFS= read -r line; do + n=$((n + 1)) + line="${line%$'\r'}" # tolerate CRLF inputs + line="${line#"${line%%[![:space:]]*}"}" # left-trim + [ -z "$line" ] && continue + case "$line" in '#'*) continue ;; esac # allow comment lines + # Never echo $line/$value on error — a caller who forgets the `NAME=` + # prefix would otherwise print a forwarded secret verbatim (GitHub + # only masks values it knows are secrets). Report by position instead. + if [ "$line" = "${line#*=}" ]; then + echo "::error::env entry #$n is not NAME=VALUE (no '=' found)"; exit 1 + fi + name="${line%%=*}" + if [[ ! "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "::error::env entry #$n has an invalid name (must match ^[A-Za-z_][A-Za-z0-9_]*\$)"; exit 1 + fi + export "$name=${line#*=}" + done <<< "$CE_ENV" + args=(run --run-dir "$CE_RUN_DIR" --junit-xml "$CE_JUNIT") [ -n "$CE_TAGS" ] && args+=(--tags "$CE_TAGS") [ -n "$CE_MODEL" ] && args+=(--model "$CE_MODEL") @@ -107,4 +151,59 @@ runs: if [ "$CE_SUMMARY" = "true" ] && [ -f "$CE_RUN_DIR/run.md" ]; then head -c 1000000 "$CE_RUN_DIR/run.md" >> "$GITHUB_STEP_SUMMARY" fi - exit $CODE + + # Optional per-task score floor. Reads the always-written run.json spine + # (task_results[*].weighted_score) rather than experiment.json, so it + # works for plain and experiment runs alike. A gate ON TOP OF CODE: + # empty CE_MIN_SCORE disables it, leaving CODE the sole verdict. Runs + # regardless of CODE (run.json is emitted even on a red run), then the + # two verdicts are combined so both surface. + GATE=0 + if [ -n "$CE_MIN_SCORE" ]; then + if [ ! -f "$CE_RUN_DIR/run.json" ]; then + echo "::error::score gate: $CE_RUN_DIR/run.json not found (cannot verify minimum-task-score=$CE_MIN_SCORE)" + GATE=1 + else + RUN_JSON="$CE_RUN_DIR/run.json" MIN_SCORE="$CE_MIN_SCORE" python3 <<'PY' || GATE=1 + import json, math, os, sys + + raw = os.environ["MIN_SCORE"] + try: + floor = float(raw) + except ValueError: + print(f"::error::minimum-task-score must be a number, got '{raw}'") + sys.exit(1) + if not math.isfinite(floor) or not (0.0 <= floor <= 1.0): + print(f"::error::minimum-task-score must be within [0.0, 1.0], got '{raw}'") + sys.exit(1) + data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) + rows = [] + for r in data.get("task_results", []): + s = r.get("weighted_score") + # bool is an int subclass — exclude it; skip errored rows (score None), + # which coder-eval's own exit code already accounts for; and skip + # non-finite (NaN/inf) — json.loads accepts them, and NaN makes + # min()/>= order-dependent, which could silently MASK a below-floor + # task. A blob-pulled/malformed run.json must fail closed, not pass. + if isinstance(s, (int, float)) and not isinstance(s, bool) and math.isfinite(s): + rows.append((float(s), str(r.get("task_id", "?")), str(r.get("variant_id") or "default"))) + for s, t, v in sorted(rows): + print(f" [{'ok' if s >= floor else 'BELOW'}] {v}/{t}: {s:.3f}") + worst = min(rows) if rows else None + ok = worst is not None and worst[0] >= floor + if ok: + print(f"score gate passed: lowest {worst[0]:.3f} >= floor {floor:.3f}") + elif worst is not None: + print(f"::error::score gate FAILED: lowest {worst[0]:.3f} ({worst[2]}/{worst[1]}) < floor {floor:.3f}") + else: + print(f"::error::score gate: no scored tasks in run.json to check against floor {floor:.3f}") + sys.exit(0 if ok else 1) + PY + fi + fi + + # Combine: coder-eval's own failure OR the score floor fails the step. + if [ "$CODE" -ne 0 ]; then + exit "$CODE" + fi + [ "$GATE" -eq 0 ] || exit 1 diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index 1e7fb200..6943ae79 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -173,7 +173,8 @@ job-summary + fail-on-failure into one step: with: tasks: tests/tasks/**/*.yaml model: claude-sonnet-5 - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} ``` The action is agent-agnostic — it installs `coder-eval` but not the agent From aa8ef40ae1676e24cd8317178a39252eb1ff85de Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 23 Jul 2026 14:25:16 -0700 Subject: [PATCH 9/9] fix(reports): JUnit CI-gate review fixes + CE027 env-var lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review fixes to the JUnit XML CI gate, plus a new doc/config lint rule and a producer→consumer parity test. reports_junit.py: - Load task.json for dataset rows: task_id "/" is a real nested dir, so validate it as a contained relative path (_is_safe_relpath) instead of rejecting any '/'. Rejects absolute/backslash/'..'/Windows-drive values; the resolve()-containment check remains the backstop. - Guard the root time against NaN/inf like per-testcase time via a shared _time_attr helper; also degrade (not crash) on a pathologically large integer duration that overflows float()/'.3f'. - Render informational (gating=False) criteria as [INFO], not [FAIL]/[PASS], mirroring reports.py; only an explicit JSON `false` is informational (null / non-bool fails safe to gating). Docs: fix CODER_EVAL_API_BACKEND -> API_BACKEND (the real Settings env name; the CODER_EVAL_-prefixed spelling was silently dropped by Settings extra="ignore", selecting no backend). CE027 (tests/lint/doc_env_parity.py): new rule flagging framework-prefixed env-var assignments in README/action.yml/docs that no Settings field/alias or src/ consumer backs — the class that produced the API_BACKEND bug. Assignment-scoped with a hardened boundary and a real-consumer src scan to avoid false positives. Tests: dataset nested-load, nested '..'/drive rejection, informational [INFO] (passing + null-gating), root/huge-int time degradation, and a producer→consumer parity test running real eval_result_to_task_dict output through generate_junit_xml. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- action.yml | 2 +- src/coder_eval/reports_junit.py | 87 +++++++++-- tests/lint/doc_env_parity.py | 125 +++++++++++++++ tests/test_custom_lint.py | 77 +++++++++ tests/test_reports_junit.py | 267 ++++++++++++++++++++++++++++++++ 6 files changed, 541 insertions(+), 19 deletions(-) create mode 100644 tests/lint/doc_env_parity.py diff --git a/README.md b/README.md index 3d5dd4e3..4c63d01e 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ it can't leak into later steps). Set whatever the run needs, Anthropic or not: tasks: tests/tasks/**/*.yaml minimum-task-score: "0.8" # fail the build if any task scores below 0.8 env: | - CODER_EVAL_API_BACKEND=bedrock + API_BACKEND=bedrock AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }} ``` diff --git a/action.yml b/action.yml index 07b15081..e4041a0d 100644 --- a/action.yml +++ b/action.yml @@ -52,7 +52,7 @@ inputs: the coder-eval process only (scoped to the run step; NOT written to $GITHUB_ENV, so nothing leaks into later job steps). This is the sole channel for credentials and backend config: set ANTHROPIC_API_KEY, - CODER_EVAL_API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names + API_BACKEND, model vars, EVALBOARD_*, plugin paths, etc. Names must match ^[A-Za-z_][A-Za-z0-9_]*$; blank lines and `#` comments are ignored. Wire values from repository secrets (secrets.MY_KEY) — never inline a secret literal. diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index 9786601e..569f7217 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -24,7 +24,7 @@ import math import re import xml.etree.ElementTree as ET -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Any, Literal from .evaluation.judge_context import truncate @@ -99,13 +99,60 @@ def _is_safe_component(value: str) -> bool: """True when ``value`` is usable as a single, contained path segment. ``run.json`` rows are untyped and may be blob-pulled from elsewhere, so a - crafted ``variant_id``/``task_id`` must not steer the lookup outside the run - directory (an absolute value would discard ``run_dir`` entirely, and ``..`` - would walk up). + crafted ``variant_id`` must not steer the lookup outside the run directory + (an absolute value would discard ``run_dir`` entirely, and ``..`` would walk + up). """ return bool(value) and value not in {".", ".."} and "/" not in value and "\\" not in value +def _is_safe_relpath(value: str) -> bool: + """True when ``value`` is a contained relative path — possibly *nested*. + + Unlike :func:`_is_safe_component`, an internal ``/`` is allowed: dataset + expansion rewrites a row's ``task_id`` to ``"/"`` + (``task_loader.expand_dataset``) and the on-disk layout is correspondingly + nested (``run_dir/////task.json``, see + ``path_utils.build_task_run_dir``). Rejecting the ``/`` would degrade every + dataset-derived row (e.g. the activation suite) to a status-only body. + + Only an absolute path, a Windows drive/UNC prefix, or a ``.``/``..`` + component could escape ``run_dir``; all are rejected here, and the + ``resolve()``-containment check in :func:`_load_task_json` is the + belt-and-braces backstop (symlinks included). + """ + # A Windows drive-qualified value (``C:/x``) reads as a plain relative path + # on POSIX but is absolute on Windows, so reject it explicitly rather than + # leaning on the resolve-containment backstop alone. + if not value or value.startswith("/") or "\\" in value or PureWindowsPath(value).drive: + return False + parts = PurePosixPath(value).parts + return bool(parts) and all(p not in {".", ".."} for p in parts) + + +def _time_attr(value: Any) -> str: + """Serialize a duration as a JUnit ``time`` attribute string. + + The value must be a finite, non-negative number; NaN/inf/negative/non-numeric + all fall back to ``"0.000"``. NaN/inf would otherwise serialize as + ``"nan"``/``"inf"`` and make the document invalid for JUnit ingesters. Shared + by the per-testcase time and the root ```` time so both are + guarded identically. + + Rows are untyped, so ``value`` may be a pathologically large JSON integer + (hundreds of digits) that overflows on the ``float()`` conversion / ``.3f`` + format — the ``OverflowError`` is caught and degraded rather than aborting + the whole report, matching this module's degrade-don't-crash contract. + """ + if isinstance(value, bool) or not isinstance(value, int | float) or value < 0: + return "0.000" + try: + as_float = float(value) + except (OverflowError, ValueError): + return "0.000" + return f"{as_float:.3f}" if math.isfinite(as_float) else "0.000" + + def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[str, Any] | None: """Best-effort load of a failed row's ``task.json`` as a plain dict. @@ -116,7 +163,7 @@ def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[st ``None`` so the caller falls back to a status-only body. """ task_id = str(row.get("task_id", "")) - if not _is_safe_component(variant) or not _is_safe_component(task_id): + if not _is_safe_component(variant) or not _is_safe_relpath(task_id): return None replicate_index = row.get("replicate_index") @@ -162,11 +209,23 @@ def _criteria_body(row: dict[str, Any], run_dir: Path, variant: str) -> str: description = str(crit.get("description", "")) score = crit.get("score") threshold = crit.get("pass_threshold") + # Only an explicit JSON ``false`` marks an informational criterion; a + # missing key or a schema-skewed value (null / non-bool) fails safe + # to gating — matching CriterionResult.gating's default and this + # module's isinstance-guarded, degrade-don't-crash reads of untyped + # rows. Informational criteria are excluded from the score/gate, so + # they are labelled [INFO] regardless of pass/fail and never rendered + # as the failure cause (mirrors reports.py `_compute_suite_rollup`'s + # `if not cr.gating: continue`). + informational = crit.get("gating", True) is False + score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score) + if informational: + lines.append(f"[INFO] {ctype}: score {score_str} — {description}") + continue passed = isinstance(score, int | float) and isinstance(threshold, int | float) and score >= threshold if passed: lines.append(f"[PASS] {ctype}: {description}") continue - score_str = f"{score:.2f}" if isinstance(score, int | float) else str(score) thr_str = f"{threshold:.2f}" if isinstance(threshold, int | float) else str(threshold) lines.append(f"[FAIL] {ctype}: score {score_str} < threshold {thr_str} — {description}") detail = crit.get("details") or crit.get("error") @@ -193,16 +252,7 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: task_path = row.get("task_path") classname = (Path(task_path).stem or variant) if isinstance(task_path, str) and task_path else variant - # time must be a finite, non-negative float: NaN/inf would serialize as - # "nan"/"inf" and make the report invalid for JUnit ingesters. - duration = row.get("duration") - valid_duration = ( - isinstance(duration, int | float) - and not isinstance(duration, bool) - and math.isfinite(duration) - and duration >= 0 - ) - time_str = f"{duration:.3f}" if valid_duration else "0.000" + time_str = _time_attr(row.get("duration")) case = ET.Element( "testcase", @@ -332,7 +382,10 @@ def generate_junit_xml(run_dir: Path) -> str: summary = RunSummary.model_validate_json(run_json.read_text(encoding="utf-8")) root = ET.Element("testsuites", {"name": _xml_safe(summary.run_id)}) - root.set("time", f"{summary.total_duration_seconds:.3f}") + # Guard the root time identically to per-testcase time: a corrupt/blob-pulled + # run.json can carry a NaN/inf total_duration_seconds (RunSummary has no + # finite validator), which would emit an invalid time="nan". + root.set("time", _time_attr(summary.total_duration_seconds)) # Group task rows by variant, preserving first-seen order. grouped: dict[str, list[dict[str, Any]]] = {} diff --git a/tests/lint/doc_env_parity.py b/tests/lint/doc_env_parity.py new file mode 100644 index 00000000..e074c146 --- /dev/null +++ b/tests/lint/doc_env_parity.py @@ -0,0 +1,125 @@ +"""CE027 — documented framework env vars must be backed by a real consumer. + +``coder_eval.config.Settings`` sets no ``env_prefix`` and uses ``extra="ignore"``, +so a documented env var whose name does not match a ``Settings`` field (or one of +its ``AliasChoices``) is **silently dropped** at runtime with zero signal — the +exact failure mode behind the ``CODER_EVAL_API_BACKEND`` doc bug (the real field +is ``API_BACKEND``, so the ``CODER_EVAL_``-prefixed spelling selected no backend +and the run fell back to Direct Anthropic). + +This rule scans the doc/config surfaces (``README.md``, ``action.yml``, +``docs/**``) for env-var **assignments** (``NAME=value`` — the copy-pasteable, +dangerous form) carrying a **framework-owned prefix** and flags any whose name is +neither a ``Settings`` env name/alias nor referenced anywhere in ``src/`` — the +framework also reads a handful of vars directly via ``os.getenv`` (e.g. +``CODER_EVAL_SKILLS_DIR``, ``CODEX_BASE_URL``, ``CODER_EVAL_IN_CONTAINER``), and +those are legitimately documentable. + +Scope note: only *assignments* are checked, not bare prose mentions. Prose +scanning is too false-positive-prone (markdown links like +``CODEX_AGENT_GUIDE.md``, secret RHS references like ``secrets.BEDROCK_TOKEN``, +regex-pattern examples like ``API_KEY = "…"``), and the assignment form is the +one users actually copy into a workflow, so it carries the real risk. + +It is intentionally NOT a ``BaseRule`` registered in ``tests/lint/runner.py``: +that runner is AST-only and walks ``.py`` files, whereas this rule reasons over +Markdown/YAML doc surfaces. It is wired as a dedicated test in +``tests/test_custom_lint.py::TestCE027DocEnvVarParity``. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from pydantic import AliasChoices + + +# Framework-owned env-var name prefixes. A documented token starting with one of +# these is owned by coder-eval and MUST be consumed by it. Deliberately EXCLUDES +# broad third-party namespaces (AWS_, ANTHROPIC_, GEMINI_, GITHUB_, EVALBOARD_, +# PLUGIN_) whose vars are consumed by SDKs / CI, not necessarily via Settings, so +# scanning them would produce false positives on legitimately-external names. +FRAMEWORK_ENV_PREFIXES: tuple[str, ...] = ( + "CODER_EVAL_", + "API_", + "BEDROCK_", + "CODEX_", + "ANTIGRAVITY_", + "TELEMETRY_", +) + +_PREFIX_ALT = "|".join(FRAMEWORK_ENV_PREFIXES) + +# A framework-prefixed env-var ASSIGNMENT in doc/config text: ``NAME=`` where NAME +# carries a framework prefix. The negative lookbehind rejects a name embedded in +# a larger token — attribute access (``secrets.BEDROCK_TOKEN``), a hyphenated +# token (``X-API_KEY=``), a path/URL segment (``dir/API_X=``, ``http://API_Y=``), +# or a Windows path (``C:\\API_Z=``). The single ``=`` (not ``==``) with no space +# before it also rejects ``API_KEY = "…"`` regex-pattern examples. +_ENV_ASSIGNMENT = re.compile(r"(? set[str]: + """Uppercased env names Settings actually reads: field names + AliasChoices.""" + from coder_eval.config import Settings + + names: set[str] = set() + for field_name, field in Settings.model_fields.items(): + names.add(field_name.upper()) + alias = field.validation_alias + if isinstance(alias, AliasChoices): + names.update(str(c).upper() for c in alias.choices if isinstance(c, str)) + elif isinstance(alias, str): + names.add(alias.upper()) + return names + + +def src_env_literals(src_root: Path) -> set[str]: + """Env-var names ``src/`` actually consumes: direct ``os.getenv``/``os.environ`` + reads plus the NAME side of inline ``"NAME=VALUE"`` child-process literals.""" + names: set[str] = set() + for py in src_root.rglob("*.py"): + text = py.read_text(encoding="utf-8") + names.update(_SRC_ENV_READ.findall(text)) + names.update(_SRC_ENV_VALUE.findall(text)) + return names + + +def scan_doc_env_assignments(text: str) -> set[str]: + """Framework-prefixed env-var names *assigned* (``NAME=…``) in a doc file.""" + return set(_ENV_ASSIGNMENT.findall(text)) + + +def find_unbacked_env_vars(doc_paths: list[Path], src_root: Path) -> dict[str, list[str]]: + """Map each doc path to the framework-prefixed env vars it assigns that + nothing in the framework consumes (would be silently dropped at runtime).""" + valid = settings_env_names() | src_env_literals(src_root) + findings: dict[str, list[str]] = {} + for path in doc_paths: + if not path.is_file(): + continue + unbacked = sorted(t for t in scan_doc_env_assignments(path.read_text(encoding="utf-8")) if t not in valid) + if unbacked: + findings[str(path)] = unbacked + return findings + + +def default_doc_paths(repo_root: Path) -> list[Path]: + """The doc/config surfaces CE027 scans: README, the published action, docs/**.""" + paths = [repo_root / "README.md", repo_root / "action.yml"] + docs = repo_root / "docs" + if docs.is_dir(): + for suffix in ("*.md", "*.yaml", "*.yml"): + paths.extend(sorted(docs.rglob(suffix))) + return paths diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 125c649f..09d7c86e 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -630,3 +630,80 @@ def test_compliant_class_not_flagged(self): def test_exempt_modules_not_scoped(self): for path in ("src/coder_eval/models/results.py", "src/coder_eval/models/telemetry.py"): assert not self._run(self._VIOLATING, path) + + +@pytest.mark.lint +class TestCE027DocEnvVarParity: + """CE027 — documented framework env-var assignments must be backed by a + real consumer (a Settings field/alias or an os.getenv read in src/). + + `Settings` sets no `env_prefix` and uses `extra="ignore"`, so a documented + `NAME=value` whose name matches no field is silently dropped at runtime — the + `CODER_EVAL_API_BACKEND` (real field: `API_BACKEND`) doc bug. Scans real + Markdown/YAML surfaces, so it lives here rather than in the AST-only runner. + """ + + REPO_ROOT = Path(__file__).parent.parent + + def test_repo_docs_have_no_unbacked_env_vars(self): + from tests.lint.doc_env_parity import default_doc_paths, find_unbacked_env_vars + + findings = find_unbacked_env_vars(default_doc_paths(self.REPO_ROOT), self.REPO_ROOT / "src") + assert not findings, ( + "\nDocumented framework env-var assignment(s) that no Settings field/alias or " + "src/ reference backs — they are silently dropped at runtime (Settings uses " + 'extra="ignore"). Fix the spelling to a real env name, or add the consumer:\n\n' + + "\n".join(f" {path}: {', '.join(names)}" for path, names in sorted(findings.items())) + ) + + def test_catches_the_coder_eval_api_backend_shadow(self): + # The exact regression: a CODER_EVAL_-prefixed spelling of a real field. + from tests.lint.doc_env_parity import scan_doc_env_assignments, settings_env_names, src_env_literals + + valid = settings_env_names() | src_env_literals(self.REPO_ROOT / "src") + found = scan_doc_env_assignments(" CODER_EVAL_API_BACKEND=bedrock\n") + assert "CODER_EVAL_API_BACKEND" in found + assert "CODER_EVAL_API_BACKEND" not in valid # would be flagged + + def test_real_backend_field_is_backed(self): + from tests.lint.doc_env_parity import settings_env_names + + assert "API_BACKEND" in settings_env_names() + + @pytest.mark.parametrize( + "line", + [ + "AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }}", # secret RHS, not an assignment of BEDROCK_TOKEN + "[Codex Agent Guide](docs/CODEX_AGENT_GUIDE.md)", # markdown link, not an assignment + 'pattern: "^API_KEY = \\"\\\\w+\\""', # regex example with a space before '=' + "the CODEX_MODEL setting selects the model", # bare prose mention, not an assignment + "X-API_KEY=v", # hyphenated token — prefix embedded, not a standalone name + "dir/API_THING=v", # path segment — prefix embedded + "http://API_HOST=1", # URL segment — prefix embedded + ], + ) + def test_non_assignment_shapes_are_not_flagged(self, line: str): + from tests.lint.doc_env_parity import scan_doc_env_assignments, settings_env_names, src_env_literals + + valid = settings_env_names() | src_env_literals(self.REPO_ROOT / "src") + unbacked = [t for t in scan_doc_env_assignments(line) if t not in valid] + assert unbacked == [], f"false positive on non-assignment shape: {unbacked}" + + def test_name_side_of_env_value_literal_counts_as_backed(self): + # `--env CODER_EVAL_IN_CONTAINER=1` in src makes the doc assignment backed. + from tests.lint.doc_env_parity import src_env_literals + + names = src_env_literals(self.REPO_ROOT / "src") + assert "CODER_EVAL_IN_CONTAINER" in names + + def test_src_scan_requires_a_real_consumer_not_any_literal(self, tmp_path: Path): + # A bare uppercase constant that no code reads must NOT count as "backed", + # or it could silently mask a documented-but-unconsumed assignment. + from tests.lint.doc_env_parity import src_env_literals + + (tmp_path / "m.py").write_text( + 'CONST = "CODER_EVAL_BOGUS"\nx = os.getenv("CODER_EVAL_REAL")\n', encoding="utf-8" + ) + names = src_env_literals(tmp_path) + assert "CODER_EVAL_REAL" in names # a genuine os.getenv read is backed + assert "CODER_EVAL_BOGUS" not in names # a bare constant is not diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py index 73f7ef18..54d44b78 100644 --- a/tests/test_reports_junit.py +++ b/tests/test_reports_junit.py @@ -20,6 +20,12 @@ from coder_eval.reports_junit import generate_junit_xml, write_junit_xml +def _props(case: Any) -> dict[str, str]: + """Map of a testcase's children.""" + properties = case.find("properties") + return {p.get("name"): p.get("value") for p in properties.findall("property")} if properties is not None else {} + + def _write_task_json( run_dir: Path, variant: str, @@ -594,3 +600,264 @@ def test_skipped_name_is_platform_independent(write_run_json: Callable[..., Path root = fromstring(generate_junit_xml(run_dir)) name = _find_testsuite(root, "skipped").find("testcase").get("name") assert name == "tasks/win/task" + + +def test_dataset_nested_task_id_loads_task_json(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A dataset-derived task_id ('/') maps to a real nested dir, + so task.json must still load and per-criterion detail must be emitted rather + than degrading to the status-only fallback body.""" + run_dir = tmp_path / "run" + rows = [_row("activation/row_0", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "activation/row_0", + 0, + [ + { + "criterion_type": "skill_triggered", + "description": "must engage the target skill", + "score": 0.0, + "pass_threshold": 0.9, + "details": "NESTED_DETAIL_LOADED", + "error": None, + } + ], + ) + root = fromstring(generate_junit_xml(run_dir)) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "skill_triggered" in body + assert "NESTED_DETAIL_LOADED" in body # nested task.json was read, not skipped + + +def test_nested_task_id_with_dotdot_still_cannot_escape(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Allowing an internal '/' must not open a '..' traversal: a nested task_id + containing a '..' component is rejected before any read.""" + outside = tmp_path / "outside" + outside.mkdir() + (outside / "task.json").write_text( + json.dumps( + { + "success_criteria_results": [ + { + "criterion_type": "leaked", + "description": "SECRET", + "score": 0.0, + "pass_threshold": 0.9, + "details": "LEAKED_VIA_NESTED_DOTDOT", + "error": None, + } + ] + } + ), + encoding="utf-8", + ) + run_dir = tmp_path / "run" + write_run_json( + run_dir, + [{"task_id": "a/../../outside", "status": "FAILURE", "variant_id": "v1", "replicate_index": 0}], + ) + xml_text = generate_junit_xml(run_dir) + assert "LEAKED_VIA_NESTED_DOTDOT" not in xml_text + root = fromstring(xml_text) + body = _find_testsuite(root, "v1").find("testcase").find("failure").text or "" + assert "FAILURE" in body # fell back to status-only body + + +def test_informational_criterion_not_rendered_as_failure(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A non-gating (informational) criterion below its threshold must render as + [INFO], never [FAIL] — it is excluded from the score/gate, so it cannot be + the failure cause (mirrors reports.py's `if not cr.gating`).""" + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "commands_efficiency", + "description": "informational efficiency metric", + "score": 0.1, + "pass_threshold": 0.9, + "gating": False, + "details": None, + "error": None, + } + ], + ) + body = _find_testsuite(fromstring(generate_junit_xml(run_dir)), "v1").find("testcase").find("failure").text or "" + assert "[INFO]" in body + assert "[FAIL]" not in body # informational criterion is not the failure cause + assert "commands_efficiency" in body + + +def test_root_time_finite_for_nonfinite_total_duration(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A corrupt/blob-pulled run.json with a NaN total_duration_seconds must not + emit an invalid root time='nan' — the root time is guarded like per-testcase + time and falls back to '0.000'. + + RunSummary has no finite validator and pydantic's JSON parser accepts a bare + ``NaN`` literal, so a foreign run.json can carry one; we inject it directly + since ``model_dump_json`` would coerce a NaN float to ``null``. + """ + run_dir = tmp_path / "run" + run_json = write_run_json(run_dir, [_row("t", "SUCCESS")]) + text = run_json.read_text(encoding="utf-8") + assert '"total_duration_seconds": 300.0' in text + run_json.write_text(text.replace('"total_duration_seconds": 300.0', '"total_duration_seconds": NaN'), "utf-8") + + root = fromstring(generate_junit_xml(run_dir)) + assert math.isfinite(float(root.get("time"))) + assert root.get("time") == "0.000" + + +def test_parity_real_producer_output_through_writer(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """Feed REAL ``eval_result_to_task_dict`` output through ``generate_junit_xml``. + + Every other test builds rows via the synthetic ``_row`` helper, which + hand-copies the keys the writer reads. This one runs the actual producer + (``reports_experiment.eval_result_to_task_dict``, the batch.py path) so a + producer-side rename of ``status`` / ``task_path`` / ``total_cost_usd`` / + ``model_used`` / ``total_tokens`` / ``visible_turns`` / ``weighted_score`` + (RunSummary.task_results is an untyped ``list[dict[str, Any]]``) can no longer + silently drop a property/classname or mis-bucket a row with zero failing + assertions. + """ + from datetime import datetime + + from coder_eval.models import AgentKind, EvaluationResult, FinalStatus, TokenUsage + from coder_eval.reports_experiment import eval_result_to_task_dict + + result = EvaluationResult( + task_id="mytask", + task_description="d", + variant_id="v1", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 7, 21, 12, 0, 0), + final_status=FinalStatus.FAILURE, + iteration_count=1, + model_used="claude-haiku-4-5-20251001", + weighted_score=0.375, + duration_seconds=2.5, + total_token_usage=TokenUsage(uncached_input_tokens=100, output_tokens=50, total_cost_usd=0.02), + ) + row = eval_result_to_task_dict( + result, + variant_id="v1", + task_path="tasks/suiteA/mytask.yaml", + replicate_index=0, + ) + + run_dir = tmp_path / "run" + write_run_json(run_dir, [row]) + root = fromstring(generate_junit_xml(run_dir)) + + # Grouped under the producer's variant_id, not "default". + ts = _find_testsuite(root, "v1") + assert ts is not None + case = ts.find("testcase") + + # classname derives from the producer's task_path stem. + assert case.get("classname") == "mytask" + # name carries the producer's replicate_index. + assert case.get("name") == "mytask[00]" + # time is the producer's duration, formatted. + assert case.get("time") == "2.500" + + # FAILURE (a "failed"-category status) becomes a , not /pass. + assert case.find("failure") is not None + + # Properties mirror the producer's values exactly (assert against the real + # dict, not hand-copied literals — this is the key-coupling contract). + props = _props(case) + assert props["model_used"] == str(row["model_used"]) + assert props["weighted_score"] == str(row["weighted_score"]) + assert props["total_cost_usd"] == str(row["total_cost_usd"]) + assert props["total_tokens"] == str(row["total_tokens"]) + assert props["visible_turns"] == str(row["visible_turns"]) + + +def test_huge_int_duration_degrades_not_crashes(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """An untyped row whose ``duration`` is a pathologically large JSON integer + must degrade to time="0.000", not crash report generation. + + ``math.isfinite(10**400)`` and ``f"{10**400:.3f}"`` both raise OverflowError; + a 400-digit integer survives JSON parsing into ``task_results`` (untyped + ``list[dict[str, Any]]``), so the writer must guard the conversion. + """ + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "t", "status": "SUCCESS", "duration": 10**400}]) + root = fromstring(generate_junit_xml(run_dir)) # must not raise + case = root.find("testsuite").find("testcase") + assert case.get("time") == "0.000" + + +def test_null_gating_fails_safe_as_gating(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A schema-skewed ``"gating": null`` must be treated as gating ([FAIL]), + not silently rendered informational — only an explicit ``false`` is INFO.""" + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "file_exists", + "description": "must exist", + "score": 0.0, + "pass_threshold": 0.9, + "gating": None, # schema skew — must NOT read as informational + "details": None, + "error": None, + } + ], + ) + body = _find_testsuite(fromstring(generate_junit_xml(run_dir)), "v1").find("testcase").find("failure").text or "" + assert "[FAIL]" in body + assert "[INFO]" not in body + + +def test_passing_informational_criterion_is_info_not_pass(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """An informational criterion that meets its threshold is still labelled + [INFO] (its nature), never [PASS] (which implies it participated in the + gate) — the label reflects gating, not the score.""" + run_dir = tmp_path / "run" + rows = [_row("t_fail", "FAILURE", variant_id="v1", replicate_index=0)] + write_run_json(run_dir, rows) + _write_task_json( + run_dir, + "v1", + "t_fail", + 0, + [ + { + "criterion_type": "commands_efficiency", + "description": "informational, passing", + "score": 1.0, + "pass_threshold": 0.9, + "gating": False, + "details": None, + "error": None, + } + ], + ) + body = _find_testsuite(fromstring(generate_junit_xml(run_dir)), "v1").find("testcase").find("failure").text or "" + assert "[INFO]" in body + assert "[PASS]" not in body + + +def test_windows_drive_task_id_degrades_to_status_body(write_run_json: Callable[..., Path], tmp_path: Path) -> None: + """A Windows drive-qualified task_id (``C:/x``) passes the nested-relpath shape + on POSIX but is rejected by the drive guard, so it degrades to a status body + instead of being resolved.""" + run_dir = tmp_path / "run" + write_run_json(run_dir, [{"task_id": "C:/evil", "status": "FAILURE", "variant_id": "v1", "replicate_index": 0}]) + body = _find_testsuite(fromstring(generate_junit_xml(run_dir)), "v1").find("testcase").find("failure").text or "" + assert "FAILURE" in body # status-only fallback, no crash