From eeb41df19d9916f722ae4bed3c4733cde3771774 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Tue, 14 Jul 2026 17:15:44 +0100 Subject: [PATCH] feat(validation): advisory-tier timeouts are YELLOW, not integrate-RED Carry an `advisory_timeouts` count through the stage report and validation_report; readiness raises a YELLOW reason (validation_advisory_timeout) when nonzero instead of RED. This lets mode=release gate on correctness rather than the perf-flake tail, so a correctness-clean run reaches its shippable GREEN/YELLOW. Docs + workspace-validation.yml updated. Mechanism for #74 (workspace advisory.yaml seeds follow). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/workspace-validation.yml | 17 ++++++-- docs/release_validation.md | 37 +++++++++++++++++ heart/readiness.py | 16 ++++++++ heart/validate.py | 40 +++++++++++++++++- tests/test_readiness.py | 21 ++++++++++ tests/test_validate.py | 48 ++++++++++++++++++++++ 6 files changed, 174 insertions(+), 5 deletions(-) diff --git a/.github/workflows/workspace-validation.yml b/.github/workflows/workspace-validation.yml index 0554918..d0965e4 100644 --- a/.github/workflows/workspace-validation.yml +++ b/.github/workflows/workspace-validation.yml @@ -301,10 +301,15 @@ jobs: # mode=release runs REAL searches (PYAUTO_TEST_MODE=0) and finite-difference # JAX gradient scripts, which legitimately exceed the 300s smoke cap # (build_util.py BUILD_SCRIPT_TIMEOUT default). Raise it here so a slow-but- - # correct script isn't a false release blocker. This is a stopgap while the - # Profiling Agent drives the JAX compile/eval-time speedups (PyAutoMind - # draft/feature/profiling/profiling_agent_jax_compile_time_scope.md); smoke - # mode is unchanged (still 300s). + # correct script isn't a false release blocker; smoke mode is unchanged + # (still 300s). A raised cap alone does NOT converge (the flaky set shifts + # run-to-run around whatever cap is chosen), so the advisory tier is the + # real fix: a timeout on a script listed in a workspace's + # config/build/advisory.yaml is recorded as `timeout_advisory` — reported + # and folded into a Heart YELLOW reason, but NOT release-blocking — while a + # timeout on any UNdeclared script still fails the run (RED). Durable + # speedups remain the Profiling Agent's job (PyAutoMind + # draft/feature/profiling/profiling_agent_jax_compile_time_scope.md). BUILD_SCRIPT_TIMEOUT: "1800" run: | # Deliberately NOT adding library source dirs to PYTHONPATH (closes @@ -541,6 +546,10 @@ jobs: if: always() run: | if [ -f stage_report.json ]; then + adv="$(python3 -c "import json;print(json.load(open('stage_report.json')).get('advisory_timeouts',0))" 2>/dev/null || echo 0)" + if [ "$adv" != "0" ]; then + echo "> ⚠️ $adv advisory-tier (declared-slow real-search) timeout(s) — reported as YELLOW, NOT release-blocking. A timeout on an undeclared script would still be RED." >> "$GITHUB_STEP_SUMMARY" + fi { echo '```json' cat stage_report.json diff --git a/docs/release_validation.md b/docs/release_validation.md index 561b6e3..fe6e479 100644 --- a/docs/release_validation.md +++ b/docs/release_validation.md @@ -121,3 +121,40 @@ dispatching `workspace-validation.yml` in `mode: release`), an ingested rehearsal-only report still (correctly) gates YELLOW: the source was built and TestPyPI-installed, but not yet exercised at release fidelity. `mode: release` is what supplies the `integrate` stage that flips this to GREEN-eligible. + +## The advisory tier (gate on correctness, not the perf-flake tail) + +`mode: release` runs the FULL script set, and a *population* of genuinely-slow +**real-search** scripts (`PYAUTO_TEST_MODE=0`/`1` real Nautilus fits + +finite-difference JAX gradients) flakes around the per-script timeout cap — the +failing set shifts run to run under CI load. Left ungated, one such timeout flips +the whole `integrate` stage to RED, conflating **correctness** with +**perf-flakiness**. Raising `BUILD_SCRIPT_TIMEOUT` and adding `no_run` SLOW-skips +one script at a time does not converge (and SLOW-skipping erodes coverage — the +script stops running at all). + +The **advisory tier** separates the two axes: + +- A workspace declares known-slow real-search scripts in a companion + **`config/build/advisory.yaml`** — same flat-list + inline-comment format and + path-matching semantics as `no_run.yaml`, marker `# ADVISORY - + `. Unlike a `no_run` SLOW-skip, an advisory-listed script **still runs + and is reported** — only its *timeout* is de-gated. Growing the list therefore + never erodes coverage, so it converges where SLOW-skips whack-a-mole. +- Build's runner (`run_python.py` → `build_util.py`) records a timeout on an + advisory-listed script as `timeout_advisory` (result_collector `Status`), not + `timeout`. `aggregate_results.py` keeps `timeout_advisory` **out** of the + failure set, so `ready` (and the stage's pass/fail) is unaffected, and surfaces + the count in an "Advisory-Tier Timeouts" report section. +- `heart/validate.py` carries the count into the report as `advisory_timeouts`; + `heart/readiness.py` raises a **YELLOW** reason when it is nonzero + (`validation_advisory_timeout`). This is what lets a correctness-clean release + reach its shippable **GREEN/YELLOW** instead of RED-blocking on the perf tail. +- A timeout on any **undeclared** script is a plain `timeout` and still **fails** + the stage (RED) — a script that should be fast timing out is a real signal. + Only the declared-slow population is de-gated, and only its *timeouts* (a + genuine exception on an advisory script is still a failure). + +The advisory list is a tracked backlog, not a hiding place: it surfaces in the +aggregate report, the CI step summary, and Heart's YELLOW reasons. Durable +speedups that let entries be removed are the Profiling Agent's job. diff --git a/heart/readiness.py b/heart/readiness.py index b70e5e4..d6d853b 100644 --- a/heart/readiness.py +++ b/heart/readiness.py @@ -132,6 +132,10 @@ "validation_unknown": (12, 12), "validation_profile": (12, 12), "validation_stale": (10, 10), + # Advisory-tier (declared-slow real-search) timeouts in the release + # validation: a YELLOW caution, not a blocker — the perf tail is de-gated + # from RED so the release can reach its shippable yellow. + "validation_advisory_timeout": (8, 8), } @@ -396,6 +400,18 @@ def scope_local(msg: str, key: str) -> None: # YELLOW axis — a passing-but-stale report is a caution, not a blocker. vr = snapshot.get("validation_report") if isinstance(vr, dict) and vr: + # Advisory-tier timeouts are a YELLOW caution regardless of the pass/fail + # axis: declared-slow real-search scripts that timed out but are NOT + # release-blocking (the mode=release perf-flake tail, de-gated from RED). + # Kept loud + tracked here rather than silently hidden; red still + # dominates yellow structurally if a real stage also failed. + advisory_timeouts = _as_int(vr.get("advisory_timeouts", 0)) + if advisory_timeouts > 0: + yellow.append( + f"{advisory_timeouts} advisory-tier script timeout(s) in release " + "validation (slow real-search; not release-blocking)" + ) + hit("validation_advisory_timeout") ready = vr.get("release_ready") if ready is False: failed_stages = [ diff --git a/heart/validate.py b/heart/validate.py index 1d78173..dd6321b 100644 --- a/heart/validate.py +++ b/heart/validate.py @@ -48,6 +48,7 @@ "integrate": {"status": "pass", "profile": "release", "run_url": "..."} }, "totals": {"passed": N, "failed": N, "skipped": N, "timeout": N}, + "advisory_timeouts": N, # declared-slow real-search timeouts (YELLOW, not RED) "per_project": { # per-workspace pass/fail/skip/timeout "autolens_workspace": {"passed": .., "failed": .., ...}, "autolens_workspace_test": {"passed": .., "failed": .., ...} @@ -59,6 +60,14 @@ "ts": "2026-06-30T12:00:00+00:00" } +``advisory_timeouts`` counts declared-slow real-search scripts that timed out +but are workspace-declared advisory-tier (``config/build/advisory.yaml``): they +still ran and are reported, but their timeout is advisory, so it does NOT flip +``release_ready``. The readiness gate raises a YELLOW reason when it is nonzero — +this is what lets ``mode=release`` reach its shippable yellow instead of +RED-blocking on the perf-flake tail. A timeout on an UNdeclared script is a +plain ``timeout`` and still fails the stage. + ``release_ready`` is the **pass/fail** axis only: it is ``false`` if any ran stage failed. Release *fidelity* and *freshness* (``profile == release``, ``commit_shas`` matching the current ``main`` HEADs, age) are judged separately @@ -187,6 +196,9 @@ def __init__(self) -> None: self.totals: dict[str, int] = {k: 0 for k in _COUNT_KEYS} self.per_project: dict[str, dict[str, int]] = {} self.failures: list[dict[str, Any]] = [] + # Advisory-tier timeouts across stages: not a failure axis (see + # add_stage), surfaced by readiness as a YELLOW reason. + self.advisory_timeouts: int = 0 self.run_urls: dict[str, str] = {} self._explicit_ready: bool | None = None # True once a real stage artifact (add_stage) has contributed counts. @@ -249,6 +261,9 @@ def add_stage(self, data: dict[str, Any]) -> None: for f in data.get("failures", []) or []: if isinstance(f, dict): self.failures.append(f) + adv = data.get("advisory_timeouts") + if isinstance(adv, (int, float)): + self.advisory_timeouts += int(adv) self.add_commit_shas(data.get("commit_shas")) def add_commit_shas(self, shas: Any) -> None: @@ -288,6 +303,9 @@ def add_report(self, data: dict[str, Any]) -> None: for f in data.get("failures", []) or []: if isinstance(f, dict): self.failures.append(f) + adv = data.get("advisory_timeouts") + if isinstance(adv, (int, float)): + self.advisory_timeouts += int(adv) for k, v in (data.get("run_urls") or {}).items(): self.run_urls.setdefault(str(k), str(v)) if isinstance(data.get("release_ready"), bool): @@ -376,6 +394,7 @@ def ingest( "totals": acc.totals, "per_project": acc.per_project, "failures": acc.failures, + "advisory_timeouts": acc.advisory_timeouts, "run_urls": acc.run_urls, "ts": _now_iso(now), } @@ -433,6 +452,21 @@ def to_stage_report( if isinstance(f, dict): failures.append(dict(f)) + # Advisory-tier timeouts are NOT failures (Build's aggregate already keeps + # them out of `ready`): a timeout on a workspace-declared advisory-tier + # (known-slow real-search) script is advisory, not release-blocking. We carry + # the COUNT through so readiness can raise a YELLOW reason — the release + # reaches its shippable yellow instead of RED-blocking on the perf-flake tail + # (PyAutoHeart mode=release advisory-tiering). Fall back to the summary count + # if the aggregate predates the explicit list. + advisory_raw = aggregate.get("advisory_timeouts") + if isinstance(advisory_raw, list): + advisory_count = len(advisory_raw) + else: + summary_counts = aggregate.get("summary") + summary_counts = summary_counts if isinstance(summary_counts, dict) else {} + advisory_count = int(summary_counts.get("timeout_advisory", 0) or 0) + # Strict boolean check (not truthiness): this is a CI contract, so a # malformed/non-boolean "ready" (e.g. a stray string "false", which is # truthy in Python) must never be read as a pass. @@ -450,6 +484,8 @@ def to_stage_report( report["summary"] = summary report["per_project"] = per_project report["failures"] = failures + if advisory_count: + report["advisory_timeouts"] = advisory_count return report @@ -512,12 +548,14 @@ def _print_summary(report: dict[str, Any]) -> None: stages = ", ".join(f"{n}:{s.get('status', '?')}" for n, s in (report.get("stages") or {}).items()) version = report.get("testpypi_version") or "?" prof = report.get("profile") or "?" + adv = int(report.get("advisory_timeouts", 0) or 0) + adv_str = f" advisory-timeouts: {adv}" if adv else "" print(f"{glyph} {c_info('validate')} {label} {c_meta(f'v{version} profile={prof}')}") print( c_meta( f" stages: {stages or 'none'} " f"totals: {t.get('passed', 0)}p/{t.get('failed', 0)}f/" - f"{t.get('skipped', 0)}s/{t.get('timeout', 0)}t" + f"{t.get('skipped', 0)}s/{t.get('timeout', 0)}t{adv_str}" ) ) diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 7049e70..94a3dfd 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -220,6 +220,27 @@ def test_validation_failed_is_red(): assert v["score"] == 60 # validation_failed penalty 40 +def test_validation_advisory_timeouts_are_yellow_not_red(): + # Declared-slow real-search timeouts de-gate the mode=release perf tail: a + # fresh, source-matching, passing report that carries advisory_timeouts is + # YELLOW (the release's shippable resting state), never RED. + report = _green_validation_report() + report["advisory_timeouts"] = 3 + v = compute(make_snapshot(validation_report=report)) + assert v["verdict"] == "yellow" + assert not v["red_reasons"] + assert any("advisory-tier script timeout(s)" in r and "not release-blocking" in r + for r in v["yellow_reasons"]) + + +def test_validation_zero_advisory_timeouts_stays_green(): + report = _green_validation_report() + report["advisory_timeouts"] = 0 + v = compute(make_snapshot(validation_report=report)) + assert v["verdict"] == "green" + assert not any("advisory-tier" in r for r in v["reasons"]) + + def test_validation_stale_by_sha_is_stale_tier(): # A report whose commit_shas no longer match the current main HEADs is # expired evidence: the source moved on since the rehearsal → freshness diff --git a/tests/test_validate.py b/tests/test_validate.py index 82be781..0e37a6e 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -363,6 +363,54 @@ def test_to_stage_report_is_ingestable(tmp_path): assert report["totals"] == {"passed": 58, "failed": 0, "skipped": 2, "timeout": 0} +# --- advisory tier: declared-slow real-search timeouts are YELLOW, not RED --- + +AGGREGATE_ADVISORY = { + "ready": True, # Build already keeps timeout_advisory out of the failure set + "summary": {"passed": 55, "failed": 0, "skipped": 2, "timeout": 0, "timeout_advisory": 3}, + "per_project": {"autolens_workspace_test": {"passed": 55, "failed": 0, "skipped": 2, "timeout": 0}}, + "failures": [], + "advisory_timeouts": [ + {"project": "autolens_workspace_test", "file": "scripts/multi/shared_preloads.py"}, + {"project": "autolens_workspace_test", "file": "scripts/interferometer/delaunay.py"}, + {"project": "autolens_workspace_test", "file": "scripts/jax_grad/imaging_lp.py"}, + ], +} + + +def test_to_stage_report_carries_advisory_timeout_count(): + """An advisory-only aggregate still passes, but carries the advisory count.""" + report = validate.to_stage_report(AGGREGATE_ADVISORY, stage="integrate", profile="release") + assert report["status"] == "pass" + assert report["advisory_timeouts"] == 3 + # Advisory timeouts are NOT folded into the failure list. + assert report["failures"] == [] + + +def test_to_stage_report_advisory_falls_back_to_summary_count(): + """If the aggregate predates the explicit list, use the summary count.""" + aggregate = { + "ready": True, + "summary": {"passed": 55, "failed": 0, "skipped": 0, "timeout": 0, "timeout_advisory": 2}, + "per_project": {}, + "failures": [], + } + report = validate.to_stage_report(aggregate, stage="integrate") + assert report["advisory_timeouts"] == 2 + + +def test_ingest_carries_advisory_timeouts_to_report(tmp_path): + stage_report = validate.to_stage_report( + AGGREGATE_ADVISORY, stage="integrate", profile="release", + version="2026.6.30.1.dev64501", commit_shas=SHAS, + ) + _write(tmp_path / "integrate.json", stage_report) + _write(tmp_path / "rehearsal.json", REHEARSAL) + report = validate.ingest([tmp_path]) + assert report["release_ready"] is True # advisory timeouts do NOT block + assert report["advisory_timeouts"] == 3 + + # --- CLI: --emit-stage-report ------------------------------------------------