feat(validation): mode=release advisory tier — gate on correctness, not the perf-flake tail

.github/workflows/workspace-validation.yml

@@ -301,10 +301,15 @@ jobs:
           # mode=release runs REAL searches (PYAUTO_TEST_MODE=0) and finite-difference
           # JAX gradient scripts, which legitimately exceed the 300s smoke cap
           # (build_util.py BUILD_SCRIPT_TIMEOUT default). Raise it here so a slow-but-
-          # correct script isn't a false release blocker. This is a stopgap while the
-          # Profiling Agent drives the JAX compile/eval-time speedups (PyAutoMind
-          # draft/feature/profiling/profiling_agent_jax_compile_time_scope.md); smoke
-          # mode is unchanged (still 300s).
+          # correct script isn't a false release blocker; smoke mode is unchanged
+          # (still 300s). A raised cap alone does NOT converge (the flaky set shifts
+          # run-to-run around whatever cap is chosen), so the advisory tier is the
+          # real fix: a timeout on a script listed in a workspace's
+          # config/build/advisory.yaml is recorded as `timeout_advisory` — reported
+          # and folded into a Heart YELLOW reason, but NOT release-blocking — while a
+          # timeout on any UNdeclared script still fails the run (RED). Durable
+          # speedups remain the Profiling Agent's job (PyAutoMind
+          # draft/feature/profiling/profiling_agent_jax_compile_time_scope.md).
           BUILD_SCRIPT_TIMEOUT: "1800"
         run: |
           # Deliberately NOT adding library source dirs to PYTHONPATH (closes
@@ -541,6 +546,10 @@ jobs:
         if: always()
         run: |
           if [ -f stage_report.json ]; then
+            adv="$(python3 -c "import json;print(json.load(open('stage_report.json')).get('advisory_timeouts',0))" 2>/dev/null || echo 0)"
+            if [ "$adv" != "0" ]; then
+              echo "> ⚠️ $adv advisory-tier (declared-slow real-search) timeout(s) — reported as YELLOW, NOT release-blocking. A timeout on an undeclared script would still be RED." >> "$GITHUB_STEP_SUMMARY"
+            fi
             {
               echo '```json'
               cat stage_report.json

docs/release_validation.md

@@ -121,3 +121,40 @@ dispatching `workspace-validation.yml` in `mode: release`), an ingested
 rehearsal-only report still (correctly) gates YELLOW: the source was built and
 TestPyPI-installed, but not yet exercised at release fidelity. `mode: release`
 is what supplies the `integrate` stage that flips this to GREEN-eligible.
+
+## The advisory tier (gate on correctness, not the perf-flake tail)
+
+`mode: release` runs the FULL script set, and a *population* of genuinely-slow
+**real-search** scripts (`PYAUTO_TEST_MODE=0`/`1` real Nautilus fits +
+finite-difference JAX gradients) flakes around the per-script timeout cap — the
+failing set shifts run to run under CI load. Left ungated, one such timeout flips
+the whole `integrate` stage to RED, conflating **correctness** with
+**perf-flakiness**. Raising `BUILD_SCRIPT_TIMEOUT` and adding `no_run` SLOW-skips
+one script at a time does not converge (and SLOW-skipping erodes coverage — the
+script stops running at all).
+
+The **advisory tier** separates the two axes:
+
+- A workspace declares known-slow real-search scripts in a companion
+  **`config/build/advisory.yaml`** — same flat-list + inline-comment format and
+  path-matching semantics as `no_run.yaml`, marker `# ADVISORY <YYYY-MM-DD> -
+  <reason>`. Unlike a `no_run` SLOW-skip, an advisory-listed script **still runs
+  and is reported** — only its *timeout* is de-gated. Growing the list therefore
+  never erodes coverage, so it converges where SLOW-skips whack-a-mole.
+- Build's runner (`run_python.py` → `build_util.py`) records a timeout on an
+  advisory-listed script as `timeout_advisory` (result_collector `Status`), not
+  `timeout`. `aggregate_results.py` keeps `timeout_advisory` **out** of the
+  failure set, so `ready` (and the stage's pass/fail) is unaffected, and surfaces
+  the count in an "Advisory-Tier Timeouts" report section.
+- `heart/validate.py` carries the count into the report as `advisory_timeouts`;
+  `heart/readiness.py` raises a **YELLOW** reason when it is nonzero
+  (`validation_advisory_timeout`). This is what lets a correctness-clean release
+  reach its shippable **GREEN/YELLOW** instead of RED-blocking on the perf tail.
+- A timeout on any **undeclared** script is a plain `timeout` and still **fails**
+  the stage (RED) — a script that should be fast timing out is a real signal.
+  Only the declared-slow population is de-gated, and only its *timeouts* (a
+  genuine exception on an advisory script is still a failure).
+
+The advisory list is a tracked backlog, not a hiding place: it surfaces in the
+aggregate report, the CI step summary, and Heart's YELLOW reasons. Durable
+speedups that let entries be removed are the Profiling Agent's job.

heart/readiness.py

@@ -132,6 +132,10 @@
     "validation_unknown": (12, 12),
     "validation_profile": (12, 12),
     "validation_stale": (10, 10),
+    # Advisory-tier (declared-slow real-search) timeouts in the release
+    # validation: a YELLOW caution, not a blocker — the perf tail is de-gated
+    # from RED so the release can reach its shippable yellow.
+    "validation_advisory_timeout": (8, 8),
 }
 
 
@@ -396,6 +400,18 @@ def scope_local(msg: str, key: str) -> None:
     # YELLOW axis — a passing-but-stale report is a caution, not a blocker.
     vr = snapshot.get("validation_report")
     if isinstance(vr, dict) and vr:
+        # Advisory-tier timeouts are a YELLOW caution regardless of the pass/fail
+        # axis: declared-slow real-search scripts that timed out but are NOT
+        # release-blocking (the mode=release perf-flake tail, de-gated from RED).
+        # Kept loud + tracked here rather than silently hidden; red still
+        # dominates yellow structurally if a real stage also failed.
+        advisory_timeouts = _as_int(vr.get("advisory_timeouts", 0))
+        if advisory_timeouts > 0:
+            yellow.append(
+                f"{advisory_timeouts} advisory-tier script timeout(s) in release "
+                "validation (slow real-search; not release-blocking)"
+            )
+            hit("validation_advisory_timeout")
         ready = vr.get("release_ready")
         if ready is False:
             failed_stages = [

heart/validate.py

@@ -48,6 +48,7 @@
         "integrate": {"status": "pass", "profile": "release", "run_url": "..."}
       },
       "totals": {"passed": N, "failed": N, "skipped": N, "timeout": N},
+      "advisory_timeouts": N,           # declared-slow real-search timeouts (YELLOW, not RED)
       "per_project": {                  # per-workspace pass/fail/skip/timeout
         "autolens_workspace":      {"passed": .., "failed": .., ...},
         "autolens_workspace_test": {"passed": .., "failed": .., ...}
@@ -59,6 +60,14 @@
       "ts": "2026-06-30T12:00:00+00:00"
     }
 
+``advisory_timeouts`` counts declared-slow real-search scripts that timed out
+but are workspace-declared advisory-tier (``config/build/advisory.yaml``): they
+still ran and are reported, but their timeout is advisory, so it does NOT flip
+``release_ready``. The readiness gate raises a YELLOW reason when it is nonzero —
+this is what lets ``mode=release`` reach its shippable yellow instead of
+RED-blocking on the perf-flake tail. A timeout on an UNdeclared script is a
+plain ``timeout`` and still fails the stage.
+
 ``release_ready`` is the **pass/fail** axis only: it is ``false`` if any ran
 stage failed. Release *fidelity* and *freshness* (``profile == release``,
 ``commit_shas`` matching the current ``main`` HEADs, age) are judged separately
@@ -187,6 +196,9 @@ def __init__(self) -> None:
         self.totals: dict[str, int] = {k: 0 for k in _COUNT_KEYS}
         self.per_project: dict[str, dict[str, int]] = {}
         self.failures: list[dict[str, Any]] = []
+        # Advisory-tier timeouts across stages: not a failure axis (see
+        # add_stage), surfaced by readiness as a YELLOW reason.
+        self.advisory_timeouts: int = 0
         self.run_urls: dict[str, str] = {}
         self._explicit_ready: bool | None = None
         # True once a real stage artifact (add_stage) has contributed counts.
@@ -249,6 +261,9 @@ def add_stage(self, data: dict[str, Any]) -> None:
         for f in data.get("failures", []) or []:
             if isinstance(f, dict):
                 self.failures.append(f)
+        adv = data.get("advisory_timeouts")
+        if isinstance(adv, (int, float)):
+            self.advisory_timeouts += int(adv)
         self.add_commit_shas(data.get("commit_shas"))
 
     def add_commit_shas(self, shas: Any) -> None:
@@ -288,6 +303,9 @@ def add_report(self, data: dict[str, Any]) -> None:
             for f in data.get("failures", []) or []:
                 if isinstance(f, dict):
                     self.failures.append(f)
+            adv = data.get("advisory_timeouts")
+            if isinstance(adv, (int, float)):
+                self.advisory_timeouts += int(adv)
         for k, v in (data.get("run_urls") or {}).items():
             self.run_urls.setdefault(str(k), str(v))
         if isinstance(data.get("release_ready"), bool):
@@ -376,6 +394,7 @@ def ingest(
         "totals": acc.totals,
         "per_project": acc.per_project,
         "failures": acc.failures,
+        "advisory_timeouts": acc.advisory_timeouts,
         "run_urls": acc.run_urls,
         "ts": _now_iso(now),
     }
@@ -433,6 +452,21 @@ def to_stage_report(
         if isinstance(f, dict):
             failures.append(dict(f))
 
+    # Advisory-tier timeouts are NOT failures (Build's aggregate already keeps
+    # them out of `ready`): a timeout on a workspace-declared advisory-tier
+    # (known-slow real-search) script is advisory, not release-blocking. We carry
+    # the COUNT through so readiness can raise a YELLOW reason — the release
+    # reaches its shippable yellow instead of RED-blocking on the perf-flake tail
+    # (PyAutoHeart mode=release advisory-tiering). Fall back to the summary count
+    # if the aggregate predates the explicit list.
+    advisory_raw = aggregate.get("advisory_timeouts")
+    if isinstance(advisory_raw, list):
+        advisory_count = len(advisory_raw)
+    else:
+        summary_counts = aggregate.get("summary")
+        summary_counts = summary_counts if isinstance(summary_counts, dict) else {}
+        advisory_count = int(summary_counts.get("timeout_advisory", 0) or 0)
+
     # Strict boolean check (not truthiness): this is a CI contract, so a
     # malformed/non-boolean "ready" (e.g. a stray string "false", which is
     # truthy in Python) must never be read as a pass.
@@ -450,6 +484,8 @@ def to_stage_report(
     report["summary"] = summary
     report["per_project"] = per_project
     report["failures"] = failures
+    if advisory_count:
+        report["advisory_timeouts"] = advisory_count
     return report
 
 
@@ -512,12 +548,14 @@ def _print_summary(report: dict[str, Any]) -> None:
     stages = ", ".join(f"{n}:{s.get('status', '?')}" for n, s in (report.get("stages") or {}).items())
     version = report.get("testpypi_version") or "?"
     prof = report.get("profile") or "?"
+    adv = int(report.get("advisory_timeouts", 0) or 0)
+    adv_str = f"  advisory-timeouts: {adv}" if adv else ""
     print(f"{glyph} {c_info('validate')} {label} {c_meta(f'v{version}  profile={prof}')}")
     print(
         c_meta(
             f"  stages: {stages or 'none'}  "
             f"totals: {t.get('passed', 0)}p/{t.get('failed', 0)}f/"
-            f"{t.get('skipped', 0)}s/{t.get('timeout', 0)}t"
+            f"{t.get('skipped', 0)}s/{t.get('timeout', 0)}t{adv_str}"
         )
     )
 

tests/test_readiness.py

@@ -220,6 +220,27 @@ def test_validation_failed_is_red():
     assert v["score"] == 60  # validation_failed penalty 40
 
 
+def test_validation_advisory_timeouts_are_yellow_not_red():
+    # Declared-slow real-search timeouts de-gate the mode=release perf tail: a
+    # fresh, source-matching, passing report that carries advisory_timeouts is
+    # YELLOW (the release's shippable resting state), never RED.
+    report = _green_validation_report()
+    report["advisory_timeouts"] = 3
+    v = compute(make_snapshot(validation_report=report))
+    assert v["verdict"] == "yellow"
+    assert not v["red_reasons"]
+    assert any("advisory-tier script timeout(s)" in r and "not release-blocking" in r
+               for r in v["yellow_reasons"])
+
+
+def test_validation_zero_advisory_timeouts_stays_green():
+    report = _green_validation_report()
+    report["advisory_timeouts"] = 0
+    v = compute(make_snapshot(validation_report=report))
+    assert v["verdict"] == "green"
+    assert not any("advisory-tier" in r for r in v["reasons"])
+
+
 def test_validation_stale_by_sha_is_stale_tier():
     # A report whose commit_shas no longer match the current main HEADs is
     # expired evidence: the source moved on since the rehearsal → freshness

tests/test_validate.py

@@ -363,6 +363,54 @@ def test_to_stage_report_is_ingestable(tmp_path):
     assert report["totals"] == {"passed": 58, "failed": 0, "skipped": 2, "timeout": 0}
 
 
+# --- advisory tier: declared-slow real-search timeouts are YELLOW, not RED ---
+
+AGGREGATE_ADVISORY = {
+    "ready": True,  # Build already keeps timeout_advisory out of the failure set
+    "summary": {"passed": 55, "failed": 0, "skipped": 2, "timeout": 0, "timeout_advisory": 3},
+    "per_project": {"autolens_workspace_test": {"passed": 55, "failed": 0, "skipped": 2, "timeout": 0}},
+    "failures": [],
+    "advisory_timeouts": [
+        {"project": "autolens_workspace_test", "file": "scripts/multi/shared_preloads.py"},
+        {"project": "autolens_workspace_test", "file": "scripts/interferometer/delaunay.py"},
+        {"project": "autolens_workspace_test", "file": "scripts/jax_grad/imaging_lp.py"},
+    ],
+}
+
+
+def test_to_stage_report_carries_advisory_timeout_count():
+    """An advisory-only aggregate still passes, but carries the advisory count."""
+    report = validate.to_stage_report(AGGREGATE_ADVISORY, stage="integrate", profile="release")
+    assert report["status"] == "pass"
+    assert report["advisory_timeouts"] == 3
+    # Advisory timeouts are NOT folded into the failure list.
+    assert report["failures"] == []
+
+
+def test_to_stage_report_advisory_falls_back_to_summary_count():
+    """If the aggregate predates the explicit list, use the summary count."""
+    aggregate = {
+        "ready": True,
+        "summary": {"passed": 55, "failed": 0, "skipped": 0, "timeout": 0, "timeout_advisory": 2},
+        "per_project": {},
+        "failures": [],
+    }
+    report = validate.to_stage_report(aggregate, stage="integrate")
+    assert report["advisory_timeouts"] == 2
+
+
+def test_ingest_carries_advisory_timeouts_to_report(tmp_path):
+    stage_report = validate.to_stage_report(
+        AGGREGATE_ADVISORY, stage="integrate", profile="release",
+        version="2026.6.30.1.dev64501", commit_shas=SHAS,
+    )
+    _write(tmp_path / "integrate.json", stage_report)
+    _write(tmp_path / "rehearsal.json", REHEARSAL)
+    report = validate.ingest([tmp_path])
+    assert report["release_ready"] is True  # advisory timeouts do NOT block
+    assert report["advisory_timeouts"] == 3
+
+
 # --- CLI: --emit-stage-report ------------------------------------------------