diff --git a/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh b/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh index b4c650a26..16a2430a9 100644 --- a/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh +++ b/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh @@ -59,7 +59,7 @@ echo "Starting login process..." # Check if Docker credentials exist if [[ -n "$DSTACK_DOCKER_USERNAME" && -n "$DSTACK_DOCKER_PASSWORD" ]]; then echo "Docker credentials found" - DOCKER_REGISTRY_TARGET="${DSTACK_DOCKER_REGISTRY:-docker.io}" + DOCKER_REGISTRY_TARGET="${DSTACK_DOCKER_REGISTRY:-ghcr.io}" echo "Target Docker registry: $DOCKER_REGISTRY_TARGET" # Check if already logged in diff --git a/packages/challenges/agent-challenge/golden/live-registry-refs.json b/packages/challenges/agent-challenge/golden/live-registry-refs.json index 66b2f1715..d5457078d 100644 --- a/packages/challenges/agent-challenge/golden/live-registry-refs.json +++ b/packages/challenges/agent-challenge/golden/live-registry-refs.json @@ -1,25 +1,28 @@ { "schema": "harbor-independence/live-registry-refs@1", - "note": "SIDE manifest (NOT the frozen golden dataset digest). Maps a small deterministic subset of Terminal-Bench 2.1 task_ids to PULLABLE, digest-pinned registry refs published to the miner's public Docker Hub namespace so an in-CVM DooD orchestrator can `docker pull` them for a live smoke E2E. This file does NOT affect golden/dataset-digest.json, its per-task content_digest_sha256, the canonical_content_digest_sha256, or the canonical compose/measurement. Resolution is opt-in and fail-closed (see agent_challenge.canonical.live_registry).", + "note": "SIDE manifest (NOT the frozen golden dataset digest). Maps a small deterministic subset of Terminal-Bench 2.1 task_ids to PULLABLE, digest-pinned GHCR refs under ghcr.io/baseintelligence so an in-CVM DooD orchestrator can pull them for a live smoke E2E. D6: only GHCR shipping refs are accepted (see agent_challenge.canonical.live_registry). This file does NOT affect golden/dataset-digest.json, its per-task content_digest_sha256, the canonical_content_digest_sha256, or the canonical compose/measurement. Resolution is opt-in and fail-closed. OPS_REQUIRED: orchestrator_image digest is the T1 local buildx manifest (not yet pushed to GHCR); task image digests retain prior content hashes under the new GHCR path pattern and must be republished to GHCR before live pull succeeds.", "dataset": "terminal-bench/terminal-bench-2-1", - "namespace": "docker.io/mathiiss", - "orchestrator_image": "docker.io/mathiiss/agent-challenge-canonical@sha256:02331f0909f617e333f113be376d353770a673669946bcddaac3c53cbde7c9d8", - "orchestrator_image_source": "services.yaml build-canonical (`uv run python -m agent_challenge.canonical.build --build`) built reproducibly (BuildKit SOURCE_DATE_EPOCH + rewrite-timestamp, provenance/sbom off) and pushed to the miner Docker Hub namespace", + "namespace": "ghcr.io/baseintelligence", + "orchestrator_image": "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:ea399cee1b3c9015024918a6070901e6b7b4bee432c8afe1e7dd40a95547e0bc", + "orchestrator_image_source": "T1 local agent-recipe buildx (SOURCE_DATE_EPOCH + rewrite-timestamp, provenance/sbom off) → manifest sha256:ea399cee1b3c9015024918a6070901e6b7b4bee432c8afe1e7dd40a95547e0bc; tag was ghcr.io/baseintelligence/agent-challenge-canonical:t1-local. OPS_REQUIRED: replace with the first published GHCR digest from agent-recipe publish-eval-image.yml before T3/prod pin.", "tasks": { "adaptive-rejection-sampler": { - "registry_ref": "docker.io/mathiiss/agent-challenge-tb21-adaptive-rejection-sampler@sha256:7c8bd5835f19506222805de68d65f83d6cca5b502f7d71dc5e2d9d4dd447c0c8", + "registry_ref": "ghcr.io/baseintelligence/agent-challenge-tb21-adaptive-rejection-sampler@sha256:7c8bd5835f19506222805de68d65f83d6cca5b502f7d71dc5e2d9d4dd447c0c8", "source_ref": "alexgshaw/adaptive-rejection-sampler:20251031", - "content_digest_sha256": "bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de" + "content_digest_sha256": "bcaa2399985cd57666018025846289ab25e193ae0dd8fb7f0ffab2410c24d4de", + "ops_status": "BLOCKED_REPUBLISH — path retargeted to GHCR; digest is historical content pin until image is pushed under ghcr.io/baseintelligence/" }, "bn-fit-modify": { - "registry_ref": "docker.io/mathiiss/agent-challenge-tb21-bn-fit-modify@sha256:c0371862a0861f282eb206471452ea9aa8d494e3687bdc8167c96ab39d73db50", + "registry_ref": "ghcr.io/baseintelligence/agent-challenge-tb21-bn-fit-modify@sha256:c0371862a0861f282eb206471452ea9aa8d494e3687bdc8167c96ab39d73db50", "source_ref": "alexgshaw/bn-fit-modify:20251031", - "content_digest_sha256": "b5f9644970c17ad9ddb46b7266f7bcd87c761d77d7e6f55d7cfe7284d5ff66e9" + "content_digest_sha256": "b5f9644970c17ad9ddb46b7266f7bcd87c761d77d7e6f55d7cfe7284d5ff66e9", + "ops_status": "BLOCKED_REPUBLISH — path retargeted to GHCR; digest is historical content pin until image is pushed under ghcr.io/baseintelligence/" }, "break-filter-js-from-html": { - "registry_ref": "docker.io/mathiiss/agent-challenge-tb21-break-filter-js-from-html@sha256:2a5bd51bab582993befc1a24252c03af673cb1db7cf7d0b2195d66a42a3872a7", + "registry_ref": "ghcr.io/baseintelligence/agent-challenge-tb21-break-filter-js-from-html@sha256:2a5bd51bab582993befc1a24252c03af673cb1db7cf7d0b2195d66a42a3872a7", "source_ref": "alexgshaw/break-filter-js-from-html:20251031", - "content_digest_sha256": "678008d1a4fd1e6e1b9b3cc9a327219fe4b410a31eafc52e9099bbf947eea600" + "content_digest_sha256": "678008d1a4fd1e6e1b9b3cc9a327219fe4b410a31eafc52e9099bbf947eea600", + "ops_status": "BLOCKED_REPUBLISH — path retargeted to GHCR; digest is historical content pin until image is pushed under ghcr.io/baseintelligence/" } } } diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py index afe7987bf..e04250f89 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py @@ -4,12 +4,15 @@ Terminal-Bench task by a *bare content digest* (``harbor_registry_ref = "sha256:<64hex>"``) with NO repository, so an in-CVM DooD orchestrator cannot ``docker pull`` it. For a live smoke E2E a small deterministic subset of task -images is published to the miner's public Docker Hub namespace as pullable, -digest-pinned refs and recorded in a SEPARATE side manifest +images is published to the org GHCR namespace (``ghcr.io/baseintelligence/…``) +as pullable, digest-pinned refs and recorded in a SEPARATE side manifest (``golden/live-registry-refs.json``). That side manifest never touches ``dataset-digest.json`` -- the frozen content digests and the canonical measurement stay byte-identical. +D6: shipping live refs MUST be GHCR. ``docker.io`` and the retired +``mathiiss`` namespace are rejected fail-closed at parse time. + Resolution is **opt-in and fail-closed**: with no live manifest configured (no explicit path, no env var) callers get NO live refs and fall back to the existing per-task behavior, so flag-off / offline runs are byte-identical. When a @@ -50,6 +53,9 @@ # and an un-namespaced ``name@sha256`` so only a real registry ref is accepted. _PULLABLE_REF_RE = re.compile(r"^(?=[^@]*/)[A-Za-z0-9][\w.\-/:]*@sha256:[0-9a-f]{64}$") +#: Shipping live-registry refs must live under this registry host prefix (D6). +LIVE_REGISTRY_HOST_PREFIX = "ghcr.io/" + class LiveRegistryError(ValueError): """The live-registry side manifest is missing, malformed, or not pullable.""" @@ -83,6 +89,32 @@ def assert_pullable_ref(ref: Any, *, what: str = "registry ref") -> str: return ref # type: ignore[return-value] +def assert_live_registry_ref(ref: Any, *, what: str = "registry ref") -> str: + """Return ``ref`` if it is a GHCR digest-pinned shipping ref (D6), else raise. + + Builds on :func:`assert_pullable_ref`, then rejects ``docker.io``, the + retired ``mathiiss`` namespace, and any non-GHCR host so live smoke never + pins a Hub image again. + """ + + pinned = assert_pullable_ref(ref, what=what) + lowered = pinned.lower() + if "mathiiss" in lowered: + raise LiveRegistryError( + f"{what} must not use the retired mathiiss namespace (D6), got {pinned!r}" + ) + if lowered.startswith("docker.io/") or lowered.startswith("index.docker.io/"): + raise LiveRegistryError( + f"{what} must not use docker.io (D6 — GHCR only), got {pinned!r}" + ) + if not lowered.startswith(LIVE_REGISTRY_HOST_PREFIX): + raise LiveRegistryError( + f"{what} must be a GHCR digest-pinned ref " + f"({LIVE_REGISTRY_HOST_PREFIX}…@sha256:<64hex>), got {pinned!r}" + ) + return pinned + + @dataclass(frozen=True) class LiveRegistry: """Parsed live-registry side manifest. @@ -111,7 +143,7 @@ def __bool__(self) -> bool: def _ref_from_entry(task_id: str, entry: Any) -> str: - """Extract + validate the pullable ref from a manifest task entry.""" + """Extract + validate the pullable GHCR ref from a manifest task entry.""" if isinstance(entry, str): ref = entry @@ -119,15 +151,16 @@ def _ref_from_entry(task_id: str, entry: Any) -> str: ref = entry.get("registry_ref") else: raise LiveRegistryError(f"live-registry task {task_id!r} is not a string or mapping") - return assert_pullable_ref(ref, what=f"live-registry ref for task {task_id!r}") + return assert_live_registry_ref(ref, what=f"live-registry ref for task {task_id!r}") def parse_live_registry(data: Mapping[str, Any]) -> LiveRegistry: """Parse + validate a live-registry side-manifest document. - Every task ref must be a pullable ``repo@sha256`` ref (a bare content digest - or floating tag is rejected). ``orchestrator_image``, when present, must be - pullable too. Task keys are normalized to their bare name. + Every task ref must be a GHCR pullable ``repo@sha256`` ref (a bare content + digest, floating tag, ``docker.io``, or retired ``mathiiss`` namespace is + rejected). ``orchestrator_image``, when present, must be GHCR-pullable too. + Task keys are normalized to their bare name. """ if not isinstance(data, Mapping): @@ -143,7 +176,7 @@ def parse_live_registry(data: Mapping[str, Any]) -> LiveRegistry: orchestrator = data.get("orchestrator_image") if orchestrator is not None: - orchestrator = assert_pullable_ref(orchestrator, what="orchestrator_image") + orchestrator = assert_live_registry_ref(orchestrator, what="orchestrator_image") return LiveRegistry(orchestrator_image=orchestrator, task_refs=task_refs, raw=dict(data)) @@ -203,9 +236,11 @@ def resolve_live_registry_refs( "DEFAULT_LIVE_REGISTRY_PATH", "LIVE_REGISTRY_ENV", "LIVE_REGISTRY_FILENAME", + "LIVE_REGISTRY_HOST_PREFIX", "LIVE_REGISTRY_SCHEMA", "LiveRegistry", "LiveRegistryError", + "assert_live_registry_ref", "assert_pullable_ref", "is_pullable_ref", "load_live_registry", diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/direct_result.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/direct_result.py index 0e0fabd37..eddadc2cc 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/direct_result.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/direct_result.py @@ -686,6 +686,18 @@ async def _load_review_envelope_for_run( ) -> Mapping[str, Any] | str | None: """Load receipted review-domain envelope for score-chain re-verify.""" + materials = await _load_review_materials_for_run(session, run) + if materials is None: + return None + return materials.get("envelope") + + +async def _load_review_materials_for_run( + session: AsyncSession, + run: EvalRun, +) -> dict[str, Any] | None: + """Load envelope + verification outcome (package residual) for score chain.""" + submission = await session.scalar( select(AgentSubmission).where(AgentSubmission.id == run.submission_id) ) @@ -703,9 +715,32 @@ async def _load_review_envelope_for_run( ): return None envelope = assignment.review_report_envelope_json + outcome_raw = assignment.review_verification_outcome_json + outcome: Mapping[str, Any] | None = None + if isinstance(outcome_raw, str) and outcome_raw.strip(): + try: + import json as _json + + parsed = _json.loads(outcome_raw) + if isinstance(parsed, dict): + outcome = parsed + except (TypeError, ValueError): + outcome = None + elif isinstance(outcome_raw, dict): + outcome = outcome_raw + residual = None + if isinstance(outcome, Mapping): + pr = outcome.get("package_residual") + if isinstance(pr, dict): + residual = pr + env_out: Mapping[str, Any] | str | None = None if isinstance(envelope, str) and envelope: - return envelope - return None + env_out = envelope + elif isinstance(envelope, dict): + env_out = envelope + if env_out is None and residual is None and outcome is None: + return None + return {"envelope": env_out, "outcome": outcome, "package_residual": residual} async def _run_gate_with_deadline( @@ -719,6 +754,8 @@ async def _run_gate_with_deadline( deadline_seconds: float, dual_flags_on: bool = False, review_envelope: Mapping[str, Any] | str | bytes | None = None, + review_outcome: Mapping[str, Any] | None = None, + package_residual: Mapping[str, Any] | None = None, key_release_grant: Mapping[str, Any] | None = None, agent_llm_kwargs: Mapping[str, Any] | None = None, settings: ChallengeSettings | None = None, @@ -756,6 +793,8 @@ def _decide() -> AttestationDecision: settings_dual_flags_on=True, eval_plan=plan, review_envelope=review_envelope, + review_outcome=review_outcome, + package_residual=package_residual, key_release_grant=key_release_grant, key_granted_flag=key_granted, score_binding=binding, @@ -892,8 +931,14 @@ async def process_direct_eval_result( review_envelope: Mapping[str, Any] | str | None = None key_release_grant: Mapping[str, Any] | None = None agent_llm_kwargs: dict[str, Any] | None = None + review_outcome: Mapping[str, Any] | None = None + package_residual: Mapping[str, Any] | None = None if dual_flags_on: - review_envelope = await _load_review_envelope_for_run(session, current) + materials = await _load_review_materials_for_run(session, current) + if materials is not None: + review_envelope = materials.get("envelope") # type: ignore[assignment] + review_outcome = materials.get("outcome") # type: ignore[assignment] + package_residual = materials.get("package_residual") # type: ignore[assignment] key_release_grant = _key_release_grant_from_result( plan=plan, validated=validated, @@ -931,6 +976,8 @@ async def process_direct_eval_result( deadline_seconds=settings.eval_result_verifier_deadline_seconds, dual_flags_on=dual_flags_on, review_envelope=review_envelope, + review_outcome=review_outcome, + package_residual=package_residual, key_release_grant=key_release_grant, agent_llm_kwargs=agent_llm_kwargs, settings=settings, diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/progress.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/progress.py new file mode 100644 index 000000000..983b7370a --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/progress.py @@ -0,0 +1,241 @@ +"""Mid-run attested Eval progress ingest (observability only). + +The CVM posts per-task phase transitions with the same Bearer ``EVAL_RUN_TOKEN`` +used for the final result route. Events land in ``TaskLogEvent`` so the existing +SSE feed surfaces them during the run. This path never mutates scores. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from agent_challenge.canonical import eval_wire +from agent_challenge.core.models import EvalRun, TaskLogEvent +from agent_challenge.evaluation.authorization import load_eval_run_plan +from agent_challenge.evaluation.task_events import ( + SAFE_TASK_PHASE_STATUSES, + record_task_event, +) + +PROGRESS_SOURCE = "eval_progress" +MAX_PROGRESS_BODY_BYTES = 16 * 1024 + + +class EvalProgressError(ValueError): + """Schema or policy failure for a progress ingest request.""" + + def __init__(self, message: str, *, code: str) -> None: + super().__init__(message) + self.code = code + + +def _metadata_dict(raw: str) -> dict[str, Any]: + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +async def _existing_progress_event( + session: AsyncSession, + *, + submission_id: int, + eval_run_id: str, + task_id: str, + client_sequence: int, +) -> TaskLogEvent | None: + rows = await session.scalars( + select(TaskLogEvent).where( + TaskLogEvent.submission_id == submission_id, + TaskLogEvent.task_id == task_id, + TaskLogEvent.event_type.in_(("task.status", "task.progress")), + ) + ) + for event in rows: + meta = _metadata_dict(event.metadata_json) + if ( + meta.get("source") == PROGRESS_SOURCE + and meta.get("eval_run_id") == eval_run_id + and meta.get("client_sequence") == client_sequence + ): + return event + return None + + +async def _max_client_sequence( + session: AsyncSession, + *, + submission_id: int, + eval_run_id: str, + task_id: str, +) -> int: + rows = await session.scalars( + select(TaskLogEvent).where( + TaskLogEvent.submission_id == submission_id, + TaskLogEvent.task_id == task_id, + TaskLogEvent.event_type.in_(("task.status", "task.progress")), + ) + ) + highest = 0 + for event in rows: + meta = _metadata_dict(event.metadata_json) + if meta.get("source") != PROGRESS_SOURCE or meta.get("eval_run_id") != eval_run_id: + continue + seq = meta.get("client_sequence") + if isinstance(seq, int) and seq > highest: + highest = seq + return highest + + +def _progress_receipt( + *, + eval_run_id: str, + task_id: str, + sequence: int, + event_id: int, + created: bool, +) -> dict[str, Any]: + return eval_wire.validate_eval_progress_receipt( + { + "schema_version": 1, + "eval_run_id": eval_run_id, + "task_id": task_id, + "sequence": sequence, + "event_id": event_id, + "created": created, + } + ) + + +async def process_eval_progress( + session: AsyncSession, + *, + run: EvalRun, + progress_request: Mapping[str, Any], +) -> tuple[dict[str, Any], bool]: + """Validate and record one mid-run progress event. + + Returns ``(receipt, created)``. ``created`` is False on idempotent replay of + the same ``(eval_run_id, task_id, sequence)``. Never mutates ``EvalRun.score`` + or any score columns. + """ + + # Fail closed if wire taxonomy drifts from the public SSE phase set. + if SAFE_TASK_PHASE_STATUSES != eval_wire.EVAL_PROGRESS_PHASES: + raise EvalProgressError( + "progress phase taxonomy mismatch", + code="progress_phase_taxonomy", + ) + + try: + validated = eval_wire.validate_eval_progress_request(progress_request) + except eval_wire.EvalWireError as exc: + message = str(exc) + if "forbids score fields" in message: + raise EvalProgressError(message, code="progress_score_forbidden") from exc + if "status is not a safe task phase" in message: + raise EvalProgressError(message, code="progress_phase_invalid") from exc + raise EvalProgressError(message, code="progress_invalid") from exc + + if validated["eval_run_id"] != run.eval_run_id: + raise EvalProgressError( + "progress run does not match route", + code="progress_run_mismatch", + ) + + plan = load_eval_run_plan(run) + if validated["submission_id"] != plan["submission_id"]: + raise EvalProgressError( + "progress submission_id does not match eval plan", + code="progress_submission_mismatch", + ) + + allowed_tasks = {item["task_id"] for item in plan["selected_tasks"]} + if validated["task_id"] not in allowed_tasks: + raise EvalProgressError( + "progress task_id is not in the eval plan", + code="progress_task_unknown", + ) + + existing = await _existing_progress_event( + session, + submission_id=run.submission_id, + eval_run_id=run.eval_run_id, + task_id=validated["task_id"], + client_sequence=validated["sequence"], + ) + if existing is not None: + return ( + _progress_receipt( + eval_run_id=run.eval_run_id, + task_id=validated["task_id"], + sequence=validated["sequence"], + event_id=existing.id, + created=False, + ), + False, + ) + + highest = await _max_client_sequence( + session, + submission_id=run.submission_id, + eval_run_id=run.eval_run_id, + task_id=validated["task_id"], + ) + if validated["sequence"] <= highest: + raise EvalProgressError( + "progress sequence must be monotone per task", + code="progress_sequence_stale", + ) + + message = validated["message"] + if message is None: + message = f"task {validated['task_id']} {validated['status']}" + + events = await record_task_event( + session, + submission_id=run.submission_id, + job_id=None, + task_id=validated["task_id"], + event_type=validated["event_type"], + message=message, + progress=validated["progress"], + status=validated["status"], + metadata={ + "source": PROGRESS_SOURCE, + "eval_run_id": run.eval_run_id, + "client_sequence": validated["sequence"], + "phase": validated["status"], + }, + ) + if not events: + raise EvalProgressError( + "progress event was not recorded", + code="progress_not_recorded", + ) + event = events[0] + await session.flush() + return ( + _progress_receipt( + eval_run_id=run.eval_run_id, + task_id=validated["task_id"], + sequence=validated["sequence"], + event_id=event.id, + created=True, + ), + True, + ) + + +__all__ = [ + "MAX_PROGRESS_BODY_BYTES", + "PROGRESS_SOURCE", + "EvalProgressError", + "process_eval_progress", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py index 989b4889f..251ae0330 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py @@ -409,6 +409,11 @@ def admit_production_score_from_chain( review_report_data_hex: str | None = None, review_domain: str | None = None, cached_review_allow: bool = False, + # AGATE: residual is bound on review verification outcome (and optionally + # envelope). Score admission must pass outcome so package_residual_missing + # is not raised when residual lives only on the outcome bag. + review_outcome: Mapping[str, Any] | None = None, + package_residual: Mapping[str, Any] | None = None, # Key-release leg (RA-TLS) key_release_grant: Mapping[str, Any] | None = None, key_granted_flag: bool = False, @@ -539,6 +544,8 @@ def admit_production_score_from_chain( dual_flags_on=dual_flags_on, require_package_residual=bool(dual_flags_on), expected_package_tree_sha=plan_tree_sha if dual_flags_on else None, + outcome=review_outcome, + package_residual=package_residual, ) if not review_decision.may_launch: code = review_decision.reason_code @@ -713,6 +720,8 @@ def admit_production_score_for_eval_result( settings_dual_flags_on: bool, eval_plan: Mapping[str, Any], review_envelope: Mapping[str, Any] | str | bytes | None, + review_outcome: Mapping[str, Any] | None = None, + package_residual: Mapping[str, Any] | None = None, key_release_grant: Mapping[str, Any] | None, key_granted_flag: bool, score_binding: Mapping[str, Any] | None, @@ -756,6 +765,8 @@ def admit_production_score_for_eval_result( return admit_production_score_from_chain( dual_flags_on=settings_dual_flags_on, review_envelope=review_envelope, + review_outcome=review_outcome, + package_residual=package_residual, key_release_grant=key_release_grant, key_granted_flag=key_granted_flag, eval_plan=eval_plan, diff --git a/packages/challenges/agent-challenge/tests/test_canonical_build_push.py b/packages/challenges/agent-challenge/tests/test_canonical_build_push.py index 2d7d8e324..3da3cad37 100644 --- a/packages/challenges/agent-challenge/tests/test_canonical_build_push.py +++ b/packages/challenges/agent-challenge/tests/test_canonical_build_push.py @@ -17,31 +17,34 @@ def test_repository_of_strips_tag_keeps_namespace(): assert ( - cbuild.repository_of("docker.io/mathiiss/agent-challenge-canonical:live") - == "docker.io/mathiiss/agent-challenge-canonical" + cbuild.repository_of("ghcr.io/baseintelligence/agent-challenge-canonical:live") + == "ghcr.io/baseintelligence/agent-challenge-canonical" ) # No tag -> unchanged. assert ( - cbuild.repository_of("docker.io/mathiiss/agent-challenge-canonical") - == "docker.io/mathiiss/agent-challenge-canonical" + cbuild.repository_of("ghcr.io/baseintelligence/agent-challenge-canonical") + == "ghcr.io/baseintelligence/agent-challenge-canonical" ) # A digest ref -> the repository component only. - assert cbuild.repository_of("docker.io/mathiiss/x@sha256:" + "a" * 64) == "docker.io/mathiiss/x" + assert ( + cbuild.repository_of("ghcr.io/baseintelligence/x@sha256:" + "a" * 64) + == "ghcr.io/baseintelligence/x" + ) def test_pushed_image_ref_is_repo_at_digest(): pushed = cbuild.PushedImage( - repository="docker.io/mathiiss/agent-challenge-canonical", + repository="ghcr.io/baseintelligence/agent-challenge-canonical", digest="sha256:" + "a" * 64, ) - assert pushed.ref == "docker.io/mathiiss/agent-challenge-canonical@sha256:" + "a" * 64 + assert pushed.ref == "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + "a" * 64 # It is digest-pinned per the compose guard (no bare tag). assert cbuild.DIGEST_PIN_RE.search(pushed.ref) def test_build_push_argv_is_reproducible_registry_push(): argv = cbuild.build_push_argv( - image_name="docker.io/mathiiss/agent-challenge-canonical:live", + image_name="ghcr.io/baseintelligence/agent-challenge-canonical:live", dockerfile="/repo/docker/canonical/Dockerfile", context="/repo", metadata_path="/tmp/meta.json", @@ -56,7 +59,7 @@ def test_build_push_argv_is_reproducible_registry_push(): # Registry push with reproducible layer-timestamp rewrite. out = argv[argv.index("--output") + 1] assert "type=image" in out - assert "name=docker.io/mathiiss/agent-challenge-canonical:live" in out + assert "name=ghcr.io/baseintelligence/agent-challenge-canonical:live" in out assert "push=true" in out assert "rewrite-timestamp=true" in out @@ -70,7 +73,7 @@ def fake_runner(argv, **kwargs): meta_path = argv[meta_idx + 1] with open(meta_path, "w", encoding="utf-8") as fh: json.dump( - {"containerimage.digest": digest, "image.name": "docker.io/mathiiss/x:live"}, + {"containerimage.digest": digest, "image.name": "ghcr.io/baseintelligence/x:live"}, fh, ) @@ -82,12 +85,12 @@ class _P: return _P() pushed = cbuild.build_and_push_image( - image_name="docker.io/mathiiss/x:live", + image_name="ghcr.io/baseintelligence/x:live", runner=fake_runner, context=str(tmp_path), dockerfile=str(cbuild.CANONICAL_DOCKERFILE), ) - assert pushed.ref == "docker.io/mathiiss/x@" + digest + assert pushed.ref == "ghcr.io/baseintelligence/x@" + digest assert cbuild.assert_pullable(pushed.ref) == pushed.ref @@ -106,7 +109,7 @@ class _P: with pytest.raises(RuntimeError): cbuild.build_and_push_image( - image_name="docker.io/mathiiss/x:live", + image_name="ghcr.io/baseintelligence/x:live", runner=fake_runner, context=str(tmp_path), ) @@ -123,7 +126,7 @@ class _P: with pytest.raises(RuntimeError, match="push"): cbuild.build_and_push_image( - image_name="docker.io/mathiiss/x:live", + image_name="ghcr.io/baseintelligence/x:live", runner=fake_runner, context=str(tmp_path), ) diff --git a/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py b/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py index 5fa3659dd..3c60ede0d 100644 --- a/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py +++ b/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py @@ -26,15 +26,17 @@ from agent_challenge.selfdeploy import eval as eval_deploy #: Digest-pinned canonical image used by the live 3-task terminal_bench smoke. +#: T2: GHCR path + T1 local buildx manifest digest (OPS_REQUIRED: swap to first +#: published GHCR digest from agent-recipe publish-eval-image.yml). LIVE_SMOKE_EVAL_IMAGE = ( - "docker.io/mathiiss/agent-challenge-canonical@sha256:" - "02331f0909f617e333f113be376d353770a673669946bcddaac3c53cbde7c9d8" + "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + "ea399cee1b3c9015024918a6070901e6b7b4bee432c8afe1e7dd40a95547e0bc" ) LIVE_SMOKE_APP_IDENTITY = "agent-challenge-eval-v1" LIVE_SMOKE_KEY_RELEASE = "ratls://84.32.70.61:8701" #: Synthetic review image pin (disjoint service inventory only). -REVIEW_IMAGE = "docker.io/mathiiss/agent-challenge-review@sha256:" + ("c" * 64) +REVIEW_IMAGE = "ghcr.io/baseintelligence/agent-challenge-review@sha256:" + ("c" * 64) #: Live residual local hash (pre-envelope, no guest golden/task bind mounts) #: for the smoke inputs above. @@ -42,8 +44,8 @@ # list includes the validator server-CA injection names (RA_TLS_SERVER_CA_*). Updated # when FAIL-CLOSED server-CA wiring lands so the discriminator still proves the # envelope factors (not allowed_envs) are what Phala provision rewrites. -# Residual pre-envelope hash after RA-TLS server-CA + OPENROUTER_API_KEY allowed_envs lands. -LIVE_RESIDUAL_NO_ENVELOPE_HASH = "5e33c9be56dc518070045596f1c6d7b31c2d73f7508056f7db726f3ccd6179a3" +# Residual pre-envelope hash after D6 GHCR-only orchestrator pin (T2). +LIVE_RESIDUAL_NO_ENVELOPE_HASH = "a4131e778128f5d22052efcab71ed2fdb6d7d5446c5f8141511ed153087a7364" def _live_smoke_compose() -> dict: @@ -129,7 +131,6 @@ def test_build_eval_deployment_plan_accepts_parity_compose_identity(): "submission_version": 1, "authorizing_review_digest": "ab" * 32, "agent_hash": "cd" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "selected_tasks": [ { "task_id": "adaptive-rejection-sampler", @@ -148,6 +149,7 @@ def test_build_eval_deployment_plan_accepts_parity_compose_identity(): }, ], "k": 1, + "n_concurrent": 4, "scoring_policy": policy, "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), "eval_app": { diff --git a/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py b/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py index b5050add8..39033f267 100644 --- a/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py +++ b/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py @@ -507,8 +507,8 @@ def _stub_eval_plan(**overrides: Any) -> dict[str, Any]: }, ], "k": 1, + "n_concurrent": 4, "agent_hash": "f" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "scoring_policy": { "schema_version": 1, "per_task_aggregation": "mean", @@ -516,7 +516,7 @@ def _stub_eval_plan(**overrides: Any) -> dict[str, Any]: }, "eval_app": { "app_identity": "agent-challenge-eval-v1", - "image_ref": "docker.io/mathiiss/agent-challenge-canonical@sha256:" + "d" * 64, + "image_ref": "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + "d" * 64, "compose_hash": "e" * 64, "measurement": { "mrtd": "1" * 96, diff --git a/packages/challenges/agent-challenge/tests/test_live_registry.py b/packages/challenges/agent-challenge/tests/test_live_registry.py index c858c7943..e612fda75 100644 --- a/packages/challenges/agent-challenge/tests/test_live_registry.py +++ b/packages/challenges/agent-challenge/tests/test_live_registry.py @@ -4,7 +4,7 @@ Terminal-Bench task by a *bare content digest* (``harbor_registry_ref = "sha256:<64hex>"``) with NO repository, so an in-CVM DooD orchestrator cannot ``docker pull`` it. For a live smoke E2E a small deterministic subset of task -images is published to a pullable, digest-pinned registry ref and recorded in a +images is published to a pullable, digest-pinned GHCR ref and recorded in a SEPARATE side manifest (``golden/live-registry-refs.json``) so the frozen content digests / canonical measurement stay byte-identical. @@ -13,6 +13,8 @@ offline/flag-off behavior is byte-identical); * every published ref is repository-qualified AND digest-pinned (a bare ``sha256:...`` content digest or a floating tag is rejected); + * D6: live-registry parse rejects Hub registry hosts and the retired personal + namespace; shipped golden is GHCR-only; * the shipped side manifest is a strict subset of the frozen golden tasks and never mutates ``dataset-digest.json``. """ @@ -36,7 +38,12 @@ FROZEN_CANONICAL_CONTENT_DIGEST = "8da006d76bcf59c2af3f36ed4420192d3930bda43683f32d80013a6ee5e7e02d" FROZEN_TASK_COUNT = 89 -_GOOD_REF = "docker.io/mathiiss/agent-challenge-tb21-x@sha256:" + ("a" * 64) +_GOOD_REF = "ghcr.io/baseintelligence/agent-challenge-tb21-x@sha256:" + ("a" * 64) + +# Forbidden shipping hosts/namespaces (D6). Kept as test inputs only. +_HUB_REF = "docker.io/library/x@sha256:" + ("a" * 64) +_RETIRED_NS_REF = "docker.io/mathiiss/agent-challenge-tb21-x@sha256:" + ("a" * 64) +_OTHER_REG_REF = "quay.io/example/x@sha256:" + ("a" * 64) # --------------------------------------------------------------------------- # @@ -45,14 +52,15 @@ def test_pullable_ref_accepts_repo_digest(): assert lr.is_pullable_ref(_GOOD_REF) assert lr.assert_pullable_ref(_GOOD_REF) == _GOOD_REF + assert lr.assert_live_registry_ref(_GOOD_REF) == _GOOD_REF @pytest.mark.parametrize( "bad", [ "sha256:" + ("a" * 64), # bare content digest (the golden behavior) - not pullable - "docker.io/mathiiss/x:latest", # floating tag, not digest-pinned - "docker.io/mathiiss/x@sha256:" + ("a" * 63), # short digest + "ghcr.io/baseintelligence/x:latest", # floating tag, not digest-pinned + "ghcr.io/baseintelligence/x@sha256:" + ("a" * 63), # short digest "plainname@sha256:" + ("a" * 64), # no repository/namespace ('/') "", 123, @@ -104,6 +112,25 @@ def test_parse_rejects_bare_digest_ref(): lr.parse_live_registry({"tasks": {"foo": {"registry_ref": "sha256:" + "a" * 64}}}) +@pytest.mark.parametrize( + "bad", + [ + _HUB_REF, + _RETIRED_NS_REF, + _OTHER_REG_REF, + "ghcr.io/mathiiss/x@sha256:" + ("a" * 64), + ], +) +def test_live_registry_ref_rejects_non_ghcr_shipping_hosts(bad): + """D6: live shipping refs must be ghcr.io and must not use retired namespaces.""" + with pytest.raises(lr.LiveRegistryError): + lr.assert_live_registry_ref(bad) + with pytest.raises(lr.LiveRegistryError): + lr.parse_live_registry({"tasks": {"foo": {"registry_ref": bad}}}) + with pytest.raises(lr.LiveRegistryError): + lr.parse_live_registry({"tasks": {}, "orchestrator_image": bad}) + + # --------------------------------------------------------------------------- # # task_id resolution (bare + dataset-prefixed) # --------------------------------------------------------------------------- # @@ -128,15 +155,32 @@ def test_shipped_live_manifest_is_valid_and_pullable(): reg = lr.load_live_registry(LIVE_MANIFEST) assert reg.task_refs, "shipped live manifest has no task refs" for task_id, ref in reg.task_refs.items(): - assert lr.is_pullable_ref(ref), (task_id, ref) + assert lr.assert_live_registry_ref(ref) == ref, (task_id, ref) def test_shipped_orchestrator_image_is_digest_pinned(): reg = lr.load_live_registry(LIVE_MANIFEST) assert reg.orchestrator_image is not None - assert lr.is_pullable_ref(reg.orchestrator_image) + assert lr.assert_live_registry_ref(reg.orchestrator_image) == reg.orchestrator_image # The deploy path's digest-pin guard accepts it (no bare tag). assert c.assert_digest_pinned(reg.orchestrator_image) == reg.orchestrator_image + assert reg.orchestrator_image.startswith( + "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ) + digest = reg.orchestrator_image.rsplit("@", 1)[-1] + assert digest.startswith("sha256:") and len(digest) == len("sha256:") + 64 + + +def test_shipped_live_manifest_has_no_hub_or_retired_namespace(): + """Golden live-registry product pin must not mention Hub host or retired ns.""" + text = LIVE_MANIFEST.read_text(encoding="utf-8") + assert "docker.io" not in text + assert "mathiiss" not in text + live = json.loads(text) + assert live["namespace"] == "ghcr.io/baseintelligence" + assert live["orchestrator_image"].startswith("ghcr.io/baseintelligence/") + for task_id, entry in live["tasks"].items(): + assert entry["registry_ref"].startswith("ghcr.io/baseintelligence/"), task_id def test_shipped_live_subset_is_subset_of_golden_and_small(): diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_backend_live_registry.py b/packages/challenges/agent-challenge/tests/test_own_runner_backend_live_registry.py index b37b6ba26..d659d6f5e 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_backend_live_registry.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_backend_live_registry.py @@ -23,7 +23,7 @@ def __init__(self, **kwargs): monkeypatch.setattr(backend, "TaskContainerBuilder", _SpyBuilder) - ref = "docker.io/mathiiss/agent-challenge-tb21-foo@sha256:" + ("a" * 64) + ref = "ghcr.io/baseintelligence/agent-challenge-tb21-foo@sha256:" + ("a" * 64) backend._build_default_preparer( task_ids=[], cache_root=backend.DEFAULT_CACHE_ROOT, @@ -60,7 +60,7 @@ def __init__(self, **kwargs): def test_main_resolves_live_refs_from_env(monkeypatch, tmp_path): - ref = "docker.io/mathiiss/agent-challenge-tb21-foo@sha256:" + ("b" * 64) + ref = "ghcr.io/baseintelligence/agent-challenge-tb21-foo@sha256:" + ("b" * 64) manifest = tmp_path / "live-registry-refs.json" manifest.write_text(json.dumps({"tasks": {"foo": {"registry_ref": ref}}}), encoding="utf-8") diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_live_registry_resolution.py b/packages/challenges/agent-challenge/tests/test_own_runner_live_registry_resolution.py index a1224a9ea..cc0ac5312 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_live_registry_resolution.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_live_registry_resolution.py @@ -20,7 +20,7 @@ from agent_challenge.evaluation.own_runner import container_builder as cb from agent_challenge.evaluation.own_runner.taskdefs import ParsedTask, parse_task -_LIVE_REF = "docker.io/mathiiss/agent-challenge-tb21-foo@sha256:" + ("a" * 64) +_LIVE_REF = "ghcr.io/baseintelligence/agent-challenge-tb21-foo@sha256:" + ("a" * 64) def _write_task(root: Path, *, docker_image: str | None) -> ParsedTask: diff --git a/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py b/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py index dc63dc713..6aeeed8b7 100644 --- a/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py +++ b/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py @@ -385,7 +385,9 @@ def fake_request( return 200, [ { "Id": "sha256:orch", - "RepoTags": ["docker.io/mathiiss/agent-challenge-canonical@sha256:deadbeef"], + "RepoTags": [ + "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:deadbeef" + ], } ] if method == "POST" and path.startswith("/containers/create"): diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py index 660ce0bbe..989495fc3 100644 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py +++ b/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py @@ -16,7 +16,7 @@ from agent_challenge.canonical import live_registry as lr from agent_challenge.selfdeploy import plan as p -PULLABLE = "docker.io/mathiiss/agent-challenge-canonical@sha256:" + ("a" * 64) +PULLABLE = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("a" * 64) KEY_URL = "https://validator.example/key-release"