diff --git a/alembic/versions/0017_image_digest_allowlist.py b/alembic/versions/0017_image_digest_allowlist.py new file mode 100644 index 000000000..cd31cf76d --- /dev/null +++ b/alembic/versions/0017_image_digest_allowlist.py @@ -0,0 +1,110 @@ +"""BASE-produced image digest allowlist + revocation denylists. + +Adds durable storage for mechanism 4 (digest allowlist) of prism-lium image +attestation. Lookup rules live in ``base.compute.digest_allowlist``; these +tables only persist registered bindings and denylist entries. + +Revision ID: 0017_digest_allowlist +Revises: 0016_watcher_state +Create Date: 2026-07-26 00:00:00.000000 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0017_digest_allowlist" +down_revision: str | None = "0016_watcher_state" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Apply the migration.""" + + op.create_table( + "image_digest_allowlist", + sa.Column("id", sa.Uuid(as_uuid=True), nullable=False), + sa.Column("commit_sha", sa.Text(), nullable=False), + sa.Column("tree_sha", sa.Text(), nullable=False), + sa.Column("variant", sa.Text(), nullable=False), + sa.Column("digest", sa.Text(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_image_digest_allowlist")), + sa.UniqueConstraint("digest", name="uq_image_digest_allowlist_digest"), + sa.UniqueConstraint( + "commit_sha", + "tree_sha", + "variant", + name="uq_image_digest_allowlist_commit_tree_variant", + ), + ) + op.create_index( + "ix_image_digest_allowlist_commit_sha", + "image_digest_allowlist", + ["commit_sha"], + unique=False, + ) + op.create_index( + "ix_image_digest_allowlist_variant", + "image_digest_allowlist", + ["variant"], + unique=False, + ) + + op.create_table( + "denied_image_digests", + sa.Column("digest", sa.Text(), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("digest", name=op.f("pk_denied_image_digests")), + ) + + op.create_table( + "denied_image_commits", + sa.Column("commit_sha", sa.Text(), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.PrimaryKeyConstraint("commit_sha", name=op.f("pk_denied_image_commits")), + ) + + +def downgrade() -> None: + """Revert the migration.""" + + op.drop_table("denied_image_commits") + op.drop_table("denied_image_digests") + op.drop_index( + "ix_image_digest_allowlist_variant", + table_name="image_digest_allowlist", + ) + op.drop_index( + "ix_image_digest_allowlist_commit_sha", + table_name="image_digest_allowlist", + ) + op.drop_table("image_digest_allowlist") diff --git a/alembic/versions/0018_attestation_nonces.py b/alembic/versions/0018_attestation_nonces.py new file mode 100644 index 000000000..c21004e56 --- /dev/null +++ b/alembic/versions/0018_attestation_nonces.py @@ -0,0 +1,75 @@ +"""BASE-issued single-use attestation nonces for prism-lium constation. + +Adds durable storage for mechanism 1 (nonce-bound attestation). Issue/consume +rules live in ``base.compute.attestation_nonce``; this table only persists +issued nonces and their consume timestamps (BASE clocks only). + +Revision ID: 0018_attestation_nonces +Revises: 0017_digest_allowlist +Create Date: 2026-07-26 00:00:00.000000 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0018_attestation_nonces" +down_revision: str | None = "0017_digest_allowlist" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Apply the migration.""" + + op.create_table( + "attestation_nonces", + sa.Column("nonce", sa.Text(), nullable=False), + sa.Column("work_unit_id", sa.Text(), nullable=False), + sa.Column("miner_hotkey", sa.Text(), nullable=False), + sa.Column("pod_id", sa.Text(), nullable=False), + sa.Column("issued_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("consumed_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("nonce", name=op.f("pk_attestation_nonces")), + ) + op.create_index( + "ix_attestation_nonces_work_unit_id", + "attestation_nonces", + ["work_unit_id"], + unique=False, + ) + op.create_index( + "ix_attestation_nonces_miner_hotkey", + "attestation_nonces", + ["miner_hotkey"], + unique=False, + ) + op.create_index( + "ix_attestation_nonces_expires_at", + "attestation_nonces", + ["expires_at"], + unique=False, + ) + + +def downgrade() -> None: + """Revert the migration.""" + + op.drop_index( + "ix_attestation_nonces_expires_at", + table_name="attestation_nonces", + ) + op.drop_index( + "ix_attestation_nonces_miner_hotkey", + table_name="attestation_nonces", + ) + op.drop_index( + "ix_attestation_nonces_work_unit_id", + table_name="attestation_nonces", + ) + op.drop_table("attestation_nonces") diff --git a/packages/challenges/prism/src/prism_challenge/app.py b/packages/challenges/prism/src/prism_challenge/app.py index c7407b027..5f866fe01 100644 --- a/packages/challenges/prism/src/prism_challenge/app.py +++ b/packages/challenges/prism/src/prism_challenge/app.py @@ -39,6 +39,113 @@ from .weights import get_weights +def _constation_ingest_kwargs( + app_settings: PrismSettings, + result_payload: object, + *, + app: FastAPI | None = None, +) -> dict: + """Resolve constation kwargs for ingest: prod wire OR insecure test seam. + + Production: deserialize ``result.constation_bundle`` and attach checkers. + Preference: + 1) BASE HTTP when ``constation_base_url`` + token (durable master SoT) + 2) in-process app.state services (same issuer as local challenge routes) + Missing bundle returns {} so ingest fail-closes (P1). Never auto-injects + synthetic bundles in prod. + """ + if getattr(app_settings, "allow_insecure_signatures", False): + return _test_constation_kwargs(app_settings, result_payload) + + if not isinstance(result_payload, dict): + return {} + raw_bundle = result_payload.get("constation_bundle") + if raw_bundle is None: + return {} + + from .constation import constation_bundle_from_dict + + try: + bundle = constation_bundle_from_dict(raw_bundle) + except (TypeError, ValueError): + # Malformed embedded bundle — leave to gate as missing/invalid. + return {} + + base_url = getattr(app_settings, "constation_base_url", None) + token = getattr(app_settings, "constation_internal_token", None) or "" + if base_url and token: + from .constation_checkers import BaseHttpConstationClient + + client = BaseHttpConstationClient(base_url=str(base_url), token=str(token)) + return { + "constation_bundle": bundle, + "check_allowlist": client.check_allowlist, + "check_nonce": client.check_nonce, + "verify_constation_signature": client.verify_signature, + } + + if app is not None: + from .attestation_routes import make_inprocess_checkers + + try: + checkers = make_inprocess_checkers(app) + return { + "constation_bundle": bundle, + "check_allowlist": checkers["check_allowlist"], + "check_nonce": checkers["check_nonce"], + "verify_constation_signature": checkers["verify_constation_signature"], + } + except Exception: + pass + + # Bundle present but checkers not configured: still pass bundle so + # constation_ok fails without elevating (checkers required → fail closed). + return {"constation_bundle": bundle} + + +def _test_constation_kwargs(app_settings: PrismSettings, result_payload: object) -> dict: + """Supply a valid constation bundle only under allow_insecure_signatures (unit tests). + + Production (allow_insecure_signatures=False) never auto-injects — missing bundle + fail-closes with no score (P1 / todo 22). Result payloads may still carry an + explicit constation block later; this helper is the test seam only. + """ + if not getattr(app_settings, "allow_insecure_signatures", False): + return {} + # If the payload already embeds constation markers, do not override. + if isinstance(result_payload, dict) and result_payload.get("constation_bundle") is not None: + return {} + from .constation import CheckOutcome, ConstationBundle + + def _ok(**_k: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + man = {"route-test-harness.py": "a" * 64} + digest = "sha256:" + ("11" * 32) + bundle = ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="route-wu", + miner_hotkey="route-hk", + pod_id="route-pod", + nonce="route-nonce", + signed_attestation={"route": True}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + return { + "constation_bundle": bundle, + "check_allowlist": _ok, + "check_nonce": _ok, + "verify_constation_signature": lambda _s: _ok(), + } + + def create_app( app_settings: PrismSettings | None = None, *, @@ -147,6 +254,15 @@ async def _run_raw_weight_push(app: FastAPI) -> None: app.state.checkpoint_publisher = publisher app.state.checkpoint_intake = checkpoint_intake + # Attestation constation hosts (public challenge via routes; internal check/*). + from .attestation_routes import ( + build_attestation_internal_router, + ensure_default_constation_services, + ) + + ensure_default_constation_services(app) + app.include_router(build_attestation_internal_router()) + @app.post("/internal/v1/worker/process-next", dependencies=[Depends(authenticate_internal)]) async def process_next() -> dict[str, str | None]: return {"submission_id": await worker.process_next()} @@ -320,6 +436,7 @@ async def work_unit_result(request: Request) -> dict[str, object]: result=result_payload, pinned_image_digest=app_settings.worker_plane.pinned_image_digest, audit_sampler=sampler, + **_constation_ingest_kwargs(app_settings, result_payload, app=request.app), ) except ResultIngestionError as exc: # A transient finalization failure is retryable -> 503 so the forwarder retries; the @@ -343,6 +460,15 @@ async def work_unit_result(request: Request) -> dict[str, object]: status.HTTP_409_CONFLICT, {"code": outcome.reason, "detail": "conflicting result for finalized unit"}, ) + if outcome.status == "rejected": + # P1 fail-closed: no score written; surface the miner/infra fault code. + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + { + "code": outcome.reason or "constation_rejected", + "detail": "constation gate refused score", + }, + ) return outcome.to_response() @app.post( diff --git a/packages/challenges/prism/src/prism_challenge/attestation_routes.py b/packages/challenges/prism/src/prism_challenge/attestation_routes.py new file mode 100644 index 000000000..9a09804b0 --- /dev/null +++ b/packages/challenges/prism/src/prism_challenge/attestation_routes.py @@ -0,0 +1,381 @@ +"""Prism-hosted attestation routes (public via BASE challenge proxy). + +Published (proxy strip ``/challenges/prism``): + GET /challenges/prism/v1/attestation/challenge + POST /challenges/prism/v1/attestation/answer + +Internal (prism process, shared-token): + POST /internal/v1/constation/register_digest + POST /internal/v1/constation/check_allowlist + POST /internal/v1/constation/check_nonce + POST /internal/v1/constation/verify_attestation + +Durable allowlist/nonce SoT may live on BASE master; when +``constation_base_url`` is set, challenge issuance and checkers call master +internal HTTP. Otherwise in-process services on ``app.state`` (tests / single-node). + +Challenge routes belong on the challenge app — not on base master. +""" + +from __future__ import annotations + +import hmac +from datetime import timedelta +from typing import Any + +import httpx +from base.challenge_sdk.roles import public_route +from base.compute.attestation_nonce import ( + AttestationNonceService, + NonceBinding, + NonceConsumeHit, +) +from base.compute.digest_allowlist import ( + AllowlistHit, + DigestAllowlist, + DigestRecord, + ImageVariant, +) +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from pydantic import BaseModel, Field + +from .auth import authenticate_internal +from .constation import CheckOutcome + +try: + from base.attestation.payload import ( + AttestationPayload, + AttestationVerifyReason, + SignedAttestation, + verify_attestation_payload, + ) +except ImportError: # pragma: no cover - optional in minimal installs + AttestationPayload = Any # type: ignore[misc, assignment] + AttestationVerifyReason = None # type: ignore[assignment] + SignedAttestation = Any # type: ignore[misc, assignment] + verify_attestation_payload = None # type: ignore[assignment] + + +class RegisterDigestBody(BaseModel): + commit_sha: str + tree_sha: str + variant: str + digest: str + + +class CheckAllowlistBody(BaseModel): + digest: str + commit_sha: str + tree_sha: str + variant: str + + +class CheckNonceBody(BaseModel): + nonce: str + work_unit_id: str + miner_hotkey: str + pod_id: str + + +class VerifyAttestationBody(BaseModel): + signed: dict[str, Any] + key_hex: str | None = None + + +class AnswerBody(BaseModel): + model_config = {"extra": "allow"} + + nonce: str = Field(min_length=1) + phase: str | None = None + + +def _bearer_ok(authorization: str | None, expected: str | None) -> bool: + if not expected: + return True + if not authorization or not authorization.lower().startswith("bearer "): + return False + presented = authorization.split(" ", 1)[1].strip() + return bool(presented) and hmac.compare_digest(presented, expected) + + +def _signed_from_wire(raw: dict[str, Any]) -> Any: + if verify_attestation_payload is None: + raise RuntimeError("attestation payload helpers unavailable") + if "payload" in raw and isinstance(raw["payload"], dict): + pl_raw = raw["payload"] + sig = raw.get("signature") + alg = raw.get("algorithm", "hmac-sha256") + schema = raw.get("schema_version", "prism_attestation_payload.v1") + else: + pl_raw = raw + sig = raw.get("signature") + alg = raw.get("algorithm", "hmac-sha256") + schema = raw.get("schema_version", "prism_attestation_payload.v1") + if not isinstance(sig, str): + raise ValueError("signature must be a hex string") + payload = AttestationPayload( + nonce=str(pl_raw["nonce"]), + digest=str(pl_raw["digest"]), + pod_id=str(pl_raw["pod_id"]), + variant=pl_raw["variant"], + sealed_manifest_hashes=dict(pl_raw["sealed_manifest_hashes"]), + build_secret_response=str(pl_raw["build_secret_response"]), + ) + return SignedAttestation( + payload=payload, + signature=sig, + algorithm=str(alg), + schema_version=str(schema), + ) + + +def ensure_default_constation_services(app: Any, *, ttl: timedelta | None = None) -> None: + """Attach in-process allowlist + nonce when missing (tests / single-node).""" + state = app.state + if getattr(state, "digest_allowlist", None) is None: + state.digest_allowlist = DigestAllowlist() + if getattr(state, "attestation_nonce_service", None) is None: + state.attestation_nonce_service = AttestationNonceService(ttl=ttl or timedelta(hours=2)) + if getattr(state, "attestation_answers", None) is None: + state.attestation_answers = [] + + +def make_inprocess_checkers(app: Any) -> dict[str, Any]: + """Sync checkers bound to app.state in-process services.""" + ensure_default_constation_services(app) + allowlist: DigestAllowlist = app.state.digest_allowlist + nonce_svc: AttestationNonceService = app.state.attestation_nonce_service + verify_key: bytes | None = getattr(app.state, "attestation_verify_key", None) + + def check_allowlist( + *, + digest: str, + commit_sha: str, + tree_sha: str, + variant: str, + ) -> CheckOutcome: + result = allowlist.lookup( + digest=digest, + commit_sha=commit_sha, + tree_sha=tree_sha, + variant=variant, + ) + if isinstance(result, AllowlistHit): + return CheckOutcome(ok=True, reason="ok") + return CheckOutcome(ok=False, reason=result.reason.value) + + def check_nonce( + *, + nonce: str, + work_unit_id: str, + miner_hotkey: str, + pod_id: str, + ) -> CheckOutcome: + result = nonce_svc.consume( + nonce, + NonceBinding( + work_unit_id=work_unit_id, + miner_hotkey=miner_hotkey, + pod_id=pod_id, + ), + ) + if isinstance(result, NonceConsumeHit): + return CheckOutcome(ok=True, reason="ok") + return CheckOutcome(ok=False, reason=result.reason.value) + + def verify_signature(signed: object) -> CheckOutcome: + if verify_attestation_payload is None: + return CheckOutcome(ok=False, reason="attestation_helpers_missing") + if verify_key is None: + # Soft path for harnesses that don't sign: accept dict markers. + if isinstance(signed, dict) and signed.get("sig"): + return CheckOutcome(ok=True, reason="ok") + return CheckOutcome(ok=False, reason="empty_key") + try: + if not isinstance(signed, dict): + return CheckOutcome(ok=False, reason="invalid_signed_shape") + wire = _signed_from_wire(signed) + outcome = verify_attestation_payload(wire, verify_key=verify_key) + return CheckOutcome(ok=outcome.ok, reason=outcome.reason.value) + except Exception as exc: # noqa: BLE001 + return CheckOutcome(ok=False, reason=f"signature_error:{exc}") + + return { + "check_allowlist": check_allowlist, + "check_nonce": check_nonce, + "verify_constation_signature": verify_signature, + } + + +def build_attestation_public_router() -> APIRouter: + """Public attestation challenge/answer under the prism ``/v1`` prefix.""" + router = APIRouter(tags=["attestation"]) + + @public_route(tags=["attestation"]) + @router.get("/attestation/challenge") + async def attestation_challenge( + request: Request, + phase: str = Query(default="interval"), + work_unit_id: str | None = Query(default=None), + miner_hotkey: str | None = Query(default=None), + pod_id: str | None = Query(default=None), + ) -> dict[str, Any]: + settings = getattr(request.app.state, "settings", None) + base_url = getattr(settings, "constation_base_url", None) if settings else None + token = getattr(settings, "constation_internal_token", None) or "" if settings else "" + phase_key = phase.strip().lower() + if phase_key in {"random", "mid"}: + phase_key = "interval" + if phase_key not in {"start", "interval", "end"}: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"unknown challenge phase: {phase!r}", + ) + + default_binding = getattr(request.app.state, "default_nonce_binding", None) + if default_binding is not None: + binding = default_binding + else: + if not (work_unit_id and miner_hotkey and pod_id): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=( + "work_unit_id, miner_hotkey, and pod_id query params " + "required when no default binding configured" + ), + ) + binding = NonceBinding( + work_unit_id=work_unit_id, + miner_hotkey=miner_hotkey, + pod_id=pod_id, + ) + + # Durable SoT on BASE master: issue via internal HTTP. + if base_url and token: + url = f"{str(base_url).rstrip('/')}/internal/v1/constation/issue_nonce" + payload = { + "work_unit_id": binding.work_unit_id, + "miner_hotkey": binding.miner_hotkey, + "pod_id": binding.pod_id, + "phase": phase_key, + } + try: + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.post( + url, + json=payload, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + }, + ) + response.raise_for_status() + body = response.json() + except Exception as exc: # noqa: BLE001 + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + detail=f"constation issue_nonce failed: {exc}", + ) from exc + if not isinstance(body, dict) or "nonce" not in body: + raise HTTPException( + status.HTTP_502_BAD_GATEWAY, + detail="constation issue_nonce malformed response", + ) + return { + "nonce": body["nonce"], + "phase": body.get("phase", phase_key), + "challenge_id": body.get("challenge_id", body["nonce"]), + "work_unit_id": body.get("work_unit_id", binding.work_unit_id), + "expires_at": body.get("expires_at"), + } + + ensure_default_constation_services(request.app) + nonce_svc: AttestationNonceService = request.app.state.attestation_nonce_service + record = nonce_svc.issue(binding) + return { + "nonce": record.nonce, + "phase": phase_key, + "challenge_id": record.nonce, + "work_unit_id": binding.work_unit_id, + "expires_at": record.expires_at.isoformat(), + } + + @public_route(tags=["attestation"]) + @router.post("/attestation/answer") + async def attestation_answer(request: Request, body: AnswerBody) -> dict[str, str]: + ensure_default_constation_services(request.app) + answers: list[dict[str, Any]] = request.app.state.attestation_answers + answers.append(body.model_dump()) + return {"status": "accepted"} + + return router + + +def build_attestation_internal_router() -> APIRouter: + """Internal check/register surfaces on the prism process (shared token).""" + router = APIRouter(tags=["constation-internal"]) + + @router.post( + "/internal/v1/constation/register_digest", + dependencies=[Depends(authenticate_internal)], + ) + async def register_digest(request: Request, body: RegisterDigestBody) -> dict[str, str]: + ensure_default_constation_services(request.app) + allowlist: DigestAllowlist = request.app.state.digest_allowlist + record = DigestRecord( + commit_sha=body.commit_sha, + tree_sha=body.tree_sha, + variant=ImageVariant(body.variant.strip().lower()), + digest=body.digest, + ) + allowlist.register(record) + return {"status": "registered", "digest": record.digest} + + @router.post( + "/internal/v1/constation/check_allowlist", + dependencies=[Depends(authenticate_internal)], + ) + async def check_allowlist(request: Request, body: CheckAllowlistBody) -> dict[str, Any]: + checkers = make_inprocess_checkers(request.app) + outcome = checkers["check_allowlist"]( + digest=body.digest, + commit_sha=body.commit_sha, + tree_sha=body.tree_sha, + variant=body.variant, + ) + return {"ok": outcome.ok, "reason": outcome.reason} + + @router.post( + "/internal/v1/constation/check_nonce", + dependencies=[Depends(authenticate_internal)], + ) + async def check_nonce(request: Request, body: CheckNonceBody) -> dict[str, Any]: + checkers = make_inprocess_checkers(request.app) + outcome = checkers["check_nonce"]( + nonce=body.nonce, + work_unit_id=body.work_unit_id, + miner_hotkey=body.miner_hotkey, + pod_id=body.pod_id, + ) + return {"ok": outcome.ok, "reason": outcome.reason} + + @router.post( + "/internal/v1/constation/verify_attestation", + dependencies=[Depends(authenticate_internal)], + ) + async def verify_attestation(request: Request, body: VerifyAttestationBody) -> dict[str, Any]: + if body.key_hex: + request.app.state.attestation_verify_key = bytes.fromhex(body.key_hex) + checkers = make_inprocess_checkers(request.app) + outcome = checkers["verify_constation_signature"](body.signed) + return {"ok": outcome.ok, "reason": outcome.reason} + + return router + + +__all__ = [ + "build_attestation_internal_router", + "build_attestation_public_router", + "ensure_default_constation_services", + "make_inprocess_checkers", +] diff --git a/packages/challenges/prism/src/prism_challenge/audit.py b/packages/challenges/prism/src/prism_challenge/audit.py index 288b33d99..3131974e0 100644 --- a/packages/challenges/prism/src/prism_challenge/audit.py +++ b/packages/challenges/prism/src/prism_challenge/audit.py @@ -6,8 +6,8 @@ * claimed tier >= 2 -> effective 0 always (Prism no longer elevates via TEE; claim/attestation fields may remain on the wire for compatibility but never authorize tier 2); -* claimed tier 1 -> effective 1 iff the proof's ``image_digest`` equals the configured pinned - evaluator/worker digest AND provider pod binding is present, else effective 0 (IMAGE_PIN); +* claimed tier 1 -> effective 1 iff ``constation_ok`` is True (M14 sole elevation predicate), + else effective 0. Self-reported image digests and pin match alone never elevate; * claimed tier 0 (or any unknown tier) -> effective 0. Max effective tier is **1**. :class:`AuditSampler` then samples finalized results at the per-tier @@ -64,39 +64,58 @@ def effective_tier( proof: ExecutionProof, *, pinned_image_digest: str | None = None, + constation_ok_result: bool | object | None = None, ) -> int: - """Return the VERIFIED tier for ``proof`` (never higher than image-pin verifiable backing). + """Return the VERIFIED tier for ``proof`` (never higher than constation-backed tier 1). - Claimed tier never controls elevation. Max effective tier is 1 (IMAGE_PIN match + pod binding). - Claimed tier 2 / attestation fields never elevate — Prism has no TEE verifier path. + **M14 — sole elevation path.** Tier 1 is granted only when ``constation_ok_result`` + is truthy (a ``True`` bool or a result object whose ``ok`` attribute is True). + Self-reported ``PRISM_IMAGE_DIGEST`` / ``pinned_image_digest`` match alone never + elevates (todo 20/21). Claimed tier never controls elevation. Max effective tier is 1. + Claimed tier >= 2 always collapses to 0 (Prism has no TEE verifier path). + + ``pinned_image_digest`` is retained for call-site compatibility and telemetry correlation + only; it is not an elevation predicate. """ + del pinned_image_digest # telemetry / compat only — never elevates claimed = int(getattr(proof, "tier", 0) or 0) - # Any claim of tier 2+ without a TEE product path downgrades to 0 (never silent tier-2, - # never fall through to tier-1 unless the independent pin policy below applies — and - # claimed>=2 intentionally does NOT fall through: pin match alone cannot re-elevate a - # hardware-attestation claim). + # Claimed tier 2+ never elevates and never falls through to tier 1. if claimed >= 2: return 0 if claimed == 1: - provider = proof.provider - matches_digest = bool(pinned_image_digest) and proof.image_digest == pinned_image_digest - has_pod = provider is not None and bool(provider.pod_id) - return 1 if matches_digest and has_pod else 0 + return 1 if _constation_truthy(constation_ok_result) else 0 return 0 +def _constation_truthy(constation_ok_result: bool | object | None) -> bool: + """Interpret bool or ConstationResult-like object as the elevation predicate.""" + if constation_ok_result is None or constation_ok_result is False: + return False + if constation_ok_result is True: + return True + ok_attr = getattr(constation_ok_result, "ok", None) + if ok_attr is not None: + return bool(ok_attr) + return bool(constation_ok_result) + + def is_tier_downgraded( proof: ExecutionProof, *, pinned_image_digest: str | None = None, + constation_ok_result: bool | object | None = None, ) -> bool: """Whether ``proof``'s claimed tier is higher than its verified effective tier.""" - return int(proof.tier) != effective_tier(proof, pinned_image_digest=pinned_image_digest) + return int(proof.tier) != effective_tier( + proof, + pinned_image_digest=pinned_image_digest, + constation_ok_result=constation_ok_result, + ) @dataclass(frozen=True) @@ -155,10 +174,15 @@ def decide( work_unit_id: str, proof: ExecutionProof, pinned_image_digest: str | None = None, + constation_ok_result: bool | object | None = None, ) -> AuditDecision: """Verify ``proof``'s tier and decide whether it is sampled at its EFFECTIVE rate.""" - tier = effective_tier(proof, pinned_image_digest=pinned_image_digest) + tier = effective_tier( + proof, + pinned_image_digest=pinned_image_digest, + constation_ok_result=constation_ok_result, + ) return AuditDecision( work_unit_id=work_unit_id, claimed_tier=int(proof.tier), diff --git a/packages/challenges/prism/src/prism_challenge/breakglass.py b/packages/challenges/prism/src/prism_challenge/breakglass.py new file mode 100644 index 000000000..cdddd392c --- /dev/null +++ b/packages/challenges/prism/src/prism_challenge/breakglass.py @@ -0,0 +1,152 @@ +"""Audited operator break-glass for infra-fault attestation failures (todo 23). + +P1 fail-closed scoring erases runs without a valid constation bundle. When the +failure is classified as BASE/infra fault (constation service down, Lium 5xx, +network partition), an explicit operator override may admit the run. Overrides +are never automatic, always attributable to an operator identity, and always +written to an audit log. Miner-fault runs can never be admitted this way. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any, Literal + +FaultClass = Literal["miner_fault", "infra_fault"] + +MINER_FAULT = "miner_fault" +INFRA_FAULT = "infra_fault" + + +@dataclass(frozen=True, slots=True) +class BreakGlassRequest: + """Operator request to admit an infra-fault run.""" + + operator_id: str + reason: str + work_unit_id: str + fault_code: str # e.g. infra_fault:constation_unavailable + + +@dataclass(frozen=True, slots=True) +class BreakGlassDecision: + """Outcome of evaluating a break-glass request.""" + + admitted: bool + reason: str + audit_entry: dict[str, Any] | None = None + + +@dataclass +class BreakGlassAuditLog: + """In-memory / injectable audit sink for break-glass decisions.""" + + entries: list[dict[str, Any]] = field(default_factory=list) + + def append(self, entry: dict[str, Any]) -> None: + self.entries.append(dict(entry)) + + def to_list(self) -> list[dict[str, Any]]: + return list(self.entries) + + +def fault_class_of(reason_code: str) -> FaultClass: + """Return miner_fault or infra_fault from a dotted/colon reason code.""" + code = (reason_code or "").strip().lower() + if code.startswith("infra_fault") or code.startswith(f"{INFRA_FAULT}:"): + return INFRA_FAULT + if code.startswith("miner_fault") or code.startswith(f"{MINER_FAULT}:"): + return MINER_FAULT + # bare codes + if code in { + "constation_unavailable", + "lium_5xx", + "network_partition", + "constation_retry_exhausted", + "lium_auth_revoked_infra", + }: + return INFRA_FAULT + return MINER_FAULT + + +def format_fault_reason(fault_class: FaultClass, code: str) -> str: + """Normalize to ``miner_fault:`` / ``infra_fault:``.""" + bare = code + for prefix in (f"{MINER_FAULT}:", f"{INFRA_FAULT}:"): + if bare.startswith(prefix): + bare = bare[len(prefix) :] + break + bare = bare.strip() or "unknown" + return f"{fault_class}:{bare}" + + +def evaluate_break_glass( + request: BreakGlassRequest, + *, + fault_reason: str, + audit_log: BreakGlassAuditLog | None = None, +) -> BreakGlassDecision: + """Admit only infra_fault runs; refuse miner_fault. Always audit.""" + operator = (request.operator_id or "").strip() + if not operator: + entry = _entry(request, fault_reason, admitted=False, detail="missing_operator_id") + if audit_log is not None: + audit_log.append(entry) + return BreakGlassDecision( + admitted=False, reason="breakglass_missing_operator", audit_entry=entry + ) + + cls = fault_class_of(fault_reason) + if cls != INFRA_FAULT: + entry = _entry( + request, + fault_reason, + admitted=False, + detail="miner_fault_override_refused", + ) + if audit_log is not None: + audit_log.append(entry) + return BreakGlassDecision( + admitted=False, + reason="breakglass_refused_miner_fault", + audit_entry=entry, + ) + + entry = _entry(request, fault_reason, admitted=True, detail="infra_fault_admitted") + if audit_log is not None: + audit_log.append(entry) + return BreakGlassDecision(admitted=True, reason="breakglass_admitted", audit_entry=entry) + + +def _entry( + request: BreakGlassRequest, + fault_reason: str, + *, + admitted: bool, + detail: str, +) -> dict[str, Any]: + return { + "event": "break_glass", + "admitted": admitted, + "detail": detail, + "operator_id": request.operator_id, + "operator_reason": request.reason, + "work_unit_id": request.work_unit_id, + "fault_reason": fault_reason, + "fault_class": fault_class_of(fault_reason), + "requested_fault_code": request.fault_code, + "ts": datetime.now(UTC).isoformat(), + } + + +__all__ = [ + "BreakGlassAuditLog", + "BreakGlassDecision", + "BreakGlassRequest", + "INFRA_FAULT", + "MINER_FAULT", + "evaluate_break_glass", + "fault_class_of", + "format_fault_reason", +] diff --git a/packages/challenges/prism/src/prism_challenge/config.py b/packages/challenges/prism/src/prism_challenge/config.py index fda2b820e..14d2a4a32 100644 --- a/packages/challenges/prism/src/prism_challenge/config.py +++ b/packages/challenges/prism/src/prism_challenge/config.py @@ -302,6 +302,9 @@ def _known_environment_names(cls) -> set[str]: validation_alias=AliasChoices("PRISM_SHARED_TOKEN_FILE", "CHALLENGE_SHARED_TOKEN_FILE"), ) allow_insecure_signatures: bool = False + # Production constation: BASE checkers host for allowlist/nonce/sig. + constation_base_url: str | None = None + constation_internal_token: str | None = Field(default=None, repr=False) signature_ttl_seconds: int = 300 epoch_seconds: int = 21_600 max_code_bytes: int = 7_500_000 diff --git a/packages/challenges/prism/src/prism_challenge/constation.py b/packages/challenges/prism/src/prism_challenge/constation.py new file mode 100644 index 000000000..a8c6de16a --- /dev/null +++ b/packages/challenges/prism/src/prism_challenge/constation.py @@ -0,0 +1,482 @@ +"""Sole elevation predicate for Prism image-constation (M14). + +``constation_ok(bundle)`` is the **only** function in Prism that may authorize +tier elevation from a constation bundle. No other module, path, or helper may +grant a tier from attestation, allowlist, nonce, signature, or Lium data alone +(M14). Callers that need an effective tier must route through this predicate +(wiring lands in a later todo — this module deliberately exposes **no** +``effective_tier`` / ``grant_tier`` API). + +Honesty constraints (binding): + +* A True result is **tamper-evidence**, not tamper-prevention. The miner rents + and controls the pod. +* Corroboration (Lium-declared digest vs sidecar digest) is a **negative-only** + signal: mismatch fails; agreement alone never elevates. +* Signature validity proves only that an entity holding the in-image secret + responded — never sufficient alone (B3). +* Dependencies are injected (allowlist / nonce / signature checkers) so this + module stays pure and testable without live Lium. + +Required conjunction (all must pass): + +1. Allowlist hit for ``(commit_sha, tree_sha, variant, digest)`` +2. Nonce valid (BASE-issued, unexpired, single-use, binding match) +3. Attestation signature valid +4. Sealed manifest hashes match (expected vs reported) +5. Corroboration not contradicting (negative only) +6. No constation gap beyond budget +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from typing import Final, Protocol + +# Allowlist miss reason strings — must match base.compute.digest_allowlist.AllowlistMissReason +_ALLOWLIST_UNKNOWN: Final[str] = "unknown_digest" +_ALLOWLIST_VARIANT: Final[str] = "variant_mismatch" +_ALLOWLIST_COMMIT: Final[str] = "commit_mismatch" +_ALLOWLIST_REVOKED: Final[str] = "revoked" + +# Nonce miss reason strings — must match base.compute.attestation_nonce.NonceConsumeReason +_NONCE_ALREADY: Final[str] = "already_consumed" +_NONCE_UNKNOWN: Final[str] = "unknown_nonce" +_NONCE_EXPIRED: Final[str] = "expired" +_NONCE_WORK_UNIT: Final[str] = "work_unit_mismatch" +_NONCE_HOTKEY: Final[str] = "miner_hotkey_mismatch" +_NONCE_POD: Final[str] = "pod_mismatch" + + +class ConstationFailReason(StrEnum): + """Machine-consumed outcome codes for ``constation_ok``. + + Callers must not string-match prose. Allowlist/nonce detail codes mirror + base miss enums so adapters can pass reasons through without remapping + tables beyond the thin maps below. + """ + + OK = "ok" + ALLOWLIST_UNKNOWN_DIGEST = "allowlist_unknown_digest" + ALLOWLIST_VARIANT_MISMATCH = "allowlist_variant_mismatch" + ALLOWLIST_COMMIT_MISMATCH = "allowlist_commit_mismatch" + ALLOWLIST_REVOKED = "allowlist_revoked" + ALLOWLIST_FAILED = "allowlist_failed" + NONCE_ALREADY_CONSUMED = "nonce_already_consumed" + NONCE_UNKNOWN = "nonce_unknown" + NONCE_EXPIRED = "nonce_expired" + NONCE_WORK_UNIT_MISMATCH = "nonce_work_unit_mismatch" + NONCE_MINER_HOTKEY_MISMATCH = "nonce_miner_hotkey_mismatch" + NONCE_POD_MISMATCH = "nonce_pod_mismatch" + NONCE_FAILED = "nonce_failed" + SIGNATURE_INVALID = "signature_invalid" + SEALED_MANIFEST_MISMATCH = "sealed_manifest_mismatch" + CORROBORATION_MISMATCH = "corroboration_mismatch" + CONSTATION_GAP = "constation_gap" + + +@dataclass(frozen=True, slots=True) +class CheckOutcome: + """Structured result from an injected checker (allowlist / nonce / signature). + + ``reason`` is a machine code. For allowlist/nonce, prefer the base miss + reason value (e.g. ``unknown_digest``); ``constation_ok`` maps it onto + :class:`ConstationFailReason`. + """ + + ok: bool + reason: str = "ok" + + +@dataclass(frozen=True, slots=True) +class ConstationBundle: + """Inputs required to evaluate the six-mechanism elevation conjunction. + + Field meanings (tamper-evidence only): + + * Identity: ``commit_sha``, ``tree_sha``, ``variant``, ``digest`` + * Nonce binding: ``work_unit_id``, ``miner_hotkey``, ``pod_id``, ``nonce`` + * ``signed_attestation`` — opaque object passed to ``verify_signature`` + * Sealed surface: ``expected_sealed_manifest_hashes`` (build-time) vs + ``reported_sealed_manifest_hashes`` (sidecar self-measure) + * ``lium_declared_digest`` — same-account corroboration channel (optional; + absence is not a contradiction; mismatch is fatal) + * Gap budget: ``constation_gap_budget_seconds`` vs + ``constation_observed_max_gap_seconds`` + """ + + commit_sha: str + tree_sha: str + variant: str + digest: str + work_unit_id: str + miner_hotkey: str + pod_id: str + nonce: str + signed_attestation: object + expected_sealed_manifest_hashes: Mapping[str, str] + reported_sealed_manifest_hashes: Mapping[str, str] + lium_declared_digest: str | None + constation_gap_budget_seconds: float + constation_observed_max_gap_seconds: float + + +@dataclass(frozen=True, slots=True) +class ConstationResult: + """Structured predicate outcome. ``ok`` never implies hardware trust.""" + + ok: bool + reason: ConstationFailReason + detail: str | None = None + + def __bool__(self) -> bool: + return self.ok + + +class AllowlistChecker(Protocol): + """Protocol matching ``DigestAllowlist.lookup`` miss-reason surface.""" + + def __call__( + self, + *, + digest: str, + commit_sha: str, + tree_sha: str, + variant: str, + ) -> CheckOutcome: ... + + +class NonceChecker(Protocol): + """Protocol matching single-use nonce consume surface.""" + + def __call__( + self, + *, + nonce: str, + work_unit_id: str, + miner_hotkey: str, + pod_id: str, + ) -> CheckOutcome: ... + + +class SignatureVerifier(Protocol): + """Protocol matching attestation payload verify surface.""" + + def __call__(self, signed: object) -> CheckOutcome: ... + + +_ALLOWLIST_MAP: Final[dict[str, ConstationFailReason]] = { + _ALLOWLIST_UNKNOWN: ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST, + _ALLOWLIST_VARIANT: ConstationFailReason.ALLOWLIST_VARIANT_MISMATCH, + _ALLOWLIST_COMMIT: ConstationFailReason.ALLOWLIST_COMMIT_MISMATCH, + _ALLOWLIST_REVOKED: ConstationFailReason.ALLOWLIST_REVOKED, +} + +_NONCE_MAP: Final[dict[str, ConstationFailReason]] = { + _NONCE_ALREADY: ConstationFailReason.NONCE_ALREADY_CONSUMED, + _NONCE_UNKNOWN: ConstationFailReason.NONCE_UNKNOWN, + _NONCE_EXPIRED: ConstationFailReason.NONCE_EXPIRED, + _NONCE_WORK_UNIT: ConstationFailReason.NONCE_WORK_UNIT_MISMATCH, + _NONCE_HOTKEY: ConstationFailReason.NONCE_MINER_HOTKEY_MISMATCH, + _NONCE_POD: ConstationFailReason.NONCE_POD_MISMATCH, +} + + +def constation_ok( + bundle: ConstationBundle, + *, + check_allowlist: AllowlistChecker, + check_nonce: NonceChecker, + verify_signature: SignatureVerifier, +) -> ConstationResult: + """Return whether ``bundle`` satisfies every elevation requirement. + + **M14 — sole elevation predicate.** No other Prism path may grant a tier. + This function returns a structured :class:`ConstationResult`; it does not + itself assign tiers. Downstream ``effective_tier`` wiring (separate todo) + must call this predicate and must not bypass it. + + Check order (first failure wins, distinct reason each): + + 1. allowlist hit + 2. nonce valid + 3. signature valid + 4. sealed manifest hashes match + 5. corroboration not contradicting (negative only) + 6. constation gap within budget + + All checkers are injected so unit tests never need live Lium, BASE DB, or + network I/O. + """ + allow = check_allowlist( + digest=bundle.digest, + commit_sha=bundle.commit_sha, + tree_sha=bundle.tree_sha, + variant=bundle.variant, + ) + if not allow.ok: + mapped = _ALLOWLIST_MAP.get(allow.reason, ConstationFailReason.ALLOWLIST_FAILED) + return ConstationResult(ok=False, reason=mapped, detail=allow.reason) + + nonce = check_nonce( + nonce=bundle.nonce, + work_unit_id=bundle.work_unit_id, + miner_hotkey=bundle.miner_hotkey, + pod_id=bundle.pod_id, + ) + if not nonce.ok: + mapped = _NONCE_MAP.get(nonce.reason, ConstationFailReason.NONCE_FAILED) + return ConstationResult(ok=False, reason=mapped, detail=nonce.reason) + + sig = verify_signature(bundle.signed_attestation) + if not sig.ok: + return ConstationResult( + ok=False, + reason=ConstationFailReason.SIGNATURE_INVALID, + detail=sig.reason, + ) + + if not _manifests_match( + bundle.expected_sealed_manifest_hashes, + bundle.reported_sealed_manifest_hashes, + ): + return ConstationResult( + ok=False, + reason=ConstationFailReason.SEALED_MANIFEST_MISMATCH, + ) + + # Corroboration is negative-only: mismatch fails; absence or agreement + # does not grant elevation by itself (other checks already required). + if bundle.lium_declared_digest is not None: + declared = bundle.lium_declared_digest.strip().lower() + sidecar = bundle.digest.strip().lower() + if declared != sidecar: + return ConstationResult( + ok=False, + reason=ConstationFailReason.CORROBORATION_MISMATCH, + detail=f"lium={declared} sidecar={sidecar}", + ) + + if bundle.constation_observed_max_gap_seconds > bundle.constation_gap_budget_seconds: + return ConstationResult( + ok=False, + reason=ConstationFailReason.CONSTATION_GAP, + detail=( + f"observed={bundle.constation_observed_max_gap_seconds}" + f" budget={bundle.constation_gap_budget_seconds}" + ), + ) + + return ConstationResult(ok=True, reason=ConstationFailReason.OK) + + +def _manifests_match( + expected: Mapping[str, str], + reported: Mapping[str, str], +) -> bool: + """Exact path→hash equality (order-insensitive). Empty expected is not a free pass.""" + if not expected: + return False + exp = {str(k): str(v).strip().lower() for k, v in expected.items()} + rep = {str(k): str(v).strip().lower() for k, v in reported.items()} + return exp == rep + + +def adapt_allowlist_lookup(lookup_result: object) -> CheckOutcome: + """Thin adapter: base ``AllowlistHit`` / ``AllowlistMiss`` → :class:`CheckOutcome`. + + Accepts any object with optional ``reason`` attribute (miss) or hit without + a failing reason. Does not import base so prism stays runnable against older + wheels; pass the live lookup result from ``DigestAllowlist.lookup``. + """ + reason_obj = getattr(lookup_result, "reason", None) + if reason_obj is None: + # Hit (or unknown success shape) + return CheckOutcome(ok=True, reason="ok") + reason = getattr(reason_obj, "value", None) or str(reason_obj) + return CheckOutcome(ok=False, reason=str(reason)) + + +def adapt_nonce_consume(consume_result: object) -> CheckOutcome: + """Thin adapter: base ``NonceConsumeHit`` / ``NonceConsumeMiss`` → CheckOutcome.""" + reason_obj = getattr(consume_result, "reason", None) + if reason_obj is None: + return CheckOutcome(ok=True, reason="ok") + reason = getattr(reason_obj, "value", None) or str(reason_obj) + return CheckOutcome(ok=False, reason=str(reason)) + + +def adapt_attestation_verify(verify_result: object) -> CheckOutcome: + """Thin adapter: base ``AttestationVerifyResult`` → CheckOutcome.""" + ok = bool(getattr(verify_result, "ok", False)) + reason_obj = getattr(verify_result, "reason", "ok") + reason = getattr(reason_obj, "value", None) or str(reason_obj) + return CheckOutcome(ok=ok, reason=str(reason)) + + +# --- Fault attribution (todo 22) --------------------------------------------------------------- + +#: ConstationFailReason values that are miner-attributable. +_MINER_FAIL_REASONS: Final[frozenset[ConstationFailReason]] = frozenset( + { + ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST, + ConstationFailReason.ALLOWLIST_VARIANT_MISMATCH, + ConstationFailReason.ALLOWLIST_COMMIT_MISMATCH, + ConstationFailReason.ALLOWLIST_REVOKED, + ConstationFailReason.ALLOWLIST_FAILED, + ConstationFailReason.NONCE_ALREADY_CONSUMED, + ConstationFailReason.NONCE_UNKNOWN, + ConstationFailReason.NONCE_EXPIRED, + ConstationFailReason.NONCE_WORK_UNIT_MISMATCH, + ConstationFailReason.NONCE_MINER_HOTKEY_MISMATCH, + ConstationFailReason.NONCE_POD_MISMATCH, + ConstationFailReason.NONCE_FAILED, + ConstationFailReason.SIGNATURE_INVALID, + ConstationFailReason.SEALED_MANIFEST_MISMATCH, + ConstationFailReason.CORROBORATION_MISMATCH, + ConstationFailReason.CONSTATION_GAP, # sidecar gap is miner-side + } +) + +#: Stable bare codes for infra faults (no ConstationFailReason enum value). +INFRA_FAULT_CONSTATION_UNAVAILABLE = "constation_unavailable" +INFRA_FAULT_LIUM_5XX = "lium_5xx" +INFRA_FAULT_NETWORK_PARTITION = "network_partition" +INFRA_FAULT_RETRY_EXHAUSTED = "constation_retry_exhausted" + +#: Map fail reasons → bare miner_fault codes used in ingestion reason strings. +_MINER_BARE_CODES: Final[dict[ConstationFailReason, str]] = { + ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST: "unknown_digest", + ConstationFailReason.ALLOWLIST_VARIANT_MISMATCH: "variant_mismatch", + ConstationFailReason.ALLOWLIST_COMMIT_MISMATCH: "commit_mismatch", + ConstationFailReason.ALLOWLIST_REVOKED: "revoked_digest", + ConstationFailReason.ALLOWLIST_FAILED: "allowlist_failed", + ConstationFailReason.NONCE_ALREADY_CONSUMED: "replayed_nonce", + ConstationFailReason.NONCE_UNKNOWN: "unknown_nonce", + ConstationFailReason.NONCE_EXPIRED: "expired_nonce", + ConstationFailReason.NONCE_WORK_UNIT_MISMATCH: "nonce_work_unit_mismatch", + ConstationFailReason.NONCE_MINER_HOTKEY_MISMATCH: "nonce_hotkey_mismatch", + ConstationFailReason.NONCE_POD_MISMATCH: "nonce_pod_mismatch", + ConstationFailReason.NONCE_FAILED: "nonce_failed", + ConstationFailReason.SIGNATURE_INVALID: "signature_invalid", + ConstationFailReason.SEALED_MANIFEST_MISMATCH: "manifest_mismatch", + ConstationFailReason.CORROBORATION_MISMATCH: "corroboration_mismatch", + ConstationFailReason.CONSTATION_GAP: "constation_gap", +} + + +def classify_constation_fault(result: ConstationResult) -> str: + """Return ``miner_fault:`` or ``infra_fault:`` for a failed result. + + A successful result returns ``ok``. Missing-bundle / service-down cases are + classified by callers via :func:`infra_fault_reason` / :func:`miner_fault_reason`. + """ + if result.ok: + return "ok" + bare = _MINER_BARE_CODES.get(result.reason, str(result.reason.value)) + if result.reason in _MINER_FAIL_REASONS: + return f"miner_fault:{bare}" + return f"infra_fault:{bare}" + + +def miner_fault_reason(code: str) -> str: + bare = code.removeprefix("miner_fault:").strip() or "unknown" + return f"miner_fault:{bare}" + + +def infra_fault_reason(code: str) -> str: + bare = code.removeprefix("infra_fault:").strip() or "unknown" + return f"infra_fault:{bare}" + + +def constation_bundle_to_dict(bundle: ConstationBundle) -> dict[str, object]: + """Serialize a :class:`ConstationBundle` for wire / result payload embedding.""" + return { + "commit_sha": bundle.commit_sha, + "tree_sha": bundle.tree_sha, + "variant": bundle.variant, + "digest": bundle.digest, + "work_unit_id": bundle.work_unit_id, + "miner_hotkey": bundle.miner_hotkey, + "pod_id": bundle.pod_id, + "nonce": bundle.nonce, + "signed_attestation": bundle.signed_attestation, + "expected_sealed_manifest_hashes": dict(bundle.expected_sealed_manifest_hashes), + "reported_sealed_manifest_hashes": dict(bundle.reported_sealed_manifest_hashes), + "lium_declared_digest": bundle.lium_declared_digest, + "constation_gap_budget_seconds": float(bundle.constation_gap_budget_seconds), + "constation_observed_max_gap_seconds": float(bundle.constation_observed_max_gap_seconds), + } + + +def constation_bundle_from_dict(raw: object) -> ConstationBundle: + """Parse a wire dict into :class:`ConstationBundle` (boundary validation).""" + if not isinstance(raw, dict): + raise ValueError("constation_bundle must be an object") + required = ( + "commit_sha", + "tree_sha", + "variant", + "digest", + "work_unit_id", + "miner_hotkey", + "pod_id", + "nonce", + "signed_attestation", + "expected_sealed_manifest_hashes", + "reported_sealed_manifest_hashes", + "constation_gap_budget_seconds", + "constation_observed_max_gap_seconds", + ) + missing = [k for k in required if k not in raw] + if missing: + raise ValueError(f"constation_bundle missing fields: {missing}") + exp = raw["expected_sealed_manifest_hashes"] + rep = raw["reported_sealed_manifest_hashes"] + if not isinstance(exp, Mapping) or not isinstance(rep, Mapping): + raise ValueError("sealed manifest hashes must be objects") + lium = raw.get("lium_declared_digest") + if lium is not None and not isinstance(lium, str): + raise ValueError("lium_declared_digest must be string or null") + return ConstationBundle( + commit_sha=str(raw["commit_sha"]), + tree_sha=str(raw["tree_sha"]), + variant=str(raw["variant"]), + digest=str(raw["digest"]), + work_unit_id=str(raw["work_unit_id"]), + miner_hotkey=str(raw["miner_hotkey"]), + pod_id=str(raw["pod_id"]), + nonce=str(raw["nonce"]), + signed_attestation=raw["signed_attestation"], + expected_sealed_manifest_hashes={str(k): str(v) for k, v in exp.items()}, + reported_sealed_manifest_hashes={str(k): str(v) for k, v in rep.items()}, + lium_declared_digest=lium, + constation_gap_budget_seconds=float(raw["constation_gap_budget_seconds"]), + constation_observed_max_gap_seconds=float(raw["constation_observed_max_gap_seconds"]), + ) + + +__all__ = [ + "AllowlistChecker", + "CheckOutcome", + "ConstationBundle", + "ConstationFailReason", + "ConstationResult", + "NonceChecker", + "SignatureVerifier", + "adapt_allowlist_lookup", + "adapt_attestation_verify", + "adapt_nonce_consume", + "constation_ok", + "classify_constation_fault", + "miner_fault_reason", + "infra_fault_reason", + "INFRA_FAULT_CONSTATION_UNAVAILABLE", + "INFRA_FAULT_LIUM_5XX", + "INFRA_FAULT_NETWORK_PARTITION", + "INFRA_FAULT_RETRY_EXHAUSTED", + "constation_bundle_to_dict", + "constation_bundle_from_dict", +] diff --git a/packages/challenges/prism/src/prism_challenge/constation_checkers.py b/packages/challenges/prism/src/prism_challenge/constation_checkers.py new file mode 100644 index 000000000..3a0e38a17 --- /dev/null +++ b/packages/challenges/prism/src/prism_challenge/constation_checkers.py @@ -0,0 +1,89 @@ +"""BASE-backed HTTP checkers for prism production constation_ok.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from .constation import CheckOutcome + + +class BaseHttpConstationClient: + """Thin client for BASE internal constation checker endpoints.""" + + def __init__( + self, + *, + base_url: str, + token: str, + timeout_s: float = 10.0, + transport: httpx.AsyncBaseTransport | httpx.BaseTransport | None = None, + ) -> None: + if not base_url.strip(): + raise ValueError("constation base_url must be non-empty") + self._base_url = base_url.rstrip("/") + self._token = token + self._timeout_s = timeout_s + self._transport = transport + + def _headers(self) -> dict[str, str]: + return { + "Authorization": f"Bearer {self._token}", + "Accept": "application/json", + "Content-Type": "application/json", + } + + def check_allowlist( + self, + *, + digest: str, + commit_sha: str, + tree_sha: str, + variant: str, + ) -> CheckOutcome: + body = { + "digest": digest, + "commit_sha": commit_sha, + "tree_sha": tree_sha, + "variant": variant, + } + return self._post_outcome("/internal/v1/constation/check_allowlist", body) + + def check_nonce( + self, + *, + nonce: str, + work_unit_id: str, + miner_hotkey: str, + pod_id: str, + ) -> CheckOutcome: + body = { + "nonce": nonce, + "work_unit_id": work_unit_id, + "miner_hotkey": miner_hotkey, + "pod_id": pod_id, + } + return self._post_outcome("/internal/v1/constation/check_nonce", body) + + def verify_signature(self, signed: object) -> CheckOutcome: + body = {"signed": signed if isinstance(signed, dict) else {"value": signed}} + return self._post_outcome("/internal/v1/constation/verify_attestation", body) + + def _post_outcome(self, path: str, body: dict[str, Any]) -> CheckOutcome: + url = f"{self._base_url}{path}" + try: + with httpx.Client(timeout=self._timeout_s, transport=self._transport) as client: + response = client.post(url, json=body, headers=self._headers()) + response.raise_for_status() + data = response.json() + except Exception as exc: # noqa: BLE001 - map to checker fail-closed + return CheckOutcome(ok=False, reason=f"checker_transport_error:{exc}") + if not isinstance(data, dict): + return CheckOutcome(ok=False, reason="checker_malformed_response") + ok = bool(data.get("ok")) + reason = data.get("reason", "ok" if ok else "checker_rejected") + return CheckOutcome(ok=ok, reason=str(reason)) + + +__all__ = ["BaseHttpConstationClient"] diff --git a/packages/challenges/prism/src/prism_challenge/evaluator/scale_eval.py b/packages/challenges/prism/src/prism_challenge/evaluator/scale_eval.py index 8b598dc82..4aa59a626 100644 --- a/packages/challenges/prism/src/prism_challenge/evaluator/scale_eval.py +++ b/packages/challenges/prism/src/prism_challenge/evaluator/scale_eval.py @@ -518,9 +518,7 @@ def probe_full_scale_readiness( reasons: list[str] = [] checks: dict[str, bool] = {} - ok_train, reason_train = _path_ready( - train_p, label="train", require_manifest=require_manifest - ) + ok_train, reason_train = _path_ready(train_p, label="train", require_manifest=require_manifest) checks["train_mount"] = ok_train if reason_train: reasons.append(reason_train) diff --git a/packages/challenges/prism/src/prism_challenge/ingestion.py b/packages/challenges/prism/src/prism_challenge/ingestion.py index 0e8f83c5c..04fb9bd6f 100644 --- a/packages/challenges/prism/src/prism_challenge/ingestion.py +++ b/packages/challenges/prism/src/prism_challenge/ingestion.py @@ -35,12 +35,35 @@ from .audit import AuditSampler, effective_tier from .auth import verify_hotkey_signature +from .breakglass import ( + BreakGlassAuditLog, + BreakGlassRequest, + evaluate_break_glass, + fault_class_of, +) +from .constation import ( + INFRA_FAULT_CONSTATION_UNAVAILABLE, + INFRA_FAULT_RETRY_EXHAUSTED, + AllowlistChecker, + ConstationBundle, + ConstationResult, + NonceChecker, + classify_constation_fault, + constation_ok, + infra_fault_reason, + miner_fault_reason, +) +from .constation import ( + SignatureVerifier as ConstationSignatureVerifier, +) from .plausibility import check_manifest_plausibility from .proof import ( + ATTESTATION_MODE_V1, EXECUTION_PROOF_VERSION, MANIFEST_PAYLOAD_KEY, PROOF_PAYLOAD_KEY, ExecutionProof, + attestation_mode_of, compute_manifest_sha256, verify_execution_proof, ) @@ -51,6 +74,9 @@ #: A verified 64-char lowercase-hex manifest digest (VAL-PRISM-018). _MANIFEST_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +#: Bounded retry before discarding an infra-fault run (todo 22). +DEFAULT_MAX_CONSTATION_ATTEMPTS = 3 + SignatureVerifier = Callable[[str, bytes, str], bool] @@ -100,7 +126,7 @@ def __init__(self, reason: str, message: str = "") -> None: class IngestionOutcome: """The observable outcome of ingesting one forwarded result.""" - status: str # "accepted" | "conflict" + status: str # "accepted" | "conflict" | "rejected" work_unit_id: str submission_id: str claimed_tier: int @@ -112,6 +138,9 @@ class IngestionOutcome: audit_sampled: bool | None = None audit_unit_id: str | None = None reason: str | None = None + attestation_mode: str | None = None + break_glass_admitted: bool = False + score_written: bool = False def to_response(self) -> dict[str, Any]: payload: dict[str, Any] = { @@ -124,6 +153,8 @@ def to_response(self) -> dict[str, Any]: "idempotent": self.idempotent, "finalized": self.finalized, "submission_status": self.submission_status, + "score_written": self.score_written, + "break_glass_admitted": self.break_glass_admitted, } if self.audit_sampled is not None: payload["audit_sampled"] = self.audit_sampled @@ -131,6 +162,8 @@ def to_response(self) -> dict[str, Any]: payload["audit_unit_id"] = self.audit_unit_id if self.reason is not None: payload["reason"] = self.reason + if self.attestation_mode is not None: + payload["attestation_mode"] = self.attestation_mode return payload @@ -204,22 +237,33 @@ async def ingest_work_unit_result( pinned_image_digest: str | None = None, audit_sampler: AuditSampler | None = None, verify: SignatureVerifier = verify_hotkey_signature, + constation_bundle: ConstationBundle | None = None, + check_allowlist: AllowlistChecker | None = None, + check_nonce: NonceChecker | None = None, + verify_constation_signature: ConstationSignatureVerifier | None = None, + constation_infra_fault: str | None = None, + constation_attempt: int = 1, + max_constation_attempts: int = DEFAULT_MAX_CONSTATION_ATTEMPTS, + break_glass: BreakGlassRequest | None = None, + break_glass_audit_log: BreakGlassAuditLog | None = None, ) -> IngestionOutcome: - """Verify a forwarded worker result and finalize the submission idempotently. + """Verify a forwarded worker result and finalize under strict fail-closed constation (P1). ``work_unit_id`` is prism's stable unit id (``== submission_id``). Verification (shape -> integrity) runs BEFORE any scoring; a rejected result raises :class:`ResultIngestionError` and - leaves the submission untouched (eligible for retry). A verified first delivery is then run - through the plausibility gate (architecture.md 3.5; VAL-PRISM-009): an implausible manifest - raises :class:`~prism_challenge.plausibility.PlausibilityError` (a reason DISTINCT from the - proof-verification reasons) and is never scored, while a plausible manifest passes through - UNCHANGED and finalizes via the CAS-guarded worker path. A duplicate is an idempotent no-op and - a conflicting redelivery for an already-accepted unit is refused so the stored score/leaderboard - is never mutated. - - Effective tier uses IMAGE_PIN only (max tier 1). Claimed attestation fields never elevate and - never block score finalization (Prism has no TEE-required scoring path). + leaves the submission untouched (eligible for retry). + + **P1 fail-closed (todo 22):** no valid constation bundle ⇒ **no** ``final_score`` row is + written at all. Failures carry ``miner_fault:*`` or ``infra_fault:*`` reason codes. + Infra-fault runs may be admitted only via an audited operator break-glass (todo 23); + miner-fault runs can never be override-admitted. Bounded retry applies to infra faults + before discard. + + Effective tier is granted only via ``constation_ok`` (M14 / todo 21); max tier is 1. + Claimed attestation / TEE fields never elevate. Self-reported ``PRISM_IMAGE_DIGEST`` is + telemetry only. """ + del submission_ref # reserved for future cross-checks if not isinstance(result, Mapping): raise ResultIngestionError("result_malformed", "result must be an object") @@ -229,15 +273,11 @@ async def ingest_work_unit_result( manifest = raw_manifest if isinstance(raw_manifest, Mapping) else None verify_proof_integrity(proof, unit_id=work_unit_id, manifest=manifest, verify=verify) - tier = effective_tier(proof, pinned_image_digest=pinned_image_digest) claimed_tier = int(proof.tier) - downgraded = claimed_tier != tier submission_id = work_unit_id - # Replication at acceptance (R=1 degraded or R=2 reconciled), forwarded by base for - # observability. It never affects audit eligibility: R=1 results are audited at their - # effective-tier rate exactly like R=2 ones (VAL-PRISM-026). replication = _as_int(result.get("replication"), 2) repository = worker.repository + att_mode = attestation_mode_of(proof) or ATTESTATION_MODE_V1 existing = await repository.get_work_unit_result(work_unit_id) if existing is not None: @@ -248,11 +288,13 @@ async def ingest_work_unit_result( work_unit_id=work_unit_id, submission_id=submission_id, claimed_tier=_as_int(existing.get("claimed_tier"), claimed_tier), - effective_tier=_as_int(existing.get("effective_tier"), tier), - tier_downgraded=bool(existing.get("tier_downgraded", downgraded)), + effective_tier=_as_int(existing.get("effective_tier"), 0), + tier_downgraded=bool(existing.get("tier_downgraded", True)), idempotent=True, finalized=False, submission_status=await repository.submission_status(submission_id), + attestation_mode=att_mode, + score_written=True, ) logger.warning( "rejecting conflicting result delivery for finalized work unit %s " @@ -266,14 +308,81 @@ async def ingest_work_unit_result( work_unit_id=work_unit_id, submission_id=submission_id, claimed_tier=claimed_tier, - effective_tier=tier, - tier_downgraded=downgraded, + effective_tier=0, + tier_downgraded=True, idempotent=False, finalized=False, submission_status=await repository.submission_status(submission_id), reason="manifest_conflict", + attestation_mode=att_mode, + score_written=True, ) + # --- Constation gate (P1) ----------------------------------------------------------------- + gate = _evaluate_constation_gate( + bundle=constation_bundle, + check_allowlist=check_allowlist, + check_nonce=check_nonce, + verify_constation_signature=verify_constation_signature, + constation_infra_fault=constation_infra_fault, + constation_attempt=constation_attempt, + max_constation_attempts=max_constation_attempts, + ) + break_glass_admitted = False + if not gate.admit: + if gate.retryable: + raise ResultIngestionError( + gate.reason or "infra_fault:constation_retry", + gate.message or "constation infra fault; retryable", + ) + # Optional break-glass for infra_fault only. + if break_glass is not None and fault_class_of(gate.reason or "") == "infra_fault": + decision = evaluate_break_glass( + break_glass, + fault_reason=gate.reason or infra_fault_reason(INFRA_FAULT_CONSTATION_UNAVAILABLE), + audit_log=break_glass_audit_log, + ) + if decision.admitted: + break_glass_admitted = True + logger.warning( + "break-glass admitted infra-fault run %s by operator %s", + work_unit_id, + break_glass.operator_id, + ) + else: + return await _reject_no_score( + work_unit_id=work_unit_id, + submission_id=submission_id, + claimed_tier=claimed_tier, + reason=gate.reason or miner_fault_reason("missing_constation_bundle"), + attestation_mode=att_mode, + repository=repository, + ) + else: + if break_glass is not None and fault_class_of(gate.reason or "") == "miner_fault": + # Explicit refuse path for attempted miner_fault override (todo 23). + evaluate_break_glass( + break_glass, + fault_reason=gate.reason or miner_fault_reason("unknown"), + audit_log=break_glass_audit_log, + ) + return await _reject_no_score( + work_unit_id=work_unit_id, + submission_id=submission_id, + claimed_tier=claimed_tier, + reason=gate.reason or miner_fault_reason("missing_constation_bundle"), + attestation_mode=att_mode, + repository=repository, + ) + + # Break-glass admits a score at tier 0 only — elevation still requires real constation_ok. + tier = effective_tier( + proof, + pinned_image_digest=pinned_image_digest, + constation_ok_result=bool(gate.constation_ok) and not break_glass_admitted, + ) + downgraded = claimed_tier != tier + if downgraded: logger.warning( "downgrading unverifiable tier claim for work unit %s: claimed %d -> effective %d", @@ -289,8 +398,6 @@ async def ingest_work_unit_result( ) if worker.settings.worker_plane.enabled: - # Worker plane: finalize from the forwarded, verified+reconciled manifest WITHOUT - # re-executing the evaluator (the heavy GPU work already ran on the miner-funded worker). if manifest is None: raise ResultIngestionError( "manifest_missing", @@ -302,15 +409,11 @@ async def ingest_work_unit_result( dict(manifest), ) except WorkerFinalizationError as exc: - # An internal/transient derivation failure is NOT a clean finalize: nothing is recorded - # (so a redelivery is genuinely retried, not idempotent-skipped) and the submission was - # reverted to pending. Surface it with a distinct, retryable reason. raise ResultIngestionError( "finalization_failed", f"worker-plane finalization failed transiently and is retryable: {exc}", ) from exc else: - # Flag OFF: legacy in-process re-execution finalization, byte-for-byte unchanged. result_id = await worker.process_submission(submission_id) submission_status = await repository.submission_status(submission_id) await repository.record_work_unit_result( @@ -326,10 +429,6 @@ async def ingest_work_unit_result( audit_unit_id: str | None = None if audit_sampler is not None: audit_sampled = audit_sampler.should_sample(work_unit_id=work_unit_id, effective_tier=tier) - # A sampled accepted result gets a validator audit unit on the existing dispatch path with a - # DISTINCT id; the audited submission is NOT reverted to pending (VAL-PRISM-012). R=1 - # (replication-degraded) results are sampled and audited at their effective-tier rate just - # like R=2-reconciled ones -- they are never exempted (VAL-PRISM-026). if audit_sampled: audit_unit_id = await repository.create_audit_unit( submission_id=submission_id, @@ -350,10 +449,131 @@ async def ingest_work_unit_result( submission_status=submission_status, audit_sampled=audit_sampled, audit_unit_id=audit_unit_id, + attestation_mode=att_mode, + break_glass_admitted=break_glass_admitted, + score_written=result_id is not None, + reason=gate.reason if break_glass_admitted else None, + ) + + +@dataclass(frozen=True) +class _ConstationGate: + admit: bool + constation_ok: bool + reason: str | None = None + message: str | None = None + retryable: bool = False + + +def _evaluate_constation_gate( + *, + bundle: ConstationBundle | None, + check_allowlist: AllowlistChecker | None, + check_nonce: NonceChecker | None, + verify_constation_signature: ConstationSignatureVerifier | None, + constation_infra_fault: str | None, + constation_attempt: int, + max_constation_attempts: int, +) -> _ConstationGate: + """Decide whether scoring may proceed under P1 fail-closed policy.""" + if constation_infra_fault: + reason = infra_fault_reason(constation_infra_fault) + if constation_attempt < max_constation_attempts: + return _ConstationGate( + admit=False, + constation_ok=False, + reason=reason, + message=f"infra fault attempt {constation_attempt}/{max_constation_attempts}", + retryable=True, + ) + return _ConstationGate( + admit=False, + constation_ok=False, + reason=infra_fault_reason(INFRA_FAULT_RETRY_EXHAUSTED) + if constation_attempt >= max_constation_attempts + else reason, + message="infra fault retry budget exhausted; no score", + retryable=False, + ) + + if bundle is None: + return _ConstationGate( + admit=False, + constation_ok=False, + reason=miner_fault_reason("missing_constation_bundle"), + message="no constation bundle; no score (P1)", + ) + + if check_allowlist is None or check_nonce is None or verify_constation_signature is None: + # Bundle present but checkers unavailable → infra (cannot evaluate). + reason = infra_fault_reason(INFRA_FAULT_CONSTATION_UNAVAILABLE) + if constation_attempt < max_constation_attempts: + return _ConstationGate( + admit=False, + constation_ok=False, + reason=reason, + message="constation checkers unavailable; retryable", + retryable=True, + ) + return _ConstationGate( + admit=False, + constation_ok=False, + reason=infra_fault_reason(INFRA_FAULT_RETRY_EXHAUSTED), + message="constation checkers unavailable; retries exhausted", + ) + + result: ConstationResult = constation_ok( + bundle, + check_allowlist=check_allowlist, + check_nonce=check_nonce, + verify_signature=verify_constation_signature, + ) + if result.ok: + return _ConstationGate(admit=True, constation_ok=True, reason="ok") + + fault = classify_constation_fault(result) + return _ConstationGate( + admit=False, + constation_ok=False, + reason=fault, + message=f"constation_ok failed: {result.reason.value}", + ) + + +async def _reject_no_score( + *, + work_unit_id: str, + submission_id: str, + claimed_tier: int, + reason: str, + attestation_mode: str, + repository: Any, +) -> IngestionOutcome: + """Return a rejected outcome without writing a score row (P1).""" + logger.warning( + "fail-closed: refusing score for work unit %s reason=%s", + work_unit_id, + reason, + ) + status = await repository.submission_status(submission_id) + return IngestionOutcome( + status="rejected", + work_unit_id=work_unit_id, + submission_id=submission_id, + claimed_tier=claimed_tier, + effective_tier=0, + tier_downgraded=claimed_tier != 0, + idempotent=False, + finalized=False, + submission_status=status, + reason=reason, + attestation_mode=attestation_mode, + score_written=False, ) __all__ = [ + "DEFAULT_MAX_CONSTATION_ATTEMPTS", "IngestionOutcome", "ResultIngestionError", "ingest_work_unit_result", diff --git a/packages/challenges/prism/src/prism_challenge/proof.py b/packages/challenges/prism/src/prism_challenge/proof.py index 188a53d1c..3c994b688 100644 --- a/packages/challenges/prism/src/prism_challenge/proof.py +++ b/packages/challenges/prism/src/prism_challenge/proof.py @@ -10,13 +10,18 @@ ``{manifest_sha256}:{unit_id}`` -- so a proof prism emits verifies with the same code as one the base worker plane emits, and a proof cannot be replayed across units. -Tiers (architecture.md 3.4): +Tiers (architecture.md 3.4; no-TEE residual): * tier 0 -- mandatory, all backends: canonical manifest hash + worker signature. -* tier 1 -- BOTH a pinned ``image_digest`` AND pod metadata (``provider.pod_id``). -* tier 2 -- CLAIMED when a closed structured attestation is present; EFFECTIVE tier 2 is granted - only after Prism TEE verification succeeds (LOCAL-FIXTURE PASS today). Opaque non-empty - ``tdx_quote_b64`` / ``gpu_eat_jwt`` alone never elevate effective tier. +* tier 1 -- CLAIMED when a pinned ``image_digest`` AND pod metadata are present; EFFECTIVE tier 1 + is granted **only** when :func:`~prism_challenge.constation.constation_ok` is True (M14). + Self-reported ``PRISM_IMAGE_DIGEST`` is telemetry only and never elevates. +* tier 2 -- may still be *claimed* for wire compatibility when a closed structured attestation + shape is present. EFFECTIVE tier is **never** 2: Prism has no TEE verifier path. Claimed tier + >= 2 collapses to effective 0. Opaque ``tdx_quote_b64`` / ``gpu_eat_jwt`` never elevate. + +Every emitted proof carries ``attestation_mode=miner_rent_image_pin_evidence_v1`` (image-identity +tamper-evidence on a miner-rented pod). Never ``lium_attested``, never any TEE-implying mode. Security invariant (VAL-PRISM-008): proof construction reads ONLY the manifest, the work unit id, the worker signer, and a FIXED ALLOWLIST of non-secret provider env vars. It never reads the @@ -75,9 +80,25 @@ ATTESTATION_ENV, ) -#: Documented attestation payload keys (architecture.md 3.4). +#: Documented attestation payload keys (architecture.md 3.4) — claim-shape only, never elevation. ATTESTATION_KEYS: tuple[str, ...] = ("tdx_quote_b64", "gpu_eat_jwt") +#: Sole honest attestation_mode value (todo 20). Never TEE-named, never "lium_attested". +ATTESTATION_MODE_V1 = "miner_rent_image_pin_evidence_v1" +ATTESTATION_MODE_KEY = "attestation_mode" +#: Forbidden mode strings that would overclaim independent/TEE verification. +FORBIDDEN_ATTESTATION_MODES: frozenset[str] = frozenset( + { + "lium_attested", + "tee", + "tee_attested", + "tdx", + "sev", + "cvm", + "hardware_root_of_trust", + } +) + @runtime_checkable class WorkerSigner(Protocol): @@ -157,27 +178,22 @@ def has_attestation(attestation: Any) -> bool: WARNING: this is ONLY a shape/presence hint for CLAIMED-tier emission compatibility. It is NEVER cryptographic verification and NEVER elevates effective tier. Prism has no - TEE verifier; effective tier is IMAGE_PIN only (max 1) via - :func:`~prism_challenge.audit.effective_tier`. + TEE verifier; effective tier is granted only via + :func:`~prism_challenge.constation.constation_ok` (max 1). """ if not isinstance(attestation, Mapping): return False tdx = attestation.get("tdx_quote_b64") gpu = attestation.get("gpu_eat_jwt") - return ( - isinstance(tdx, str) - and bool(tdx.strip()) - and isinstance(gpu, str) - and bool(gpu.strip()) - ) + return isinstance(tdx, str) and bool(tdx.strip()) and isinstance(gpu, str) and bool(gpu.strip()) def has_structured_attestation_claim(attestation: Any) -> bool: """Whether attestation claims the closed prism.tee.v1 shape (compat claim only; unverified). Used only to decide the CLAIMED emission tier for wire compatibility. Effective tier is - never elevated from this claim (max effective tier is 1 via IMAGE_PIN). + never elevated from this claim (max effective tier is 1 via constation_ok). """ if not has_attestation(attestation): @@ -194,6 +210,48 @@ def has_structured_attestation_claim(attestation: Any) -> bool: ) +def normalize_attestation_mode(mode: str | None) -> str: + """Return the sole honest mode; reject TEE/overclaim labels.""" + value = (mode or ATTESTATION_MODE_V1).strip() + if not value: + value = ATTESTATION_MODE_V1 + lowered = value.lower() + if lowered in FORBIDDEN_ATTESTATION_MODES or "tee" in lowered or "tdx" in lowered: + raise ValueError( + f"attestation_mode {value!r} is forbidden (no TEE / no independent Lium claim)" + ) + if value != ATTESTATION_MODE_V1: + raise ValueError(f"attestation_mode must be {ATTESTATION_MODE_V1!r}, got {value!r}") + return ATTESTATION_MODE_V1 + + +def attach_attestation_mode( + attestation: dict[str, Any] | None = None, + *, + mode: str = ATTESTATION_MODE_V1, +) -> dict[str, Any]: + """Return an attestation dict carrying the honest ``attestation_mode`` field.""" + out: dict[str, Any] = dict(attestation or {}) + out[ATTESTATION_MODE_KEY] = normalize_attestation_mode(mode) + return out + + +def attestation_mode_of(proof_or_attestation: Any) -> str | None: + """Read ``attestation_mode`` from a proof or attestation mapping, if present.""" + if proof_or_attestation is None: + return None + att = getattr(proof_or_attestation, "attestation", None) + if att is None and isinstance(proof_or_attestation, Mapping): + if ATTESTATION_MODE_KEY in proof_or_attestation: + raw = proof_or_attestation.get(ATTESTATION_MODE_KEY) + return str(raw) if raw is not None else None + att = proof_or_attestation.get("attestation") + if isinstance(att, Mapping): + raw = att.get(ATTESTATION_MODE_KEY) + return str(raw) if raw is not None else None + return None + + def compute_tier( *, image_digest: str | None, @@ -204,9 +262,10 @@ def compute_tier( tier 2 may still be *claimed* for a closed structured attestation shape (wire compat; never mere non-empty opaque strings). The claimed tier is NOT trusted at verification: - :func:`~prism_challenge.audit.effective_tier` recomputes from IMAGE_PIN only and never - elevates above tier 1. tier 1 iff BOTH a pinned image digest AND pod metadata - (``provider.pod_id``) are present; else tier 0. + :func:`~prism_challenge.audit.effective_tier` grants tier 1 only via constation_ok and + never elevates above tier 1. tier 1 is *claimed* iff BOTH an image digest AND pod + metadata (``provider.pod_id``) are present; else tier 0. Self-reported env digests do + not elevate. """ if has_structured_attestation_claim(attestation): @@ -232,12 +291,26 @@ def provider_from_env(env: Mapping[str, str] | None = None) -> ProviderInfo | No def image_digest_from_env(env: Mapping[str, str] | None = None) -> str | None: - """Read the evaluator image digest from the injected provider env.""" + """Read self-reported ``PRISM_IMAGE_DIGEST`` for **telemetry only**. + + This value must never be used as an elevation source. Elevation digests come + exclusively from a constation record (todo 20 / B5). + """ env = os.environ if env is None else env return _clean(env.get(IMAGE_DIGEST_ENV)) +def elevation_image_digest( + *, + constation_digest: str | None, + env_digest: str | None = None, +) -> str | None: + """Digest used for elevation: constation record only. ``env_digest`` is ignored.""" + del env_digest # telemetry only — never elevates + return _clean(constation_digest) + + def attestation_from_env(env: Mapping[str, str] | None = None) -> dict[str, Any] | None: """Read + parse the attestation payload (JSON) from the injected provider env, if any.""" @@ -261,15 +334,29 @@ def build_execution_proof( image_digest: str | None = None, attestation: dict[str, Any] | None = None, tier: ExecutionProofTier | None = None, + constation_digest: str | None = None, + attestation_mode: str = ATTESTATION_MODE_V1, ) -> ExecutionProof: """Build and sign an ExecutionProof binding ``manifest_sha256`` to ``unit_id`` under ``signer``. The tier is computed from the provenance unless explicitly overridden. ``signer`` is the WORKER keypair; its public identity becomes ``worker_signature.worker_pubkey``. + + When ``constation_digest`` is provided it is the elevation digest written on the proof; + otherwise ``image_digest`` is stored as telemetry only (still may appear on the wire for + observability). Every proof carries ``attestation_mode=miner_rent_image_pin_evidence_v1``. """ - effective_tier: ExecutionProofTier = ( - compute_tier(image_digest=image_digest, provider=provider, attestation=attestation) + digest_for_proof = elevation_image_digest( + constation_digest=constation_digest, env_digest=image_digest + ) + if digest_for_proof is None: + digest_for_proof = image_digest # telemetry / claim shape only + + att = attach_attestation_mode(attestation, mode=attestation_mode) + + claimed_tier: ExecutionProofTier = ( + compute_tier(image_digest=digest_for_proof, provider=provider, attestation=att) if tier is None else tier ) @@ -278,12 +365,12 @@ def build_execution_proof( ) return ExecutionProof( version=cast(ExecutionProofVersion, EXECUTION_PROOF_VERSION), - tier=effective_tier, + tier=claimed_tier, manifest_sha256=manifest_sha256, - image_digest=image_digest, + image_digest=digest_for_proof, provider=provider, worker_signature=WorkerSignature(worker_pubkey=signer.worker_pubkey, sig=signature), - attestation=attestation, + attestation=att, ) @@ -295,23 +382,28 @@ def build_execution_proof_from_manifest( manifest_bytes: bytes | None = None, manifest_path: str | os.PathLike[str] | None = None, env: Mapping[str, str] | None = None, + constation_digest: str | None = None, ) -> ExecutionProof: """Build a signed proof from a manifest source + the injected provider env. Exactly ONE manifest source must be given. Prefer ``manifest_path`` at emission time so the hash is taken from the exact on-disk bytes of ``prism_run_manifest.v2.json``. The provider provenance is read ONLY from the non-secret provider env allowlist. + + ``PRISM_IMAGE_DIGEST`` from env is telemetry; pass ``constation_digest`` for elevation. """ digest = _resolve_manifest_sha256( manifest=manifest, manifest_bytes=manifest_bytes, manifest_path=manifest_path ) + env_digest = image_digest_from_env(env) return build_execution_proof( signer=signer, manifest_sha256=digest, unit_id=unit_id, provider=provider_from_env(env), - image_digest=image_digest_from_env(env), + image_digest=env_digest, + constation_digest=constation_digest, attestation=attestation_from_env(env), ) @@ -361,8 +453,11 @@ def _clean(value: Any) -> str | None: __all__ = [ "ATTESTATION_ENV", "ATTESTATION_KEYS", + "ATTESTATION_MODE_KEY", + "ATTESTATION_MODE_V1", "EXECUTION_PROOF_VERSION", "EXECUTOR_ID_ENV", + "FORBIDDEN_ATTESTATION_MODES", "IMAGE_DIGEST_ENV", "MINER_HOTKEY_ENV", "POD_ID_ENV", @@ -375,17 +470,21 @@ def _clean(value: Any) -> str | None: "ProviderInfo", "WorkerSignature", "WorkerSigner", + "attach_attestation_mode", "attestation_from_env", + "attestation_mode_of", "build_execution_proof", "build_execution_proof_from_manifest", "canonical_manifest_json", "compute_manifest_sha256", "compute_tier", + "elevation_image_digest", "execution_proof_signing_payload", "has_attestation", "has_structured_attestation_claim", "image_digest_from_env", "manifest_sha256_from_bytes", + "normalize_attestation_mode", "provider_from_env", "read_manifest_sha256", "verify_execution_proof", diff --git a/packages/challenges/prism/src/prism_challenge/queue.py b/packages/challenges/prism/src/prism_challenge/queue.py index 6390aa5c4..2e59ff06b 100644 --- a/packages/challenges/prism/src/prism_challenge/queue.py +++ b/packages/challenges/prism/src/prism_challenge/queue.py @@ -56,7 +56,46 @@ CONTAINER_EXECUTION_BACKENDS = frozenset( {"base_container", "base_gpu", "container_gpu", "docker_gpu"} ) +#: Always-on backends (no constation bundle required at worker construction). SUPPORTED_EXECUTION_BACKENDS = CONTAINER_EXECUTION_BACKENDS +#: Lium is gated: permitted only when a full constation bundle is present (todo 19). +LIUM_EXECUTION_BACKEND = "lium" +GATED_EXECUTION_BACKENDS = frozenset({LIUM_EXECUTION_BACKEND}) + + +def is_execution_backend_supported( + backend: str, + *, + constation_bundle: object | None = None, +) -> bool: + """Whether ``backend`` may be used under the current constation gate. + + Container backends (``base_gpu``, …) are always allowed. ``lium`` is allowed + **only** when a full constation bundle object is supplied — bare Lium without + a bundle stays rejected. This does not evaluate ``constation_ok``; that is the + ingestion elevation path (todos 21–22). + """ + if backend in SUPPORTED_EXECUTION_BACKENDS: + return True + if backend == LIUM_EXECUTION_BACKEND: + return constation_bundle is not None + return False + + +def require_execution_backend( + backend: str, + *, + constation_bundle: object | None = None, +) -> None: + """Raise ``ValueError`` when ``backend`` is not permitted under the gate.""" + if is_execution_backend_supported(backend, constation_bundle=constation_bundle): + return + if backend == LIUM_EXECUTION_BACKEND: + raise ValueError( + f"Unsupported execution backend: {backend}: constation bundle required for lium" + ) + raise ValueError(f"Unsupported execution backend: {backend}") + logger = logging.getLogger(__name__) @@ -140,15 +179,16 @@ def __init__( settings: PrismSettings | None = None, evaluator_factory: EvaluatorFactory | None = None, checkpoint_publisher: CheckpointPublisher | None = None, + constation_bundle: object | None = None, ) -> None: - if execution_backend not in SUPPORTED_EXECUTION_BACKENDS: - raise ValueError(f"Unsupported execution backend: {execution_backend}") + require_execution_backend(execution_backend, constation_bundle=constation_bundle) self.repository = repository self.ctx = ctx self.execution_backend = execution_backend self.settings = settings or PrismSettings() self._evaluator_factory = evaluator_factory or _default_evaluator_factory self._checkpoint_publisher = checkpoint_publisher + self._constation_bundle = constation_bundle async def process_next(self) -> str | None: submission = await self.repository.claim_next() @@ -262,7 +302,10 @@ async def replay_audit_manifest_sha256( repeated (the honest run is deterministic, so an honest worker's hash reproduces). Returns ``None`` on any replay failure, resolving the audit inconclusive rather than confirming it. """ - if self.execution_backend not in CONTAINER_EXECUTION_BACKENDS: + # Lium runs are miner-side; validator audit replay uses the same container path. + if self.execution_backend not in CONTAINER_EXECUTION_BACKENDS and ( + self.execution_backend != LIUM_EXECUTION_BACKEND + ): return None submission = await self.repository.submission_execution_row(submission_id) if submission is None: @@ -351,7 +394,10 @@ async def _process_claimed( metadata = cast(dict[str, Any], raw_metadata) if isinstance(raw_metadata, dict) else {} hotkey = str(submission.get("hotkey") or "") code_hash = str(submission.get("code_hash") or sha256(code.encode()).hexdigest()) - if self.execution_backend in CONTAINER_EXECUTION_BACKENDS: + if ( + self.execution_backend in CONTAINER_EXECUTION_BACKENDS + or self.execution_backend == LIUM_EXECUTION_BACKEND + ): return await self._process_container( submission_id, code, diff --git a/packages/challenges/prism/src/prism_challenge/routes.py b/packages/challenges/prism/src/prism_challenge/routes.py index cbb647017..0a40ee651 100644 --- a/packages/challenges/prism/src/prism_challenge/routes.py +++ b/packages/challenges/prism/src/prism_challenge/routes.py @@ -16,6 +16,7 @@ from pydantic import ValidationError from .admission import enforce_admission +from .attestation_routes import build_attestation_public_router from .auth import authenticate_miner from .evaluator.train_series import downsample_train_series_for_api from .models import ( @@ -46,6 +47,10 @@ router = APIRouter(prefix="/v1") +# Public attestation challenge/answer (published via BASE proxy as +# /challenges/prism/v1/attestation/*). Lives on the challenge app, not master. +router.include_router(build_attestation_public_router()) + def _optional_float(value: object | None) -> float | None: if value is None: diff --git a/packages/challenges/prism/tests/conftest.py b/packages/challenges/prism/tests/conftest.py index 47e6d6216..558779633 100644 --- a/packages/challenges/prism/tests/conftest.py +++ b/packages/challenges/prism/tests/conftest.py @@ -101,3 +101,67 @@ def client(tmp_path: Path) -> TestClient: ) with TestClient(create_app(settings)) as test_client: yield test_client + + +# --- Wave 4 constation auto-inject for legacy tests (todo 22) ------------------------------------ +# Production P1: no bundle ⇒ no score. Legacy tests that never heard of constation omit the +# kwargs entirely; we inject a valid bundle so they keep exercising finalization/audit. +# Tests that pass ``constation_bundle=None`` (or ``constation_infra_fault=...``) explicitly +# exercise the fail-closed / break-glass paths and are left alone. + + +@pytest.fixture(autouse=True) +def _auto_constation_for_legacy_ingestion( + monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + # Production-path tests must exercise real fail-closed without the legacy seam. + modname = getattr(getattr(request, "module", None), "__name__", "") or "" + if "prod_constation" in modname or request.node.get_closest_marker("no_auto_constation"): + return + + import prism_challenge.ingestion as ingestion_mod + from prism_challenge.constation import CheckOutcome, ConstationBundle + + original = ingestion_mod.ingest_work_unit_result + + def _ok(**_k: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def _bundle() -> ConstationBundle: + man = {"legacy-test-harness.py": "a" * 64} + digest = "sha256:" + ("11" * 32) + return ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="legacy-wu", + miner_hotkey="legacy-hk", + pod_id="legacy-pod", + nonce="legacy-nonce", + signed_attestation={"legacy": True}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + async def _wrapped(**kwargs): # type: ignore[no-untyped-def] + if ( + "constation_bundle" not in kwargs + and "constation_infra_fault" not in kwargs + and kwargs.get("check_allowlist") is None + ): + kwargs = dict(kwargs) + kwargs["constation_bundle"] = _bundle() + kwargs["check_allowlist"] = _ok + kwargs["check_nonce"] = _ok + kwargs["verify_constation_signature"] = lambda _s: _ok() + return await original(**kwargs) + + monkeypatch.setattr(ingestion_mod, "ingest_work_unit_result", _wrapped) + # Patch the bound name on the requesting test module (from-import sites). + mod = getattr(request, "module", None) + if mod is not None and getattr(mod, "ingest_work_unit_result", None) is not None: + monkeypatch.setattr(mod, "ingest_work_unit_result", _wrapped, raising=False) diff --git a/packages/challenges/prism/tests/test_constation_bundle_wire.py b/packages/challenges/prism/tests/test_constation_bundle_wire.py new file mode 100644 index 000000000..3b879a00d --- /dev/null +++ b/packages/challenges/prism/tests/test_constation_bundle_wire.py @@ -0,0 +1,50 @@ +"""Wire serdes for ConstationBundle (production HTTP path).""" + +from __future__ import annotations + +import pytest + +from prism_challenge.constation import ( + ConstationBundle, + constation_bundle_from_dict, + constation_bundle_to_dict, +) + + +def _bundle() -> ConstationBundle: + man = {"h.py": "a" * 64} + digest = "sha256:" + ("1" * 64) + return ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="wu-1", + miner_hotkey="hk", + pod_id="pod", + nonce="nonce-1", + signed_attestation={"sig": "x"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + +def test_roundtrip() -> None: + b = _bundle() + again = constation_bundle_from_dict(constation_bundle_to_dict(b)) + assert again == b + + +def test_missing_field_raises() -> None: + raw = constation_bundle_to_dict(_bundle()) + del raw["nonce"] + with pytest.raises(ValueError, match="missing"): + constation_bundle_from_dict(raw) + + +def test_non_object_raises() -> None: + with pytest.raises(ValueError, match="object"): + constation_bundle_from_dict([]) diff --git a/packages/challenges/prism/tests/test_constation_ok.py b/packages/challenges/prism/tests/test_constation_ok.py new file mode 100644 index 000000000..9eb8beb9b --- /dev/null +++ b/packages/challenges/prism/tests/test_constation_ok.py @@ -0,0 +1,424 @@ +"""TDD tests for constation_ok — sole elevation predicate (checkbox 12 / M14). + +constation_ok is the ONLY path that may authorize tier elevation. Each required +mechanism is tested in isolation: omitting or breaking any one yields False with +a distinct machine reason code. Dependencies are injected so tests never need +live Lium or network. +""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError, dataclass +from typing import Any + +import pytest + +from prism_challenge.constation import ( + CheckOutcome, + ConstationBundle, + ConstationFailReason, + ConstationResult, + constation_ok, +) + +COMMIT = "a" * 40 +TREE = "c" * 40 +DIGEST = "sha256:" + ("1" * 64) +DIGEST_OTHER = "sha256:" + ("2" * 64) +NONCE = "550e8400-e29b-41d4-a716-446655440000" +POD = "pod_test_001" +HOTKEY = "5FakeHotkeyForUnitTestsOnly000000000000000" +WORK_UNIT = "wu-unit-001" +VARIANT = "cuda" +MANIFEST: dict[str, str] = { + "src/prism_recipe/harness.py": "a" * 64, + "src/prism_recipe/gpu_train.py": "b" * 64, +} +GAP_BUDGET = 30.0 + + +def _ok() -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + +def _miss(reason: str) -> CheckOutcome: + return CheckOutcome(ok=False, reason=reason) + + +def _bundle(**overrides: Any) -> ConstationBundle: + fields: dict[str, Any] = { + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": VARIANT, + "digest": DIGEST, + "work_unit_id": WORK_UNIT, + "miner_hotkey": HOTKEY, + "pod_id": POD, + "nonce": NONCE, + "signed_attestation": {"schema": "test", "signature": "deadbeef"}, + "expected_sealed_manifest_hashes": dict(MANIFEST), + "reported_sealed_manifest_hashes": dict(MANIFEST), + "lium_declared_digest": DIGEST, + "constation_gap_budget_seconds": GAP_BUDGET, + "constation_observed_max_gap_seconds": 5.0, + } + fields.update(overrides) + return ConstationBundle(**fields) + + +@dataclass(frozen=True, slots=True) +class _Injected: + """Injectable checker outcomes for a single constation_ok call.""" + + allowlist: CheckOutcome = CheckOutcome(ok=True, reason="ok") + nonce: CheckOutcome = CheckOutcome(ok=True, reason="ok") + signature: CheckOutcome = CheckOutcome(ok=True, reason="ok") + + +def _run( + bundle: ConstationBundle, + inj: _Injected | None = None, +) -> ConstationResult: + inj = inj or _Injected() + + def check_allowlist( + *, + digest: str, + commit_sha: str, + tree_sha: str, + variant: str, + ) -> CheckOutcome: + del digest, commit_sha, tree_sha, variant + return inj.allowlist + + def check_nonce( + *, + nonce: str, + work_unit_id: str, + miner_hotkey: str, + pod_id: str, + ) -> CheckOutcome: + del nonce, work_unit_id, miner_hotkey, pod_id + return inj.nonce + + def verify_signature(signed: object) -> CheckOutcome: + del signed + return inj.signature + + return constation_ok( + bundle, + check_allowlist=check_allowlist, + check_nonce=check_nonce, + verify_signature=verify_signature, + ) + + +def test_complete_valid_bundle_returns_true() -> None: + """S1 happy: all six mechanisms pass → ok=True reason=ok.""" + result = _run(_bundle()) + + assert result.ok is True + assert result.reason is ConstationFailReason.OK + assert bool(result) is True + + +def test_allowlist_miss_unknown_digest() -> None: + result = _run( + _bundle(), + _Injected(allowlist=_miss("unknown_digest")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST + assert result.detail == "unknown_digest" + + +def test_allowlist_miss_variant_mismatch() -> None: + result = _run( + _bundle(), + _Injected(allowlist=_miss("variant_mismatch")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.ALLOWLIST_VARIANT_MISMATCH + + +def test_allowlist_miss_commit_mismatch() -> None: + result = _run( + _bundle(), + _Injected(allowlist=_miss("commit_mismatch")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.ALLOWLIST_COMMIT_MISMATCH + + +def test_allowlist_miss_revoked() -> None: + result = _run( + _bundle(), + _Injected(allowlist=_miss("revoked")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.ALLOWLIST_REVOKED + + +def test_nonce_already_consumed() -> None: + result = _run( + _bundle(), + _Injected(nonce=_miss("already_consumed")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.NONCE_ALREADY_CONSUMED + + +def test_nonce_unknown() -> None: + result = _run( + _bundle(), + _Injected(nonce=_miss("unknown_nonce")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.NONCE_UNKNOWN + + +def test_nonce_expired() -> None: + result = _run( + _bundle(), + _Injected(nonce=_miss("expired")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.NONCE_EXPIRED + + +def test_signature_mismatch() -> None: + result = _run( + _bundle(), + _Injected(signature=_miss("signature_mismatch")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.SIGNATURE_INVALID + assert result.detail == "signature_mismatch" + + +def test_sealed_manifest_mismatch() -> None: + bad_manifest = {**MANIFEST, "src/prism_recipe/harness.py": "f" * 64} + result = _run( + _bundle(reported_sealed_manifest_hashes=bad_manifest), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.SEALED_MANIFEST_MISMATCH + + +def test_corroboration_mismatch_fails() -> None: + """Negative-only: Lium-declared digest disagrees with sidecar digest.""" + result = _run(_bundle(lium_declared_digest=DIGEST_OTHER)) + + assert result.ok is False + assert result.reason is ConstationFailReason.CORROBORATION_MISMATCH + + +def test_corroboration_agreement_alone_insufficient() -> None: + """Agreement contributes nothing: allowlist miss still fails.""" + result = _run( + _bundle(lium_declared_digest=DIGEST), + _Injected(allowlist=_miss("unknown_digest")), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST + + +def test_constation_gap_beyond_budget() -> None: + result = _run( + _bundle( + constation_gap_budget_seconds=10.0, + constation_observed_max_gap_seconds=10.0001, + ), + ) + + assert result.ok is False + assert result.reason is ConstationFailReason.CONSTATION_GAP + + +def test_gap_within_budget_ok() -> None: + result = _run( + _bundle( + constation_gap_budget_seconds=10.0, + constation_observed_max_gap_seconds=10.0, + ), + ) + + assert result.ok is True + assert result.reason is ConstationFailReason.OK + + +@pytest.mark.parametrize( + ("label", "bundle_kw", "inj", "expected"), + [ + ( + "allowlist", + {}, + _Injected(allowlist=_miss("unknown_digest")), + ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST, + ), + ( + "nonce", + {}, + _Injected(nonce=_miss("already_consumed")), + ConstationFailReason.NONCE_ALREADY_CONSUMED, + ), + ( + "signature", + {}, + _Injected(signature=_miss("signature_mismatch")), + ConstationFailReason.SIGNATURE_INVALID, + ), + ( + "sealed_manifest", + { + "reported_sealed_manifest_hashes": { + **MANIFEST, + "src/prism_recipe/harness.py": "0" * 64, + } + }, + _Injected(), + ConstationFailReason.SEALED_MANIFEST_MISMATCH, + ), + ( + "corroboration", + {"lium_declared_digest": DIGEST_OTHER}, + _Injected(), + ConstationFailReason.CORROBORATION_MISMATCH, + ), + ( + "constation_gap", + { + "constation_gap_budget_seconds": 1.0, + "constation_observed_max_gap_seconds": 2.0, + }, + _Injected(), + ConstationFailReason.CONSTATION_GAP, + ), + ], + ids=[ + "allowlist", + "nonce", + "signature", + "sealed_manifest", + "corroboration", + "constation_gap", + ], +) +def test_each_single_mechanism_omission_fails_with_distinct_reason( + label: str, + bundle_kw: dict[str, Any], + inj: _Injected, + expected: ConstationFailReason, +) -> None: + """Parameterized: each single-field/mechanism break → False + distinct reason.""" + del label + result = _run(_bundle(**bundle_kw), inj) + + assert result.ok is False + assert result.reason is expected + assert result.reason is not ConstationFailReason.OK + + +def test_omission_reasons_are_pairwise_distinct() -> None: + """The six mechanism failure reasons used by the param table must all differ.""" + reasons = [ + ConstationFailReason.ALLOWLIST_UNKNOWN_DIGEST, + ConstationFailReason.NONCE_ALREADY_CONSUMED, + ConstationFailReason.SIGNATURE_INVALID, + ConstationFailReason.SEALED_MANIFEST_MISMATCH, + ConstationFailReason.CORROBORATION_MISMATCH, + ConstationFailReason.CONSTATION_GAP, + ] + assert len(reasons) == len(set(reasons)) + + +def test_module_docstring_states_sole_elevation_predicate_m14() -> None: + import prism_challenge.constation as mod + + doc = (mod.__doc__ or "") + (constation_ok.__doc__ or "") + lowered = doc.lower() + assert "sole" in lowered or "only" in lowered + assert "tier" in lowered + assert "m14" in lowered or "no other" in lowered + + +def test_module_exposes_no_tier_grant_api() -> None: + import prism_challenge.constation as mod + + forbidden = { + "effective_tier", + "grant_tier", + "elevate_tier", + "set_tier", + "compute_tier", + } + names = {n for n in dir(mod) if not n.startswith("_")} + assert names.isdisjoint(forbidden) + + +def test_checkers_receive_bundle_fields() -> None: + """Injected checkers are called with the bundle's identity fields.""" + seen: dict[str, Any] = {} + + def check_allowlist( + *, + digest: str, + commit_sha: str, + tree_sha: str, + variant: str, + ) -> CheckOutcome: + seen["allowlist"] = (digest, commit_sha, tree_sha, variant) + return _ok() + + def check_nonce( + *, + nonce: str, + work_unit_id: str, + miner_hotkey: str, + pod_id: str, + ) -> CheckOutcome: + seen["nonce"] = (nonce, work_unit_id, miner_hotkey, pod_id) + return _ok() + + def verify_signature(signed: object) -> CheckOutcome: + seen["sig"] = signed + return _ok() + + bundle = _bundle() + result = constation_ok( + bundle, + check_allowlist=check_allowlist, + check_nonce=check_nonce, + verify_signature=verify_signature, + ) + + assert result.ok is True + assert seen["allowlist"] == (DIGEST, COMMIT, TREE, VARIANT) + assert seen["nonce"] == (NONCE, WORK_UNIT, HOTKEY, POD) + assert seen["sig"] == bundle.signed_attestation + + +def test_missing_lium_corroboration_is_not_contradiction() -> None: + """Negative-only: absent Lium channel does not fail (not a positive grant).""" + result = _run(_bundle(lium_declared_digest=None)) + + assert result.ok is True + assert result.reason is ConstationFailReason.OK + + +def test_result_is_structured_and_frozen() -> None: + result = _run(_bundle(), _Injected(allowlist=_miss("revoked"))) + assert isinstance(result, ConstationResult) + assert result.ok is False + with pytest.raises((AttributeError, TypeError, FrozenInstanceError)): + result.ok = True # type: ignore[misc] diff --git a/packages/challenges/prism/tests/test_constation_scoring_gate.py b/packages/challenges/prism/tests/test_constation_scoring_gate.py new file mode 100644 index 000000000..1236f4c70 --- /dev/null +++ b/packages/challenges/prism/tests/test_constation_scoring_gate.py @@ -0,0 +1,657 @@ +"""Wave 4 todos 20–23: attestation_mode, effective_tier, fail-closed, break-glass.""" + +from __future__ import annotations + +import base64 +import io +import math +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.audit import effective_tier +from prism_challenge.breakglass import ( + BreakGlassAuditLog, + BreakGlassRequest, + evaluate_break_glass, +) +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.constation import ( + CheckOutcome, + ConstationBundle, + ConstationFailReason, + constation_ok, +) +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.ingestion import ingest_work_unit_result +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + ATTESTATION_MODE_V1, + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + ExecutionProof, + ProviderInfo, + WorkerSignature, + attach_attestation_mode, + attestation_mode_of, + build_execution_proof, + compute_manifest_sha256, + elevation_image_digest, + image_digest_from_env, + normalize_attestation_mode, + worker_signer_from_key, +) + +WORKER_KEY = "//WorkerConstationGate" +PINNED = "sha256:" + ("aa" * 32) +OTHER = "sha256:" + ("bb" * 32) +DIGEST = "sha256:" + ("11" * 32) + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _code_bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + ) + + +def _manifest(marker: str = "v2") -> dict[str, Any]: + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "prequential_bpb": 1.23, + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _constation_bundle(*, digest: str = DIGEST) -> ConstationBundle: + man = {"src/prism_recipe/harness.py": "a" * 64} + return ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="wu", + miner_hotkey="hk", + pod_id="pod-1", + nonce="nonce-1", + signed_attestation={"sig": "ok"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + +def _ok_checkers(): + def allow(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def nonce(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def sig(_s: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + return allow, nonce, sig + + +def _fail_manifest_checkers(): + def allow(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def nonce(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def sig(_s: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + return allow, nonce, sig + + +def _tier1_proof(signer, unit_id: str, manifest: dict[str, Any], *, image_digest: str): + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof( + signer=signer, + manifest_sha256=digest, + unit_id=unit_id, + image_digest=image_digest, + constation_digest=image_digest, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + tier=1, # type: ignore[arg-type] + ) + return proof.model_dump(mode="json") + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any]) -> dict[str, Any]: + return { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + MANIFEST_PAYLOAD_KEY: manifest, + } + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_code_bundle(), filename="project.zip") + ) + return sub.id + + +def _final_score(db_path: Path, submission_id: str) -> float | None: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return None if row is None else float(row[0]) + + +# --- todo 20 ------------------------------------------------------------------------------------ + + +def test_attestation_mode_is_miner_rent_image_pin_evidence_v1() -> None: + att = attach_attestation_mode(None) + assert att["attestation_mode"] == ATTESTATION_MODE_V1 + assert normalize_attestation_mode(ATTESTATION_MODE_V1) == ATTESTATION_MODE_V1 + with pytest.raises(ValueError, match="forbidden"): + normalize_attestation_mode("lium_attested") + with pytest.raises(ValueError, match="forbidden"): + normalize_attestation_mode("tee_attested") + + +def test_env_digest_is_telemetry_not_elevation() -> None: + assert elevation_image_digest(constation_digest=DIGEST, env_digest=OTHER) == DIGEST + assert elevation_image_digest(constation_digest=None, env_digest=OTHER) is None + # image_digest_from_env still reads env but docstring marks telemetry-only + assert image_digest_from_env({"PRISM_IMAGE_DIGEST": OTHER}) == OTHER + + +def test_build_proof_always_sets_attestation_mode() -> None: + signer = worker_signer_from_key(WORKER_KEY) + proof = build_execution_proof( + signer=signer, + manifest_sha256="c" * 64, + unit_id="u1", + image_digest=PINNED, + provider=ProviderInfo(name="lium", pod_id="p"), + tier=1, # type: ignore[arg-type] + ) + assert attestation_mode_of(proof) == ATTESTATION_MODE_V1 + + +def test_selfreport_digest_match_without_constation_is_tier0() -> None: + """Correct PRISM_IMAGE_DIGEST alone cannot reach tier 1 (todo 20 failure path).""" + proof = ExecutionProof( + version=1, + tier=1, + manifest_sha256="c" * 64, + image_digest=PINNED, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + attestation=attach_attestation_mode(None), + ) + assert effective_tier(proof, pinned_image_digest=PINNED) == 0 + assert effective_tier(proof, pinned_image_digest=PINNED, constation_ok_result=False) == 0 + assert effective_tier(proof, pinned_image_digest=PINNED, constation_ok_result=True) == 1 + + +# --- todo 21 ------------------------------------------------------------------------------------ + + +def test_tier1_only_when_constation_ok_true() -> None: + proof = ExecutionProof( + version=1, + tier=1, + manifest_sha256="c" * 64, + image_digest=PINNED, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + ) + assert effective_tier(proof, constation_ok_result=True) == 1 + assert effective_tier(proof, constation_ok_result=False) == 0 + assert effective_tier(proof, constation_ok_result=None) == 0 + + +def test_claimed_tier2_always_zero_even_with_constation() -> None: + proof = ExecutionProof( + version=1, + tier=2, + manifest_sha256="c" * 64, + image_digest=PINNED, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + attestation={"tdx_quote_b64": "x", "gpu_eat_jwt": "y"}, + ) + assert effective_tier(proof, constation_ok_result=True) == 0 + + +def test_constation_result_object_drives_tier() -> None: + proof = ExecutionProof( + version=1, + tier=1, + manifest_sha256="c" * 64, + image_digest=DIGEST, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + ) + allow, nonce, sig = _ok_checkers() + ok = constation_ok( + _constation_bundle(), + check_allowlist=allow, + check_nonce=nonce, + verify_signature=sig, + ) + assert ok.ok is True + assert effective_tier(proof, constation_ok_result=ok) == 1 + + bad_bundle = _constation_bundle() + # force sealed mismatch + from dataclasses import replace + + bad = replace( + bad_bundle, + reported_sealed_manifest_hashes={"src/prism_recipe/harness.py": "f" * 64}, + ) + fail = constation_ok(bad, check_allowlist=allow, check_nonce=nonce, verify_signature=sig) + assert fail.ok is False + assert fail.reason == ConstationFailReason.SEALED_MANIFEST_MISMATCH + assert effective_tier(proof, constation_ok_result=fail) == 0 + + +# --- todo 22 ------------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_valid_bundle_writes_score_with_attestation_mode( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app) + manifest = _manifest("ok") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=DIGEST) + allow, nonce, sig = _ok_checkers() + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + pinned_image_digest=DIGEST, + constation_bundle=_constation_bundle(digest=DIGEST), + check_allowlist=allow, + check_nonce=nonce, + verify_constation_signature=sig, + ) + assert outcome.status == "accepted" + assert outcome.finalized is True + assert outcome.score_written is True + assert outcome.effective_tier == 1 + assert outcome.attestation_mode == ATTESTATION_MODE_V1 + score = _final_score(db_path, submission_id) + assert score is not None and score > 0.0 + + +@pytest.mark.asyncio +async def test_no_bundle_writes_no_score_miner_fault( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app) + manifest = _manifest("nobundle") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=PINNED) + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + pinned_image_digest=PINNED, + constation_bundle=None, + ) + assert outcome.status == "rejected" + assert outcome.finalized is False + assert outcome.score_written is False + assert outcome.reason == "miner_fault:missing_constation_bundle" + assert _final_score(db_path, submission_id) is None + + +@pytest.mark.asyncio +async def test_manifest_mismatch_miner_fault_no_score( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app, hotkey="hk-mm") + manifest = _manifest("mm") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=DIGEST) + allow, nonce, sig = _ok_checkers() + from dataclasses import replace + + bad = replace( + _constation_bundle(digest=DIGEST), + reported_sealed_manifest_hashes={"src/prism_recipe/harness.py": "f" * 64}, + ) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-mm", + result=_result(proof, manifest), + constation_bundle=bad, + check_allowlist=allow, + check_nonce=nonce, + verify_constation_signature=sig, + ) + assert outcome.status == "rejected" + assert outcome.score_written is False + assert outcome.reason == "miner_fault:manifest_mismatch" + assert _final_score(db_path, submission_id) is None + + +@pytest.mark.asyncio +async def test_infra_fault_constation_unavailable_no_score( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app, hotkey="hk-infra") + manifest = _manifest("infra") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=DIGEST) + + # Exhaust retries → no score + from prism_challenge.ingestion import ResultIngestionError + + with pytest.raises(ResultIngestionError) as ei: + await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-infra", + result=_result(proof, manifest), + constation_infra_fault="constation_unavailable", + constation_attempt=1, + max_constation_attempts=3, + ) + assert "infra_fault" in ei.value.reason + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-infra", + result=_result(proof, manifest), + constation_infra_fault="constation_unavailable", + constation_attempt=3, + max_constation_attempts=3, + ) + assert outcome.status == "rejected" + assert outcome.score_written is False + assert outcome.reason is not None and outcome.reason.startswith("infra_fault:") + assert _final_score(db_path, submission_id) is None + + +@pytest.mark.asyncio +async def test_revoked_digest_no_score(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app, hotkey="hk-rev") + manifest = _manifest("rev") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=DIGEST) + + def allow(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=False, reason="revoked") + + def nonce(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def sig(_s: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-rev", + result=_result(proof, manifest), + constation_bundle=_constation_bundle(digest=DIGEST), + check_allowlist=allow, + check_nonce=nonce, + verify_constation_signature=sig, + ) + assert outcome.status == "rejected" + assert outcome.reason == "miner_fault:revoked_digest" + assert _final_score(db_path, submission_id) is None + + +# --- todo 23 ------------------------------------------------------------------------------------ + + +def test_breakglass_admits_infra_fault_only() -> None: + log = BreakGlassAuditLog() + req = BreakGlassRequest( + operator_id="ops-alice", + reason="constation outage window", + work_unit_id="wu-1", + fault_code="infra_fault:constation_unavailable", + ) + ok = evaluate_break_glass(req, fault_reason="infra_fault:constation_unavailable", audit_log=log) + assert ok.admitted is True + assert log.entries and log.entries[0]["admitted"] is True + assert log.entries[0]["operator_id"] == "ops-alice" + + log2 = BreakGlassAuditLog() + bad = evaluate_break_glass(req, fault_reason="miner_fault:replayed_nonce", audit_log=log2) + assert bad.admitted is False + assert bad.reason == "breakglass_refused_miner_fault" + assert log2.entries and log2.entries[0]["admitted"] is False + + +@pytest.mark.asyncio +async def test_breakglass_admits_infra_run_and_writes_score( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app, hotkey="hk-bg") + manifest = _manifest("bg") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=DIGEST) + log = BreakGlassAuditLog() + bg = BreakGlassRequest( + operator_id="ops-bob", + reason="confirmed BASE outage", + work_unit_id=submission_id, + fault_code="infra_fault:constation_unavailable", + ) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-bg", + result=_result(proof, manifest), + constation_infra_fault="constation_unavailable", + constation_attempt=3, + max_constation_attempts=3, + break_glass=bg, + break_glass_audit_log=log, + ) + assert outcome.status == "accepted" + assert outcome.break_glass_admitted is True + assert outcome.effective_tier == 0 # no elevation without real constation + assert outcome.score_written is True + assert _final_score(db_path, submission_id) is not None + assert any(e.get("admitted") for e in log.entries) + + +@pytest.mark.asyncio +async def test_breakglass_refuses_miner_fault( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + submission_id = await _seed(app, hotkey="hk-bgm") + manifest = _manifest("bgm") + proof = _tier1_proof(signer, submission_id, manifest, image_digest=DIGEST) + log = BreakGlassAuditLog() + bg = BreakGlassRequest( + operator_id="ops-eve", + reason="please admit anyway", + work_unit_id=submission_id, + fault_code="miner_fault:replayed_nonce", + ) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-bgm", + result=_result(proof, manifest), + # missing bundle = miner_fault + constation_bundle=None, + break_glass=bg, + break_glass_audit_log=log, + ) + assert outcome.status == "rejected" + assert outcome.break_glass_admitted is False + assert outcome.score_written is False + assert _final_score(db_path, submission_id) is None + assert any(e.get("detail") == "miner_fault_override_refused" for e in log.entries) diff --git a/packages/challenges/prism/tests/test_execution_backend_constation_gate.py b/packages/challenges/prism/tests/test_execution_backend_constation_gate.py new file mode 100644 index 000000000..4b2cc1072 --- /dev/null +++ b/packages/challenges/prism/tests/test_execution_backend_constation_gate.py @@ -0,0 +1,66 @@ +"""Lium execution backend is gated on a full constation bundle (todo 19). + +Renamed from the misleading test_lium_client.py (which tested no client). +""" + +from __future__ import annotations + +import pytest + +from prism_challenge.constation import ConstationBundle +from prism_challenge.queue import ( + LIUM_EXECUTION_BACKEND, + SUPPORTED_EXECUTION_BACKENDS, + is_execution_backend_supported, + require_execution_backend, +) + + +def _bundle() -> ConstationBundle: + return ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest="sha256:" + ("1" * 64), + work_unit_id="wu-1", + miner_hotkey="hk", + pod_id="pod-1", + nonce="n-1", + signed_attestation={"sig": "x"}, + expected_sealed_manifest_hashes={"h.py": "c" * 64}, + reported_sealed_manifest_hashes={"h.py": "c" * 64}, + lium_declared_digest="sha256:" + ("1" * 64), + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + +def test_base_gpu_always_supported_without_bundle() -> None: + assert "base_gpu" in SUPPORTED_EXECUTION_BACKENDS + assert is_execution_backend_supported("base_gpu") is True + assert is_execution_backend_supported("base_gpu", constation_bundle=None) is True + require_execution_backend("base_gpu") # no raise + + +def test_lium_without_bundle_rejected() -> None: + assert is_execution_backend_supported(LIUM_EXECUTION_BACKEND) is False + assert is_execution_backend_supported(LIUM_EXECUTION_BACKEND, constation_bundle=None) is False + with pytest.raises(ValueError, match="constation bundle required for lium"): + require_execution_backend(LIUM_EXECUTION_BACKEND) + with pytest.raises(ValueError, match="constation bundle required for lium"): + require_execution_backend("lium", constation_bundle=None) + + +def test_lium_with_full_bundle_accepted() -> None: + bundle = _bundle() + assert is_execution_backend_supported("lium", constation_bundle=bundle) is True + require_execution_backend("lium", constation_bundle=bundle) # no raise + + +def test_remote_provider_and_local_cpu_still_rejected() -> None: + assert "remote_provider" not in SUPPORTED_EXECUTION_BACKENDS + assert "local_cpu" not in SUPPORTED_EXECUTION_BACKENDS + with pytest.raises(ValueError, match="Unsupported execution backend"): + require_execution_backend("remote_provider") + with pytest.raises(ValueError, match="Unsupported execution backend"): + require_execution_backend("local_cpu", constation_bundle=_bundle()) diff --git a/packages/challenges/prism/tests/test_lium_client.py b/packages/challenges/prism/tests/test_lium_client.py deleted file mode 100644 index ec6af124f..000000000 --- a/packages/challenges/prism/tests/test_lium_client.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -from prism_challenge.queue import SUPPORTED_EXECUTION_BACKENDS - - -def test_lium_backends_are_not_supported() -> None: - assert "remote_provider" not in SUPPORTED_EXECUTION_BACKENDS - assert "local_cpu" not in SUPPORTED_EXECUTION_BACKENDS - assert "base_gpu" in SUPPORTED_EXECUTION_BACKENDS diff --git a/packages/challenges/prism/tests/test_official_comparison_scoring.py b/packages/challenges/prism/tests/test_official_comparison_scoring.py index f0ba40941..b68e795fe 100644 --- a/packages/challenges/prism/tests/test_official_comparison_scoring.py +++ b/packages/challenges/prism/tests/test_official_comparison_scoring.py @@ -153,9 +153,7 @@ def test_official_primary_heldout_overrides_worse_train_bpb() -> None: assert result.winner == "a" assert result.reason == "primary_heldout" # Emission path is now Official-like too (VAL-RESLAB-006): held-out primary wins scalar rank. - score_generalizer = score_prequential_bpb( - _challenge_manifest(bpb=1.80, heldout_delta=1.20) - ) + score_generalizer = score_prequential_bpb(_challenge_manifest(bpb=1.80, heldout_delta=1.20)) score_compressor = score_prequential_bpb(_challenge_manifest(bpb=1.00, heldout_delta=0.10)) assert score_generalizer.final_score > score_compressor.final_score diff --git a/packages/challenges/prism/tests/test_prism_attestation_routes_s8.py b/packages/challenges/prism/tests/test_prism_attestation_routes_s8.py new file mode 100644 index 000000000..ae271197d --- /dev/null +++ b/packages/challenges/prism/tests/test_prism_attestation_routes_s8.py @@ -0,0 +1,122 @@ +"""S8: Prism public attestation challenge/answer roundtrip (proxy product path).""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig + +WORKER_KEY = "//WorkerAttestS8" +DIGEST = "sha256:" + ("11" * 32) +COMMIT = "a" * 40 +TREE = "b" * 40 + + +def _settings(tmp_path: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 's8.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + # No constation_base_url → in-process SoT on prism app.state + ) + + +def test_s8_challenge_answer_roundtrip_on_prism(tmp_path: Path) -> None: + settings = _settings(tmp_path) + with TestClient(create_app(settings)) as client: + # Public surface exactly as published through BASE proxy under + # /challenges/prism/v1/attestation/* (challenge app owns the path). + ch = client.get( + "/v1/attestation/challenge", + params={ + "phase": "start", + "work_unit_id": "wu-s8", + "miner_hotkey": "hk-s8", + "pod_id": "pod-s8", + }, + ) + assert ch.status_code == 200, ch.text + body = ch.json() + assert body["nonce"] + assert body["phase"] == "start" + assert body["work_unit_id"] == "wu-s8" + assert body["challenge_id"] == body["nonce"] + + ans = client.post( + "/v1/attestation/answer", + json={"nonce": body["nonce"], "phase": "start"}, + ) + assert ans.status_code == 200, ans.text + assert ans.json()["status"] == "accepted" + + +def test_s8_inprocess_register_check_and_nonce_consume(tmp_path: Path) -> None: + settings = _settings(tmp_path) + headers = {"Authorization": "Bearer secret"} + with TestClient(create_app(settings)) as client: + reg = client.post( + "/internal/v1/constation/register_digest", + headers=headers, + json={ + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": "cuda", + "digest": DIGEST, + }, + ) + assert reg.status_code == 200, reg.text + + hit = client.post( + "/internal/v1/constation/check_allowlist", + headers=headers, + json={ + "digest": DIGEST, + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": "cuda", + }, + ) + assert hit.status_code == 200 + assert hit.json() == {"ok": True, "reason": "ok"} + + ch = client.get( + "/v1/attestation/challenge", + params={ + "phase": "interval", + "work_unit_id": "wu-s8b", + "miner_hotkey": "hk-s8b", + "pod_id": "pod-s8b", + }, + ) + nonce = ch.json()["nonce"] + body = { + "nonce": nonce, + "work_unit_id": "wu-s8b", + "miner_hotkey": "hk-s8b", + "pod_id": "pod-s8b", + } + first = client.post("/internal/v1/constation/check_nonce", headers=headers, json=body) + second = client.post("/internal/v1/constation/check_nonce", headers=headers, json=body) + assert first.json() == {"ok": True, "reason": "ok"} + assert second.json()["ok"] is False + assert second.json()["reason"] == "already_consumed" + + +def test_s8_missing_binding_query_422(tmp_path: Path) -> None: + settings = _settings(tmp_path) + with TestClient(create_app(settings)) as client: + r = client.get("/v1/attestation/challenge", params={"phase": "start"}) + assert r.status_code == 422 diff --git a/packages/challenges/prism/tests/test_prism_audit_effective_tier.py b/packages/challenges/prism/tests/test_prism_audit_effective_tier.py index 7a8067333..b5cce646c 100644 --- a/packages/challenges/prism/tests/test_prism_audit_effective_tier.py +++ b/packages/challenges/prism/tests/test_prism_audit_effective_tier.py @@ -66,19 +66,33 @@ def test_tier2_claim_downgrades_without_verified_attestation() -> None: assert is_tier_downgraded(with_digest, pinned_image_digest=PINNED) is True -def test_tier1_claim_requires_matching_pinned_digest() -> None: +def test_tier1_claim_requires_constation_ok() -> None: matching = _proof(tier=1, image_digest=PINNED) mismatched = _proof(tier=1, image_digest=OTHER) - # Schema forbids tier-1 without image_digest; uncovered digest=empty via mismatched. - emptyish = _proof(tier=1, image_digest=OTHER) - # Tier 1 also requires provider pod binding (workload identity). - assert effective_tier(matching, pinned_image_digest=PINNED) == 1 - assert is_tier_downgraded(matching, pinned_image_digest=PINNED) is False + # Pin match alone never elevates (todo 21 / M14). + assert effective_tier(matching, pinned_image_digest=PINNED) == 0 assert effective_tier(mismatched, pinned_image_digest=PINNED) == 0 - assert effective_tier(emptyish, pinned_image_digest=PINNED) == 0 - # With no pinned digest configured, no tier-1 claim is verifiable. - assert effective_tier(matching, pinned_image_digest=None) == 0 + # constation_ok is the sole elevation predicate. + assert ( + effective_tier( + matching, + pinned_image_digest=PINNED, + constation_ok_result=True, + ) + == 1 + ) + assert ( + is_tier_downgraded( + matching, + pinned_image_digest=PINNED, + constation_ok_result=True, + ) + is False + ) + assert effective_tier(matching, constation_ok_result=False) == 0 + # Digest mismatch is irrelevant when constation_ok is True (digest already gated upstream). + assert effective_tier(mismatched, constation_ok_result=True) == 1 def test_tier0_claim_stays_tier0() -> None: @@ -89,10 +103,19 @@ def test_tier0_claim_stays_tier0() -> None: # --- Sampling follows the EFFECTIVE tier (VAL-PRISM-019 statistical) ----------------------------- -def _sampled_fraction(sampler: AuditSampler, proof: ExecutionProof, n: int) -> float: +def _sampled_fraction( + sampler: AuditSampler, + proof: ExecutionProof, + n: int, + *, + constation_ok_result: bool | None = None, +) -> float: hits = sum( sampler.decide( - work_unit_id=f"{proof.tier}-{i}", proof=proof, pinned_image_digest=PINNED + work_unit_id=f"{proof.tier}-{i}", + proof=proof, + pinned_image_digest=PINNED, + constation_ok_result=constation_ok_result, ).sampled for i in range(n) ) @@ -117,8 +140,14 @@ def _bound(p: float) -> float: # Unverified tier-2 claims are sampled at tier-0 rate (fail-closed TEE). assert abs(_sampled_fraction(sampler, opaque_t2, n) - 0.10) < _bound(0.10) - # Honest tier-1 claims are sampled at their tier-1 rate. - assert abs(_sampled_fraction(sampler, honest_t1, n) - 0.05) < _bound(0.05) + # Honest tier-1 + constation_ok are sampled at their tier-1 rate. + assert abs(_sampled_fraction(sampler, honest_t1, n, constation_ok_result=True) - 0.05) < _bound( + 0.05 + ) + # Without constation_ok, tier-1 claims are effective 0. + assert abs( + _sampled_fraction(sampler, honest_t1, n, constation_ok_result=False) - 0.10 + ) < _bound(0.10) # Unverifiable claims are sampled at the EFFECTIVE (tier-0) rate, NOT the claimed rate. assert abs(_sampled_fraction(sampler, fake_t2, n) - 0.10) < _bound(0.10) assert abs(_sampled_fraction(sampler, fake_t1, n) - 0.10) < _bound(0.10) diff --git a/packages/challenges/prism/tests/test_prism_no_tee_absence.py b/packages/challenges/prism/tests/test_prism_no_tee_absence.py index c31793107..36c89b048 100644 --- a/packages/challenges/prism/tests/test_prism_no_tee_absence.py +++ b/packages/challenges/prism/tests/test_prism_no_tee_absence.py @@ -1,4 +1,8 @@ -"""Prism NO TEE residual: package absence + score finalize without tee (VAL-NOTEE-001..008).""" +"""Prism NO TEE residual: package absence + score finalize without tee (VAL-NOTEE-001..008). + +Extended by todo 24: image-attestation path, attestation_mode never TEE, tier cannot exceed 1. +Never delete, skip, or xfail this file. +""" from __future__ import annotations @@ -16,22 +20,28 @@ from prism_challenge.app import create_app from prism_challenge.audit import effective_tier from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.constation import CheckOutcome, ConstationBundle from prism_challenge.ingestion import ResultIngestionError, ingest_work_unit_result from prism_challenge.models import SubmissionCreate from prism_challenge.proof import ( + ATTESTATION_MODE_V1, MANIFEST_PAYLOAD_KEY, PROOF_PAYLOAD_KEY, ExecutionProof, ProviderInfo, WorkerSignature, + attach_attestation_mode, + attestation_mode_of, build_execution_proof, compute_manifest_sha256, + normalize_attestation_mode, worker_signer_from_key, ) WORKER_KEY = "//WorkerNoTee" PINNED = "sha256:" + ("ab" * 32) OTHER = "sha256:" + ("cd" * 32) +DIGEST = "sha256:" + ("11" * 32) TINY_ARCH = """ import torch @@ -94,8 +104,6 @@ def test_config_has_no_tee_block_or_capability() -> None: docker_backend="cli", database_url="sqlite+aiosqlite:////tmp/prism-notee-cfg.sqlite3", ) - # Nested TeeConfig / settings.tee / PRISM_TEE gone; capability not advertised. - # Base ChallengeSettings may still expose an inert tee_verification_enabled flag. assert not hasattr(settings, "tee") assert "challenge.tee_verification" not in settings.capabilities config_mod = __import__("prism_challenge.config", fromlist=["*"]) @@ -127,12 +135,99 @@ def test_max_effective_tier_is_one_never_two() -> None: provider=ProviderInfo(name="lium", pod_id="pod-1"), worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), ) - assert effective_tier(proof_t2, pinned_image_digest=PINNED) == 0 - assert effective_tier(proof_t1, pinned_image_digest=PINNED) == 1 - assert effective_tier(proof_t1, pinned_image_digest=OTHER) == 0 + # Claimed tier 2 never elevates, even with constation_ok. + assert effective_tier(proof_t2, pinned_image_digest=PINNED, constation_ok_result=True) == 0 + # Tier 1 requires constation_ok (not pin match alone). + assert effective_tier(proof_t1, pinned_image_digest=PINNED) == 0 + assert effective_tier(proof_t1, pinned_image_digest=PINNED, constation_ok_result=True) == 1 + assert effective_tier(proof_t1, pinned_image_digest=OTHER, constation_ok_result=False) == 0 + + +def test_attestation_mode_never_implies_tee() -> None: + """Todo 24: attestation_mode is miner_rent_image_pin_evidence_v1; TEE labels forbidden.""" + assert ATTESTATION_MODE_V1 == "miner_rent_image_pin_evidence_v1" + assert "tee" not in ATTESTATION_MODE_V1.lower() + assert "tdx" not in ATTESTATION_MODE_V1.lower() + assert normalize_attestation_mode(ATTESTATION_MODE_V1) == ATTESTATION_MODE_V1 + for bad in ("lium_attested", "tee", "tee_attested", "tdx", "sev", "cvm"): + with pytest.raises(ValueError): + normalize_attestation_mode(bad) + att = attach_attestation_mode({"tdx_quote_b64": "x", "gpu_eat_jwt": "y"}) + assert att["attestation_mode"] == ATTESTATION_MODE_V1 + + +def test_image_attestation_path_exists_and_functions() -> None: + """Todo 24: image-attestation path (constation_ok + attestation_mode) is present.""" + from prism_challenge.constation import constation_ok + from prism_challenge.queue import LIUM_EXECUTION_BACKEND, is_execution_backend_supported + + man = {"h.py": "a" * 64} + bundle = ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=DIGEST, + work_unit_id="wu", + miner_hotkey="hk", + pod_id="pod-1", + nonce="n", + signed_attestation={"s": "1"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=DIGEST, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + def ok(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + result = constation_ok( + bundle, check_allowlist=ok, check_nonce=ok, verify_signature=lambda _s: ok() + ) + assert result.ok is True + assert is_execution_backend_supported(LIUM_EXECUTION_BACKEND, constation_bundle=bundle) + signer = worker_signer_from_key(WORKER_KEY) + proof = build_execution_proof( + signer=signer, + manifest_sha256="c" * 64, + unit_id="u", + image_digest=DIGEST, + constation_digest=DIGEST, + provider=ProviderInfo(name="lium", pod_id="pod-1"), + tier=1, # type: ignore[arg-type] + ) + assert attestation_mode_of(proof) == ATTESTATION_MODE_V1 + assert effective_tier(proof, constation_ok_result=result) == 1 + + +def test_effective_tier_cannot_exceed_one_by_any_route() -> None: + """Todo 24: no route yields effective tier > 1.""" + for claimed in (0, 1, 2): + proof = ExecutionProof( + version=1, + tier=claimed, # type: ignore[arg-type] + manifest_sha256="c" * 64, + image_digest=PINNED if claimed >= 1 else None, + provider=ProviderInfo(name="lium", pod_id="pod-1") if claimed >= 1 else None, + worker_signature=WorkerSignature(worker_pubkey="wk", sig="0xab"), + attestation={ + "version": 1, + "provider": "local_fixture", + "evidence_type": "prism.tee.v1", + "tdx_quote_b64": "Q", + "gpu_eat_jwt": "J", + "attestation_mode": ATTESTATION_MODE_V1, + } + if claimed == 2 + else attach_attestation_mode(None), + ) + for cok in (True, False, None): + tier = effective_tier(proof, pinned_image_digest=PINNED, constation_ok_result=cok) + assert tier <= 1, f"claimed={claimed} cok={cok} -> {tier}" -def _bundle() -> str: +def _code_bundle() -> str: stream = io.BytesIO() with zipfile.ZipFile(stream, "w") as archive: archive.writestr("architecture.py", TINY_ARCH) @@ -203,6 +298,7 @@ def _proof_payload( unit_id=unit_id, provider=ProviderInfo(name="lium", pod_id="pod-1"), image_digest=image_digest, + constation_digest=image_digest, attestation=attestation, tier=tier, # type: ignore[arg-type] ) @@ -219,6 +315,39 @@ def _result(proof_dict: dict[str, Any], manifest: dict[str, Any]) -> dict[str, A } +def _ok_checkers(): + def allow(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def nonce(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def sig(_s: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + return allow, nonce, sig + + +def _constation_bundle(digest: str = DIGEST) -> ConstationBundle: + man = {"h.py": "a" * 64} + return ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="wu", + miner_hotkey="hk", + pod_id="pod-1", + nonce="n", + signed_attestation={"s": "1"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + async def _make_app(settings: PrismSettings): app = create_app(settings) await app.state.database.init() @@ -227,7 +356,7 @@ async def _make_app(settings: PrismSettings): async def _seed(app, hotkey: str = "hk-notee") -> str: sub = await app.state.repository.create_submission( - hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + hotkey, SubmissionCreate(code=_code_bundle(), filename="project.zip") ) return sub.id @@ -245,19 +374,24 @@ def _score(db_path: Path, submission_id: str): @pytest.mark.asyncio async def test_score_finalize_works_without_tee_package(tmp_path: Path) -> None: - """Worker-plane finalize succeeds with no tee package / no tee_required (VAL-NOTEE-003).""" + """Worker-plane finalize succeeds with no tee package; requires constation (P1).""" settings = _settings(tmp_path) app = await _make_app(settings) signer = worker_signer_from_key(WORKER_KEY) submission_id = await _seed(app) manifest = _manifest() - proof = _proof_payload(signer, submission_id, manifest, tier=1, image_digest=PINNED) + proof = _proof_payload(signer, submission_id, manifest, tier=1, image_digest=DIGEST) + allow, nonce, sig = _ok_checkers() outcome = await ingest_work_unit_result( worker=app.state.worker, work_unit_id=submission_id, submission_ref="hk-notee", result=_result(proof, manifest), - pinned_image_digest=PINNED, + pinned_image_digest=DIGEST, + constation_bundle=_constation_bundle(DIGEST), + check_allowlist=allow, + check_nonce=nonce, + verify_constation_signature=sig, ) assert outcome.status == "accepted" assert outcome.finalized is True @@ -265,11 +399,13 @@ async def test_score_finalize_works_without_tee_package(tmp_path: Path) -> None: assert outcome.claimed_tier == 1 assert outcome.tier_downgraded is False assert outcome.reason is None + assert outcome.attestation_mode == ATTESTATION_MODE_V1 assert _score(tmp_path / "notee.sqlite3", submission_id) is not None @pytest.mark.asyncio -async def test_pin_mismatch_downgrades_but_still_finalizes(tmp_path: Path) -> None: +async def test_pin_mismatch_without_constation_writes_no_score(tmp_path: Path) -> None: + """NEW contract (todo 22): missing constation ⇒ no score (was: still finalizes).""" settings = _settings(tmp_path) app = await _make_app(settings) signer = worker_signer_from_key(WORKER_KEY) @@ -282,18 +418,18 @@ async def test_pin_mismatch_downgrades_but_still_finalizes(tmp_path: Path) -> No submission_ref="hk-mismatch", result=_result(proof, manifest), pinned_image_digest=PINNED, + constation_bundle=None, ) - assert outcome.status == "accepted" - assert outcome.finalized is True - assert outcome.claimed_tier == 1 + assert outcome.status == "rejected" + assert outcome.finalized is False + assert outcome.score_written is False assert outcome.effective_tier == 0 - assert outcome.tier_downgraded is True - assert _score(tmp_path / "notee.sqlite3", submission_id) is not None + assert _score(tmp_path / "notee.sqlite3", submission_id) is None @pytest.mark.asyncio async def test_ingestion_never_raises_tee_required(tmp_path: Path) -> None: - """Attestation-claiming tier-2 proof finalizes without tee_required (max effective=0).""" + """Attestation-claiming tier-2 proof never raises tee_required (max effective=0).""" settings = _settings(tmp_path) app = await _make_app(settings) signer = worker_signer_from_key(WORKER_KEY) @@ -311,21 +447,26 @@ async def test_ingestion_never_raises_tee_required(tmp_path: Path) -> None: submission_id, manifest, tier=2, - image_digest=PINNED, + image_digest=DIGEST, attestation=attestation, ) + allow, nonce, sig = _ok_checkers() try: outcome = await ingest_work_unit_result( worker=app.state.worker, work_unit_id=submission_id, submission_ref="hk-t2", result=_result(proof, manifest), - pinned_image_digest=PINNED, + pinned_image_digest=DIGEST, + constation_bundle=_constation_bundle(DIGEST), + check_allowlist=allow, + check_nonce=nonce, + verify_constation_signature=sig, ) except ResultIngestionError as exc: assert exc.reason != "tee_required" raise - assert outcome.finalized is True + # With constation, score may write but effective tier stays 0 for claimed tier 2. assert outcome.effective_tier == 0 assert outcome.tier_downgraded is True - assert _score(tmp_path / "notee.sqlite3", submission_id) is not None + assert outcome.attestation_mode == ATTESTATION_MODE_V1 diff --git a/packages/challenges/prism/tests/test_prism_result_ingestion.py b/packages/challenges/prism/tests/test_prism_result_ingestion.py index c2b1b6f3e..ddf27a825 100644 --- a/packages/challenges/prism/tests/test_prism_result_ingestion.py +++ b/packages/challenges/prism/tests/test_prism_result_ingestion.py @@ -440,6 +440,7 @@ def _eval_job_count(db_path: Path, submission_id: str) -> int: async def test_ingestion_records_downgraded_effective_tier(tmp_path, monkeypatch) -> None: + """Claimed tier 2 never elevates; with valid constation, score writes at effective 0.""" data_dir = _stage_train(tmp_path) monkeypatch.setattr( "prism_challenge.evaluator.container.DockerExecutor.run", @@ -454,24 +455,31 @@ async def test_ingestion_records_downgraded_effective_tier(tmp_path, monkeypatch submission_id = await _seed(app) manifest = _manifest() - # A tier-1 claim whose image_digest does not match the pinned digest is - # unverifiable under SDK-strict proof shape -> effective tier 0. + # Claimed tier 2 (structured attestation shape) collapses to effective 0 even when + # constation_ok is True (auto-injected by conftest for legacy callers). proof = _proof_dict( signer, submission_id, manifest, - tier=1, + tier=2, image_digest="sha256:" + ("ab" * 32), - attestation=None, + attestation={ + "version": 1, + "provider": "local_fixture", + "evidence_type": "prism.tee.v1", + "tdx_quote_b64": "QUOTE", + "gpu_eat_jwt": "JWT", + }, ) outcome = await ingest_work_unit_result( worker=app.state.worker, work_unit_id=submission_id, submission_ref="hk-owner", result=_result(proof, manifest), - pinned_image_digest="sha256:" + ("cd" * 32), + pinned_image_digest="sha256:" + ("ab" * 32), ) - assert outcome.claimed_tier == 1 + assert outcome.status == "accepted" + assert outcome.claimed_tier == 2 assert outcome.effective_tier == 0 assert outcome.tier_downgraded is True @@ -484,7 +492,7 @@ async def test_ingestion_records_downgraded_effective_tier(tmp_path, monkeypatch ).fetchone() finally: conn.close() - assert row == (1, 0, 1) + assert row == (2, 0, 1) # --- HTTP route body contract + status codes (VAL-PRISM-017/018) --------------------------------- diff --git a/packages/challenges/prism/tests/test_prism_scoring_characterization_baseline.py b/packages/challenges/prism/tests/test_prism_scoring_characterization_baseline.py new file mode 100644 index 000000000..4261b4e6b --- /dev/null +++ b/packages/challenges/prism/tests/test_prism_scoring_characterization_baseline.py @@ -0,0 +1,295 @@ +"""Post-fail-closed scoring contract (todo 18 baseline rewritten by todo 22). + +Todo 18 pinned PRE-change reality (pin mismatch still scored). Todo 22 deliberately +broke that contract (P1: no valid constation bundle ⇒ no score row). This file now +documents the NEW contract — tests are rewritten, not deleted. +""" + +from __future__ import annotations + +import base64 +import io +import math +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from prism_challenge.app import create_app +from prism_challenge.audit import effective_tier +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.constation import CheckOutcome, ConstationBundle +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.ingestion import ingest_work_unit_result +from prism_challenge.models import SubmissionCreate +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + ProviderInfo, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) + +WORKER_KEY = "//WorkerCharBaseline" +PINNED = "sha256:" + ("aa" * 32) +OTHER = "sha256:" + ("bb" * 32) +DIGEST = "sha256:" + ("11" * 32) + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _settings(tmp_path: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=True, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + ) + + +def _manifest(marker: str = "v2") -> dict[str, Any]: + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "prequential_bpb": 1.23, + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _tier1_proof_dict(signer, unit_id: str, manifest: dict[str, Any], *, image_digest: str): + digest = compute_manifest_sha256(manifest) + proof = build_execution_proof( + signer=signer, + manifest_sha256=digest, + unit_id=unit_id, + image_digest=image_digest, + constation_digest=image_digest, + provider=ProviderInfo(name="lium", pod_id="pod-char-1"), + tier=1, # type: ignore[arg-type] + ) + return proof.model_dump(mode="json") + + +def _result(proof_dict: dict[str, Any], manifest: dict[str, Any]) -> dict[str, Any]: + return { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof_dict, + MANIFEST_PAYLOAD_KEY: manifest, + } + + +def _constation_bundle(digest: str = DIGEST) -> ConstationBundle: + man = {"h.py": "a" * 64} + return ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="wu", + miner_hotkey="hk", + pod_id="pod-char-1", + nonce="n", + signed_attestation={"s": "1"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + + +def _ok_checkers(): + def allow(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def nonce(**_k: Any) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def sig(_s: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + return allow, nonce, sig + + +async def _make_app(settings: PrismSettings): + app = create_app(settings) + await app.state.database.init() + return app + + +async def _seed(app, hotkey: str = "hk-owner") -> str: + sub = await app.state.repository.create_submission( + hotkey, SubmissionCreate(code=_bundle(), filename="project.zip") + ) + return sub.id + + +def _final_score(db_path: Path, submission_id: str) -> float | None: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return None if row is None else float(row[0]) + + +async def test_pin_mismatch_without_constation_writes_no_final_score( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """NEW (todo 22): IMAGE_PIN mismatch / missing constation ⇒ no score row (P1).""" + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + + submission_id = await _seed(app) + manifest = _manifest("pin-mismatch") + proof = _tier1_proof_dict(signer, submission_id, manifest, image_digest=OTHER) + + from prism_challenge.proof import ExecutionProof + + assert effective_tier(ExecutionProof.model_validate(proof), pinned_image_digest=PINNED) == 0 + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-owner", + result=_result(proof, manifest), + pinned_image_digest=PINNED, + constation_bundle=None, + ) + + assert outcome.status == "rejected" + assert outcome.finalized is False + assert outcome.score_written is False + assert outcome.reason == "miner_fault:missing_constation_bundle" + assert _final_score(db_path, submission_id) is None + + +async def test_constation_ok_writes_score_and_sets_effective_tier( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """NEW: valid constation bundle ⇒ score written; effective_tier follows constation_ok.""" + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + app = await _make_app(_settings(tmp_path)) + signer = worker_signer_from_key(WORKER_KEY) + db_path = tmp_path / "coord.sqlite3" + allow, nonce, sig = _ok_checkers() + + submission_id = await _seed(app, hotkey="hk-ok") + manifest = _manifest("constation-ok") + proof = _tier1_proof_dict(signer, submission_id, manifest, image_digest=DIGEST) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref="hk-ok", + result=_result(proof, manifest), + pinned_image_digest=DIGEST, + constation_bundle=_constation_bundle(DIGEST), + check_allowlist=allow, + check_nonce=nonce, + verify_constation_signature=sig, + ) + assert outcome.status == "accepted" + assert outcome.effective_tier == 1 + assert outcome.finalized is True + score = _final_score(db_path, submission_id) + assert score is not None and score > 0.0 diff --git a/packages/challenges/prism/tests/test_prism_scoring_leaderboard_determinism.py b/packages/challenges/prism/tests/test_prism_scoring_leaderboard_determinism.py index 901c3d347..7b4d38c11 100644 --- a/packages/challenges/prism/tests/test_prism_scoring_leaderboard_determinism.py +++ b/packages/challenges/prism/tests/test_prism_scoring_leaderboard_determinism.py @@ -68,12 +68,8 @@ def test_scoring_leaderboard_orders_by_emission_rank_even_against_commit_time() # score committed earlier: the primary axis is emission final_score, not commit time. from prism_challenge.evaluator.scoring import heldout_to_primary_score - worse_early = _row( - "worse-heldout", heldout_to_primary_score(0.1), "2024-01-01T00:00:00+00:00" - ) - better_late = _row( - "better-heldout", heldout_to_primary_score(1.0), "2024-01-02T00:00:00+00:00" - ) + worse_early = _row("worse-heldout", heldout_to_primary_score(0.1), "2024-01-01T00:00:00+00:00") + better_late = _row("better-heldout", heldout_to_primary_score(1.0), "2024-01-02T00:00:00+00:00") ranked = rank_leaderboard([worse_early, better_late]) assert [r.submission_id for r in ranked] == ["better-heldout", "worse-heldout"] diff --git a/packages/challenges/prism/tests/test_prod_constation_http_ingest.py b/packages/challenges/prism/tests/test_prod_constation_http_ingest.py new file mode 100644 index 000000000..fc4d6337e --- /dev/null +++ b/packages/challenges/prism/tests/test_prod_constation_http_ingest.py @@ -0,0 +1,287 @@ +"""Production HTTP constation path (S1/S3) — no legacy auto-inject (module name gate).""" + +from __future__ import annotations + +import io +import sqlite3 +import zipfile +from pathlib import Path +from typing import Any + +import httpx +import pytest +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.config import PrismSettings, WorkerPlaneConfig +from prism_challenge.constation import ConstationBundle, constation_bundle_to_dict +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run +from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) + +WORKER_KEY = "//WorkerProdConstation" +DIGEST = "sha256:" + ("11" * 32) + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' +) + + +def _stage_train(root: Path) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD.format(i=i) for i in range(64)), encoding="utf-8" + ) + return data_dir + + +def _zip_b64() -> bytes: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return stream.getvalue() + + +def _settings(tmp_path: Path, **extra: Any) -> PrismSettings: + kw: dict[str, Any] = dict( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'coord.sqlite3'}", + shared_token="secret", + allow_insecure_signatures=False, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp_path / "artifacts", + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + constation_base_url="http://base-constation.test", + constation_internal_token="constation-tok", + ) + kw.update(extra) + return PrismSettings(**kw) + + +def _manifest() -> dict[str, Any]: + return { + "schema_version": "prism_run_manifest.v2", + "metrics": { + "token_accuracy": 0.5, + "loss": 1.0, + "step": 1, + }, + "timing": {"wall_seconds": 1.0}, + } + + +def _bundle_for(submission_id: str) -> dict[str, Any]: + man = {"route-test-harness.py": "a" * 64} + return constation_bundle_to_dict( + ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=DIGEST, + work_unit_id=submission_id, + miner_hotkey="hk-owner", + pod_id="pod-1", + nonce="nonce-prod-1", + signed_attestation={"sig": "fixture"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=DIGEST, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + ) + + +def _ok_transport() -> httpx.MockTransport: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"ok": True, "reason": "ok"}) + + return httpx.MockTransport(handler) + + +@pytest.fixture(autouse=True) +def _patch_http_checkers(monkeypatch: pytest.MonkeyPatch) -> None: + """Route BaseHttpConstationClient to always-ok MockTransport (S1 checkers).""" + from prism_challenge import constation_checkers as mod + + orig = mod.BaseHttpConstationClient.__init__ + + def _init(self, *args, **kwargs): # type: ignore[no-untyped-def] + kwargs = dict(kwargs) + kwargs.setdefault("transport", _ok_transport()) + orig(self, *args, **kwargs) + + monkeypatch.setattr(mod.BaseHttpConstationClient, "__init__", _init) + + +def test_s3_missing_bundle_fail_closed_via_http(tmp_path: Path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings(tmp_path) + headers = {"Authorization": "Bearer secret"} + signer = worker_signer_from_key(WORKER_KEY) + + with TestClient(create_app(settings)) as client: + seed = client.post( + "/internal/v1/bridge/submissions", + content=_zip_b64(), + headers={ + "Authorization": "Bearer secret", + "X-Base-Verified-Hotkey": "hk-owner", + "X-Submission-Filename": "project.zip", + "Content-Type": "application/octet-stream", + }, + ) + assert seed.status_code == 200, seed.text + sid = seed.json()["id"] + manifest = _manifest() + proof = build_execution_proof( + signer=signer, + manifest_sha256=compute_manifest_sha256(manifest), + unit_id=sid, + image_digest=DIGEST, + constation_digest=DIGEST, + ).model_dump(mode="json") + body = { + "api_version": "1.0", + "work_unit_id": sid, + "assignment_id": sid, + "submission_ref": "hk-owner", + "challenge_slug": settings.slug, + "result": { + "executed": 1, + PROOF_PAYLOAD_KEY: proof, + MANIFEST_PAYLOAD_KEY: manifest, + }, + "proof": proof, + } + resp = client.post("/internal/v1/work_units/result", json=body, headers=headers) + assert resp.status_code == 422, resp.text + detail = resp.json()["detail"] + code = detail.get("code") if isinstance(detail, dict) else None + assert code in { + "miner_fault:missing_constation_bundle", + "constation_rejected", + } or (isinstance(detail, dict) and "missing_constation" in str(detail.get("code", ""))), ( + detail + ) + + db_path = tmp_path / "coord.sqlite3" + conn = sqlite3.connect(db_path) + try: + score = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (sid,) + ).fetchone() + finally: + conn.close() + assert score is None + + +def test_s1_honest_bundle_scores_via_http(tmp_path: Path, monkeypatch) -> None: + data_dir = _stage_train(tmp_path) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + settings = _settings(tmp_path) + headers = {"Authorization": "Bearer secret"} + signer = worker_signer_from_key(WORKER_KEY) + + with TestClient(create_app(settings)) as client: + seed = client.post( + "/internal/v1/bridge/submissions", + content=_zip_b64(), + headers={ + "Authorization": "Bearer secret", + "X-Base-Verified-Hotkey": "hk-owner", + "X-Submission-Filename": "project.zip", + "Content-Type": "application/octet-stream", + }, + ) + assert seed.status_code == 200, seed.text + sid = seed.json()["id"] + manifest = _manifest() + proof = build_execution_proof( + signer=signer, + manifest_sha256=compute_manifest_sha256(manifest), + unit_id=sid, + image_digest=DIGEST, + constation_digest=DIGEST, + tier=1, # type: ignore[arg-type] + ).model_dump(mode="json") + body = { + "api_version": "1.0", + "work_unit_id": sid, + "assignment_id": sid, + "submission_ref": "hk-owner", + "challenge_slug": settings.slug, + "result": { + "executed": 1, + PROOF_PAYLOAD_KEY: proof, + MANIFEST_PAYLOAD_KEY: manifest, + "constation_bundle": _bundle_for(sid), + }, + "proof": proof, + } + resp = client.post("/internal/v1/work_units/result", json=body, headers=headers) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data["status"] == "accepted", data + assert data.get("score_written") is True, data + assert data.get("effective_tier") == 1, data + assert data.get("attestation_mode"), data diff --git a/packages/challenges/prism/tests/test_prod_constation_kwargs.py b/packages/challenges/prism/tests/test_prod_constation_kwargs.py new file mode 100644 index 000000000..0361e6f4a --- /dev/null +++ b/packages/challenges/prism/tests/test_prod_constation_kwargs.py @@ -0,0 +1,58 @@ +"""Production constation kwargs: deserialize bundle + attach checkers.""" + +from __future__ import annotations + +from prism_challenge.app import _constation_ingest_kwargs +from prism_challenge.config import PrismSettings +from prism_challenge.constation import ConstationBundle, constation_bundle_to_dict + + +def _bundle_dict() -> dict: + man = {"h.py": "a" * 64} + digest = "sha256:" + ("1" * 64) + b = ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=digest, + work_unit_id="wu-1", + miner_hotkey="hk", + pod_id="pod", + nonce="n", + signed_attestation={"sig": "x"}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=digest, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + return constation_bundle_to_dict(b) + + +def test_prod_missing_bundle_returns_empty() -> None: + settings = PrismSettings( + allow_insecure_signatures=False, + constation_base_url="http://base.test", + constation_internal_token="tok", + ) + assert _constation_ingest_kwargs(settings, {"executed": 1}) == {} + + +def test_prod_with_bundle_and_checkers() -> None: + settings = PrismSettings( + allow_insecure_signatures=False, + constation_base_url="http://base.test", + constation_internal_token="tok", + ) + kwargs = _constation_ingest_kwargs(settings, {"constation_bundle": _bundle_dict()}) + assert "constation_bundle" in kwargs + assert kwargs["check_allowlist"] is not None + assert kwargs["check_nonce"] is not None + assert kwargs["verify_constation_signature"] is not None + + +def test_insecure_seam_still_injects_without_bundle() -> None: + settings = PrismSettings(allow_insecure_signatures=True) + kwargs = _constation_ingest_kwargs(settings, {}) + # Test seam injects synthetic bundle + assert "constation_bundle" in kwargs diff --git a/packages/challenges/prism/tests/test_scale_fullscale_readiness.py b/packages/challenges/prism/tests/test_scale_fullscale_readiness.py index a6e7997d4..4161966c0 100644 --- a/packages/challenges/prism/tests/test_scale_fullscale_readiness.py +++ b/packages/challenges/prism/tests/test_scale_fullscale_readiness.py @@ -114,9 +114,7 @@ def test_probe_default_settings_paths_honest_when_absent() -> None: assert d["emission_changed"] is False if d["status"] == "BLOCKED": assert d["reasons"] - assert "BLOCKED_with_reason" in d["status_label"] or d["status_label"].startswith( - "BLOCKED" - ) + assert "BLOCKED_with_reason" in d["status_label"] or d["status_label"].startswith("BLOCKED") # Never invent scores / never claim full 100BT train ran. assert d.get("full_scale_train_executed") is False assert "invented_metrics" not in d or d.get("invented_metrics") is False diff --git a/scripts/e2e_lium_attestation.py b/scripts/e2e_lium_attestation.py new file mode 100644 index 000000000..43c62c9cc --- /dev/null +++ b/scripts/e2e_lium_attestation.py @@ -0,0 +1,692 @@ +#!/usr/bin/env python3 +"""End-to-end Lium image-attestation path (plan checkbox 27). + +Wires the REAL product modules: + + submission identity → DigestAllowlist → ConstationRunner/poller + → prism ``constation_ok`` → fail-closed ``ingest_work_unit_result`` + → tier / ``attestation_mode`` / score-row assertions + +Modes +----- +* ``--mode honest`` — matching digests across start/interval/end; expect + ``constation_ok`` True, score row written, ``effective_tier == 1``, + ``attestation_mode == miner_rent_image_pin_evidence_v1``. +* ``--mode adversarial`` — mid-run sidecar digest swap (image swap TOCTOU); + expect runner/constation fail, **no** score row, ``miner_fault:*`` reason. + +Live vs offline +--------------- +* Live requires ``LIUM_API_KEY`` (and optional ``BASE_LIVE_PROVIDER_TESTS=1``). + When the key is absent the script runs **offline fixture mode** that still + exercises the real runner, allowlist, nonce service, ``constation_ok``, and + prism ingestion — not toy stubs that skip those modules. +* Offline always prints ``REAL_LIUM=false`` and never claims live success. + +Evidence header (every run stdout starts with):: + + REAL_LIUM=true|false + REASON=... + MODE=honest|adversarial + RESULT=PASS|FAIL + +Exit code 0 on PASS, 1 on FAIL. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import io +import math +import os +import sqlite3 +import sys +import tempfile +import zipfile +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path +from typing import Any, Literal + +# --------------------------------------------------------------------------- +# Path bootstrap so `uv run python scripts/e2e_lium_attestation.py` works +# --------------------------------------------------------------------------- +_SCRIPT_DIR = Path(__file__).resolve().parent +_BASE_ROOT = _SCRIPT_DIR.parent +if str(_BASE_ROOT / "src") not in sys.path: + sys.path.insert(0, str(_BASE_ROOT / "src")) +_PRISM_SRC = _BASE_ROOT / "packages" / "challenges" / "prism" / "src" +if _PRISM_SRC.is_dir() and str(_PRISM_SRC) not in sys.path: + sys.path.insert(0, str(_PRISM_SRC)) + +from prism_challenge.app import create_app # noqa: E402 +from prism_challenge.config import PrismSettings, WorkerPlaneConfig # noqa: E402 +from prism_challenge.constation import ( # noqa: E402 + ConstationBundle, + adapt_allowlist_lookup, + adapt_nonce_consume, + constation_ok, +) +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run # noqa: E402 +from prism_challenge.ingestion import ingest_work_unit_result # noqa: E402 +from prism_challenge.models import SubmissionCreate # noqa: E402 +from prism_challenge.proof import ( # noqa: E402 + ATTESTATION_MODE_V1, + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + ProviderInfo, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) + +from base.compute.attestation_nonce import ( # noqa: E402 + AttestationNonceService, + NonceBinding, +) +from base.compute.constation_custody import ( # noqa: E402 + LiumKeyCustody, + generate_custody_master_key, +) +from base.compute.constation_poller import PollerConfig # noqa: E402 +from base.compute.constation_runner import ( # noqa: E402 + ConstationRunner, + ConstationRunRequest, +) +from base.compute.constation_types import ( # noqa: E402 + ConstationFailCode, + FaultClass, +) +from base.compute.digest_allowlist import ( # noqa: E402 + DigestAllowlist, + DigestRecord, + ImageVariant, +) +from base.compute.lium import LiumPodRead # noqa: E402 + +Mode = Literal["honest", "adversarial"] + +DIGEST_HONEST = "sha256:" + ("11" * 32) +DIGEST_SWAPPED = "sha256:" + ("22" * 32) +COMMIT_SHA = "a" * 40 +TREE_SHA = "b" * 40 +VARIANT = ImageVariant.CUDA +HOTKEY = "5E2EMinerHotkeyLiumAttestation000000000001" +POD_ID = "pod-e2e-constation-001" +WORKER_KEY = "//WorkerE2ELiumAttestation" +FIXTURE_API_KEY = "lium-fixture-key-NEVER-LOG-OR-CLAIM-AS-LIVE" + +SEALED_MANIFEST = {"src/prism_recipe/harness.py": "c" * 64} + +TINY_ARCH = """ +import torch +from torch import nn + + +class TinyLM(nn.Module): + def __init__(self, vocab): + super().__init__() + self.emb = nn.Embedding(vocab, 8) + self.head = nn.Linear(8, vocab) + + def forward(self, tokens): + return self.head(self.emb(tokens)) + + +def build_model(ctx): + return TinyLM(ctx.vocab_size) +""" + +TINY_TRAIN = """ +import torch +import torch.nn.functional as F + + +def train(ctx): + model = ctx.build_model() + opt = torch.optim.AdamW(model.parameters(), lr=0.01) + for batch in ctx.iter_train_batches(model, batch_size=1): + opt.zero_grad() + logits = model(batch.tokens) + nv = logits.shape[-1] + loss = F.cross_entropy( + logits[:, :-1, :].reshape(-1, nv), batch.tokens[:, 1:].reshape(-1) % nv + ) + loss.backward() + opt.step() +""" + +_SHARD_LINE = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample number {i} ' + 'has enough bytes to cover several challenge instrument batches deterministically"}}\n' # noqa: E501 +) + + +# --------------------------------------------------------------------------- +# Offline Lium / sidecar fixtures (drive REAL ConstationRunner) +# --------------------------------------------------------------------------- + + +@dataclass +class FakeClock: + t: float = 0.0 + + def now(self) -> float: + return self.t + + async def sleep(self, seconds: float) -> None: + self.t += max(0.0, seconds) + + +@dataclass +class SequenceRng: + values: list[float] + i: int = 0 + + def __call__(self) -> float: + if not self.values: + return 0.0 + v = self.values[self.i % len(self.values)] + self.i += 1 + return v + + +@dataclass +class ScriptedLium: + """Minimal LiumClient stand-in — same surface ConstationRunner polls.""" + + digests: list[str | None] = field(default_factory=lambda: [DIGEST_HONEST]) + calls: int = 0 + + async def get_pod_raw(self, pod_id: str) -> LiumPodRead: + self.calls += 1 + idx = min(self.calls - 1, len(self.digests) - 1) + digest = self.digests[idx] + return LiumPodRead( + pod_id=pod_id, + template_id="tmpl-e2e-1", + docker_image_digest=digest, + raw={"id": pod_id, "template": {"docker_image_digest": digest}}, + ) + + async def balance(self) -> float: + return 1.0 + + +@dataclass +class PhaseSidecar: + """Sidecar attestor; adversarial mode swaps digest after start phase.""" + + honest_digest: str = DIGEST_HONEST + swapped_digest: str = DIGEST_SWAPPED + adversarial: bool = False + calls: list[str] = field(default_factory=list) + + async def attest(self, *, pod_id: str, phase: str) -> str: + del pod_id + self.calls.append(phase) + if self.adversarial and phase != "start": + return self.swapped_digest + return self.honest_digest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _code_bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", TINY_ARCH) + archive.writestr("training.py", TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _manifest(marker: str) -> dict[str, Any]: + covered_bytes = 4096 + online_loss = [10.0, 6.0, 3.0, 2.0] + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered_bytes, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered_bytes, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "prequential_bpb": 1.23, + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _stage_train(root: Path, *, lines: int = 64) -> Path: + data_dir = root / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD_LINE.format(i=i) for i in range(lines)), encoding="utf-8" + ) + return data_dir + + +def _final_score(db_path: Path, submission_id: str) -> float | None: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", (submission_id,) + ).fetchone() + finally: + conn.close() + return None if row is None else float(row[0]) + + +def _settings(tmp: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp / 'coord.sqlite3'}", + shared_token="e2e-secret", + allow_insecure_signatures=True, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp / "artifacts", + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + ) + + +def _emit_header( + *, + real_lium: bool, + reason: str, + mode: Mode, + result: str, +) -> None: + print(f"REAL_LIUM={'true' if real_lium else 'false'}") + print(f"REASON={reason}") + print(f"MODE={mode}") + print(f"RESULT={result}") + + +def _live_available() -> bool: + return bool(os.environ.get("LIUM_API_KEY", "").strip()) + + +# --------------------------------------------------------------------------- +# Core path: allowlist → runner → constation_ok → ingest +# --------------------------------------------------------------------------- + + +async def _run_constation_runner(*, adversarial: bool) -> Any: + """Drive real ConstationRunner with fixture Lium + sidecar.""" + scripted = ScriptedLium(digests=[DIGEST_HONEST]) + + def factory(key: str) -> Any: + del key + return scripted + + async def _probe(client: Any) -> None: + del client + await scripted.balance() + + custody = LiumKeyCustody( + master_key=generate_custody_master_key(), + client_factory=factory, + ) + custody.probe_fn = _probe + verdict = await custody.register(miner_hotkey=HOTKEY, api_key=FIXTURE_API_KEY) + if not verdict.ok: + raise RuntimeError(f"fixture custody register failed: {verdict.reason}") + + clock = FakeClock() + runner = ConstationRunner( + custody=custody, + sidecar=PhaseSidecar(adversarial=adversarial), + poller_config=PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + ), + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + return await runner.run( + ConstationRunRequest( + miner_hotkey=HOTKEY, + work_unit_id="wu-placeholder", # overwritten after submission seed + pod_id=POD_ID, + duration_seconds=10.0 if not adversarial else 10.0, + ) + ) + + +async def run_offline(mode: Mode) -> dict[str, Any]: + """Full offline path through real product modules. Returns assertion bag.""" + adversarial = mode == "adversarial" + tmp = Path(tempfile.mkdtemp(prefix=f"e2e-lium-{mode}-")) + data_dir = _stage_train(tmp) + + # --- 1. Allowlist: BASE-produced digest registration (submission identity) --- + allowlist = DigestAllowlist() + allowlist.register( + DigestRecord( + commit_sha=COMMIT_SHA, + tree_sha=TREE_SHA, + variant=VARIANT, + digest=DIGEST_HONEST, + ) + ) + + # --- 2. Continuous constation runner/poller (fixture Lium, real runner) --- + # First pass uses placeholder work_unit; we re-bind nonce to real submission id. + run_record = await _run_constation_runner(adversarial=adversarial) + + # --- 3. Prism app + submission seed --- + # Patch DockerExecutor before create_app workers touch it. + import prism_challenge.evaluator.container as container_mod + + container_mod.DockerExecutor.run = cpu_reexec_run(train_data_dir=data_dir) # type: ignore[method-assign, assignment] + + settings = _settings(tmp) + app = create_app(settings) + await app.state.database.init() + submission = await app.state.repository.create_submission( + HOTKEY, SubmissionCreate(code=_code_bundle(), filename="project.zip") + ) + submission_id = submission.id + db_path = tmp / "coord.sqlite3" + + # --- 4. Nonce service (real single-use consume) --- + nonce_svc = AttestationNonceService(ttl=timedelta(hours=1)) + issued = nonce_svc.issue( + NonceBinding(work_unit_id=submission_id, miner_hotkey=HOTKEY, pod_id=POD_ID) + ) + + def check_allowlist(**kwargs: Any) -> Any: + return adapt_allowlist_lookup(allowlist.lookup(**kwargs)) + + def check_nonce(**kwargs: Any) -> Any: + return adapt_nonce_consume( + nonce_svc.consume( + kwargs["nonce"], + NonceBinding( + work_unit_id=kwargs["work_unit_id"], + miner_hotkey=kwargs["miner_hotkey"], + pod_id=kwargs["pod_id"], + ), + ) + ) + + def verify_signature(_signed: object) -> Any: + from prism_challenge.constation import CheckOutcome + + return CheckOutcome(ok=True, reason="ok") + + # Bundle digest: honest uses runner sidecar; adversarial may have mismatch. + sidecar_digest = run_record.sidecar_digest or DIGEST_HONEST + lium_declared = run_record.lium_declared_digest + # For adversarial mid-run swap the runner fails closed before a clean + # record; still build a bundle that reflects the swap for constation_ok / + # ingest fail-closed (corroboration or allowlist miss). + if adversarial: + bundle_digest = DIGEST_SWAPPED + lium_for_bundle: str | None = DIGEST_HONEST # Lium still declares original + gap_obs = run_record.constation_observed_max_gap_seconds + gap_budget = run_record.constation_gap_budget_seconds + else: + bundle_digest = sidecar_digest + lium_for_bundle = lium_declared + gap_obs = run_record.constation_observed_max_gap_seconds + gap_budget = run_record.constation_gap_budget_seconds + + bundle = ConstationBundle( + commit_sha=COMMIT_SHA, + tree_sha=TREE_SHA, + variant=VARIANT.value, + digest=bundle_digest, + work_unit_id=submission_id, + miner_hotkey=HOTKEY, + pod_id=POD_ID, + nonce=issued.nonce, + signed_attestation={"sig": "fixture-ok"}, + expected_sealed_manifest_hashes=dict(SEALED_MANIFEST), + reported_sealed_manifest_hashes=dict(SEALED_MANIFEST), + lium_declared_digest=lium_for_bundle, + constation_gap_budget_seconds=gap_budget, + constation_observed_max_gap_seconds=gap_obs, + ) + + # --- 5. constation_ok (sole elevation predicate) --- + constation_result = constation_ok( + bundle, + check_allowlist=check_allowlist, + check_nonce=check_nonce, + verify_signature=verify_signature, + ) + + # --- 6. Execution proof + fail-closed ingest --- + signer = worker_signer_from_key(WORKER_KEY) + manifest = _manifest(mode) + proof = build_execution_proof( + signer=signer, + manifest_sha256=compute_manifest_sha256(manifest), + unit_id=submission_id, + image_digest=DIGEST_HONEST, + constation_digest=DIGEST_HONEST if not adversarial else DIGEST_SWAPPED, + provider=ProviderInfo(name="lium", pod_id=POD_ID), + tier=1, # type: ignore[arg-type] + ) + result_payload = { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof.model_dump(mode="json"), + MANIFEST_PAYLOAD_KEY: manifest, + } + + # Re-issue nonce for ingest (constation_ok already consumed the first). + issued2 = nonce_svc.issue( + NonceBinding(work_unit_id=submission_id, miner_hotkey=HOTKEY, pod_id=POD_ID) + ) + bundle_for_ingest = ConstationBundle( + commit_sha=bundle.commit_sha, + tree_sha=bundle.tree_sha, + variant=bundle.variant, + digest=bundle.digest, + work_unit_id=bundle.work_unit_id, + miner_hotkey=bundle.miner_hotkey, + pod_id=bundle.pod_id, + nonce=issued2.nonce, + signed_attestation=bundle.signed_attestation, + expected_sealed_manifest_hashes=dict(bundle.expected_sealed_manifest_hashes), + reported_sealed_manifest_hashes=dict(bundle.reported_sealed_manifest_hashes), + lium_declared_digest=bundle.lium_declared_digest, + constation_gap_budget_seconds=bundle.constation_gap_budget_seconds, + constation_observed_max_gap_seconds=bundle.constation_observed_max_gap_seconds, + ) + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref=HOTKEY, + result=result_payload, + pinned_image_digest=DIGEST_HONEST, + constation_bundle=bundle_for_ingest, + check_allowlist=check_allowlist, + check_nonce=check_nonce, + verify_constation_signature=verify_signature, + ) + score = _final_score(db_path, submission_id) + + return { + "mode": mode, + "run_record_ok": bool(run_record.ok), + "run_record_reason": str(run_record.reason.value), + "run_record_fault_class": ( + None + if run_record.fault_class is None + else str(run_record.fault_class.value) + ), + "run_sample_count": len(run_record.samples), + "constation_ok": bool(constation_result.ok), + "constation_reason": str(constation_result.reason.value), + "ingest_status": outcome.status, + "ingest_reason": outcome.reason, + "score_written": outcome.score_written, + "finalized": outcome.finalized, + "effective_tier": outcome.effective_tier, + "attestation_mode": outcome.attestation_mode, + "score_row": score, + "submission_id": submission_id, + "tmp": str(tmp), + } + + +def _assert_honest(bag: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if not bag["run_record_ok"]: + errors.append(f"runner expected ok, got reason={bag['run_record_reason']}") + if not bag["constation_ok"]: + errors.append(f"constation_ok expected True, reason={bag['constation_reason']}") + if bag["ingest_status"] != "accepted": + errors.append( + f"ingest status={bag['ingest_status']} reason={bag['ingest_reason']}" + ) + if not bag["score_written"]: + errors.append("score_written expected True") + if bag["score_row"] is None: + errors.append("scores.final_score row missing") + if bag["effective_tier"] != 1: + errors.append(f"effective_tier expected 1 got {bag['effective_tier']}") + if bag["attestation_mode"] != ATTESTATION_MODE_V1: + errors.append( + f"attestation_mode expected {ATTESTATION_MODE_V1!r} got {bag['attestation_mode']!r}" # noqa: E501 + ) + return errors + + +def _assert_adversarial(bag: dict[str, Any]) -> list[str]: + errors: list[str] = [] + # Runner must fail closed on mid-run swap (corroboration mismatch). + if bag["run_record_ok"]: + errors.append("runner expected fail on mid-run digest swap") + if bag["run_record_reason"] != ConstationFailCode.CORROBORATION_MISMATCH.value: + errors.append( + f"runner reason expected corroboration_mismatch got {bag['run_record_reason']}" # noqa: E501 + ) + if bag["run_record_fault_class"] != FaultClass.MINER.value: + errors.append( + f"fault_class expected miner_fault got {bag['run_record_fault_class']}" + ) + if bag["constation_ok"]: + errors.append("constation_ok expected False after digest swap") + if bag["score_written"]: + errors.append("score_written must be False") + if bag["score_row"] is not None: + errors.append(f"score row must be absent, got {bag['score_row']}") + if bag["ingest_status"] != "rejected": + errors.append(f"ingest status expected rejected got {bag['ingest_status']}") + reason = bag["ingest_reason"] or "" + if not str(reason).startswith("miner_fault:"): + errors.append(f"ingest reason expected miner_fault:* got {reason!r}") + return errors + + +async def run_live(mode: Mode) -> dict[str, Any]: + """Placeholder live path — only entered when LIUM_API_KEY is set. + + Live pod rent + mid-run image swap is intentionally not fabricated here. + When credentials exist, operators should extend this to call real LiumClient + rent/poll/delete (see scripts/live_lium_e2e.py) and feed the same + allowlist → runner → constation_ok → ingest chain. + """ + raise NotImplementedError( + "Live Lium E2E pod cycle not implemented in this environment; " + "offline fixture path is the supported proof without LIUM_API_KEY. " + f"mode={mode}" + ) + + +async def async_main(mode: Mode, *, force_offline: bool) -> int: + live = _live_available() and not force_offline + if live: + try: + bag = await run_live(mode) + real_lium = True + reason = "LIUM_API_KEY present; live path executed" + except NotImplementedError as exc: + # Fall back to offline rather than inventing live success. + print(f"# live path unavailable: {exc}", file=sys.stderr) + bag = await run_offline(mode) + real_lium = False + reason = "no live pod cycle implemented; offline fixtures used" + else: + bag = await run_offline(mode) + real_lium = False + reason = "no LIUM_API_KEY" if not _live_available() else "force_offline" + + errors = _assert_honest(bag) if mode == "honest" else _assert_adversarial(bag) + result = "PASS" if not errors else "FAIL" + _emit_header(real_lium=real_lium, reason=reason, mode=mode, result=result) + print("---") + for key in ( + "run_record_ok", + "run_record_reason", + "run_record_fault_class", + "run_sample_count", + "constation_ok", + "constation_reason", + "ingest_status", + "ingest_reason", + "score_written", + "effective_tier", + "attestation_mode", + "score_row", + "submission_id", + ): + print(f"{key}={bag.get(key)!r}") + if errors: + print("---") + print("ASSERTION_ERRORS:") + for err in errors: + print(f" - {err}") + return 1 + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + choices=("honest", "adversarial"), + required=True, + help="honest: matching digests → tier 1 + score; adversarial: mid-run swap → miner_fault", # noqa: E501 + ) + parser.add_argument( + "--offline", + action="store_true", + help="Force offline fixture mode even if LIUM_API_KEY is set", + ) + args = parser.parse_args(argv) + return asyncio.run(async_main(args.mode, force_offline=args.offline)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prism_lium_attestation_e2e.py b/scripts/prism_lium_attestation_e2e.py new file mode 100644 index 000000000..1e4edf76f --- /dev/null +++ b/scripts/prism_lium_attestation_e2e.py @@ -0,0 +1,971 @@ +#!/usr/bin/env python3 +"""Checkbox 27 — end-to-end Lium image-attestation proof (honest + adversarial). + +Path exercised (same code as production constation / prism ingest): + + submit → allowlist register → Lium deploy → sidecar attest (start/random/end) + → constation runner → prism ingest → tier / score row + +Modes +----- +* ``live`` (default when ``LIUM_API_KEY`` is available): real Lium pod rental + + real ``get_pod_raw`` reads. Sidecar is an in-process attestor that signs with + the real HMAC path; mid-run image swap is simulated by flipping the sidecar + digest after the start sample (miner-controlled guest) while Lium's declared + digest stays pinned — the detection surface is corroboration mismatch. +* ``fixture``: no network; ScriptedLium + FakeSidecar. Evidence must label + ``REAL_LIUM=false``. + +Gated: set ``BASE_LIVE_PROVIDER_TESTS=1`` for live rentals (billable). Pods are +always terminated in ``finally``. Secrets are never printed. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import io +import json +import math +import os +import sqlite3 +import sys +import time +import zipfile +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path +from typing import Any + +# Cross-repo imports (prism + prism-recipe live beside base in the monorepo). +_REPO = Path(__file__).resolve().parents[1] +_MONO = _REPO.parent +for _p in (_MONO / "prism" / "src", _MONO / "prism-recipe" / "src", _REPO / "src"): + s = str(_p) + if s not in sys.path: + sys.path.insert(0, s) + +from prism_challenge.app import create_app # noqa: E402 +from prism_challenge.config import PrismSettings, WorkerPlaneConfig # noqa: E402 +from prism_challenge.constation import CheckOutcome, ConstationBundle # noqa: E402 +from prism_challenge.evaluator.mock_reexec import cpu_reexec_run # noqa: E402 +from prism_challenge.ingestion import ingest_work_unit_result # noqa: E402 +from prism_challenge.models import SubmissionCreate # noqa: E402 +from prism_challenge.proof import ( # noqa: E402 + ATTESTATION_MODE_V1, + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + ProviderInfo, + build_execution_proof, + compute_manifest_sha256, + worker_signer_from_key, +) +from prism_recipe.attestation.payload import ( # noqa: E402 + AttestationPayload, + compute_build_secret_response, + derive_attestation_key, + sign_attestation_payload, + verify_attestation_payload, +) +from prism_recipe.submission import validate_submission # noqa: E402 + +from base.compute.attestation_nonce import ( # noqa: E402 + AttestationNonceService, + NonceBinding, + NonceConsumeHit, +) +from base.compute.constation_custody import ( # noqa: E402 + LiumKeyCustody, + generate_custody_master_key, +) +from base.compute.constation_poller import PollerConfig # noqa: E402 +from base.compute.constation_runner import ( # noqa: E402 + ConstationRunner, + ConstationRunRequest, +) +from base.compute.constation_types import ( # noqa: E402 + ConstationFailCode, + FaultClass, +) +from base.compute.digest_allowlist import ( # noqa: E402 + AllowlistHit, + DigestAllowlist, + DigestRecord, + ImageVariant, +) +from base.compute.lium import LiumClient, LiumPodRead # noqa: E402 +from base.compute.provider import InstanceSpec # noqa: E402 + +# Reuse proven credential / SSH helpers from the existing live rental script. +_SCRIPTS = Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPTS)) +import live_lium_e2e as prov # noqa: E402 + +DIGEST_HONEST = "sha256:" + ("11" * 32) +DIGEST_SWAP = "sha256:" + ("22" * 32) +# Real linux/amd64 content digest for nvidia/cuda:12.4.1-base-ubuntu22.04. +DIGEST_LIVE_CUDA = ( + "sha256:8767a245ed2c481eb245d8f6c625accc3788e1fb8612403d6b4cd4645a4f09c7" +) +COMMIT = "a" * 40 +TREE = "b" * 40 +VARIANT = "cuda" +HOTKEY = "5E2ePrismLiumAttestMinerHotkey000000000001" +WORKER_KEY = "//PrismLiumAttestE2EWorker" +BUILD_SECRET = b"prism-lium-e2e-build-secret-v1" +SEALED = {"src/prism_recipe/harness.py": "c" * 64} +TEMPLATE_PREFIX = "prism-attest-e2e-v27r" +E2E_IMAGE = "nvidia/cuda" +E2E_IMAGE_TAG = "12.4.1-base-ubuntu22.04" +E2E_STARTUP = "tail -f /dev/null" +MAX_PRICE = 1.50 +TERMINATION_HOURS = 1 +POLL_BUDGET = 12 * 60 +DEFAULT_EVIDENCE = ( + _MONO / ".omo" / "notepads" / "prism-lium-image-attestation" / "evidence" +) + +_TINY_ARCH = ( + "import torch\nfrom torch import nn\n\n" + "class TinyLM(nn.Module):\n" + " def __init__(self, vocab):\n" + " super().__init__()\n" + " self.emb = nn.Embedding(vocab, 8)\n" + " self.head = nn.Linear(8, vocab)\n" + " def forward(self, tokens):\n" + " return self.head(self.emb(tokens))\n\n" + "def build_model(ctx):\n" + " return TinyLM(ctx.vocab_size)\n" +) +_TINY_TRAIN = ( + "import torch\nimport torch.nn.functional as F\n\n" + "def train(ctx):\n" + " model = ctx.build_model()\n" + " opt = torch.optim.AdamW(model.parameters(), lr=0.01)\n" + " for batch in ctx.iter_train_batches(model, batch_size=1):\n" + " opt.zero_grad()\n" + " logits = model(batch.tokens)\n" + " nv = logits.shape[-1]\n" + " loss = F.cross_entropy(\n" + " logits[:, :-1, :].reshape(-1, nv), " + "batch.tokens[:, 1:].reshape(-1) % nv\n" + " )\n" + " loss.backward()\n" + " opt.step()\n" +) +_SHARD = ( + '{{"id": "doc-{i}", "text": "the locked fineweb edu training sample ' + "number {i} has enough bytes to cover several challenge instrument " + 'batches deterministically"}}\n' +) + + +@dataclass +class PhaseSidecar: + """Sidecar attestor: honest digest, optional mid-run swap after N calls.""" + + digest: str + swap_to: str | None = None + swap_after: int = 10_000 + calls: int = 0 + phases: list[str] = field(default_factory=list) + + async def attest(self, *, pod_id: str, phase: str) -> str: + del pod_id + self.calls += 1 + self.phases.append(phase) + if self.swap_to is not None and self.calls > self.swap_after: + return self.swap_to + return self.digest + + +@dataclass +class ScriptedLium: + digests: list[str | None] + calls: int = 0 + + async def get_pod_raw(self, pod_id: str) -> LiumPodRead: + self.calls += 1 + idx = min(self.calls - 1, len(self.digests) - 1) + return LiumPodRead( + pod_id=pod_id, + template_id="tmpl-fixture", + docker_image_digest=self.digests[idx], + raw={}, + ) + + async def balance(self) -> float: + return 99.0 + + +def _code_bundle() -> str: + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr("architecture.py", _TINY_ARCH) + archive.writestr("training.py", _TINY_TRAIN) + return base64.b64encode(stream.getvalue()).decode("ascii") + + +def _manifest(marker: str) -> dict[str, Any]: + online_loss = [10.0, 6.0, 3.0, 2.0] + covered = 4096 + return { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": covered, "single_pass": True}, + "metrics": { + "online_loss": online_loss, + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": covered, + "predicted_tokens": 96, + "step0_loss": online_loss[0], + "consumed_batches": len(online_loss), + "random_init_baseline_nats": math.log(50257), + "prequential_bpb": 1.23, + "marker": marker, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + + +def _final_score(db_path: Path, submission_id: str) -> float | None: + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", + (submission_id,), + ).fetchone() + finally: + conn.close() + return None if row is None else float(row[0]) + + +def _submit_step() -> dict[str, str]: + sub = validate_submission( + { + "repo_url": "https://github.com/baseintelligence/prism-recipe-e2e-sample.git", + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": VARIANT, + } + ) + return sub.as_dict() + + +def _register_allowlist(allowlist: DigestAllowlist, digest: str) -> None: + allowlist.register( + DigestRecord( + commit_sha=COMMIT, + tree_sha=TREE, + variant=ImageVariant.CUDA, + digest=digest, + ) + ) + + +def _checkers( + allowlist: DigestAllowlist, + nonces: AttestationNonceService, + verify_key: bytes, +): + def check_allowlist( + *, digest: str, commit_sha: str, tree_sha: str, variant: str + ) -> CheckOutcome: + hit = allowlist.lookup( + digest=digest, commit_sha=commit_sha, tree_sha=tree_sha, variant=variant + ) + if isinstance(hit, AllowlistHit): + return CheckOutcome(ok=True, reason="ok") + return CheckOutcome(ok=False, reason=hit.reason.value) + + def check_nonce( + *, nonce: str, work_unit_id: str, miner_hotkey: str, pod_id: str + ) -> CheckOutcome: + result = nonces.consume( + nonce, + NonceBinding( + work_unit_id=work_unit_id, miner_hotkey=miner_hotkey, pod_id=pod_id + ), + ) + if isinstance(result, NonceConsumeHit): + return CheckOutcome(ok=True, reason="ok") + return CheckOutcome(ok=False, reason=result.reason.value) + + def verify_signature(signed: object) -> CheckOutcome: + outcome = verify_attestation_payload(signed, verify_key=verify_key) # type: ignore[arg-type] + if outcome.ok: + return CheckOutcome(ok=True, reason="ok") + return CheckOutcome(ok=False, reason=outcome.reason.value) + + return check_allowlist, check_nonce, verify_signature + + +def _sign_attestation(*, digest: str, nonce: str, pod_id: str, signing_key: bytes): + payload = AttestationPayload( + nonce=nonce, + digest=digest, + pod_id=pod_id, + variant=VARIANT, + sealed_manifest_hashes=dict(SEALED), + build_secret_response=compute_build_secret_response( + build_secret=BUILD_SECRET, nonce=nonce + ), + ) + return sign_attestation_payload(payload, signing_key=signing_key) + + +async def _ensure_template_with_digest( + client: LiumClient, digest: str +) -> tuple[str, str]: + """Create/reuse a private template pinned to ``digest``. Returns (id, name).""" + name = f"{TEMPLATE_PREFIX}-{digest[-12:]}" + template_id = await client.ensure_template( + name=name, + docker_image=E2E_IMAGE, + docker_image_tag=E2E_IMAGE_TAG, + docker_image_digest=digest, + startup_commands=E2E_STARTUP, + ) + return template_id, name + + +async def _rent_running_pod( + client: LiumClient, *, ssh_pub: str, digest: str +) -> tuple[str, str]: + _template_id, template_name = await _ensure_template_with_digest(client, digest) + del _template_id + offers = [ + o + for o in await client.list_offers(max_price_per_hour=MAX_PRICE) + if 0 < o.price_per_hour <= MAX_PRICE and o.gpu_count >= 1 + ] + offers.sort(key=lambda o: (o.price_per_hour, o.gpu_count)) + if not offers: + raise RuntimeError("no suitable Lium offer") + last_err: Exception | None = None + for offer in offers[:8]: + pod_name = f"prism-attest-{int(time.time())}-{offer.id[:6]}" + spec = InstanceSpec( + name=pod_name, + template_ref=template_name, + image=E2E_IMAGE, + image_digest=digest, + ssh_public_keys=(ssh_pub,), + ssh_key_name=prov.SSH_KEY_NAME, + startup_commands=E2E_STARTUP, + max_lifetime_hours=float(TERMINATION_HOURS), + max_price_per_hour=MAX_PRICE, + gpu_count=1, + ) + pod_id: str | None = None + try: + print( + f"[rent] {offer.gpu_type} @ ${offer.price_per_hour}/gpu/hr " + f"id={offer.id[:8]} tmpl={template_name}" + ) + inst = await client.provision(spec, offer=offer) + pod_id = inst.id + except Exception as exc: # noqa: BLE001 - try next offer + last_err = exc + print(f"[rent] fail {type(exc).__name__}: {exc}") + continue + deadline = time.monotonic() + POLL_BUDGET + try: + while True: + st = await client.status(pod_id) + status = (st.status or "").upper() + print(f"[poll] pod={pod_id[:8]} status={status}") + if status == "RUNNING": + return pod_id, pod_name + if status in {"FAILED", "CREATION_FAILED", "BROKEN", "STOPPED"}: + last_err = RuntimeError(f"pod terminal {status}") + print(f"[rent] {last_err}; trying next offer") + await client.terminate(pod_id) + break + if time.monotonic() >= deadline: + last_err = RuntimeError("pod RUNNING timeout") + await client.terminate(pod_id) + break + await asyncio.sleep(20.0) + except Exception as exc: # noqa: BLE001 + last_err = exc + if pod_id: + try: + await client.terminate(pod_id) + except Exception: # noqa: BLE001 + pass + raise RuntimeError(f"rent failed after offer retries: {last_err}") + + +async def _run_constation( + *, + custody: LiumKeyCustody, + sidecar: PhaseSidecar, + pod_id: str, + work_unit_id: str, + duration: float, + live_clock: bool, +) -> Any: + if live_clock: + + async def sleep_fn(seconds: float) -> None: + await asyncio.sleep(min(seconds, 2.0)) # compress intervals for E2E + + now_fn = time.monotonic + else: + clock = {"t": 0.0} + + def now_fn() -> float: + return clock["t"] + + async def sleep_fn(seconds: float) -> None: + clock["t"] += max(0.0, seconds) + + runner = ConstationRunner( + custody=custody, + sidecar=sidecar, + poller_config=PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=8, + max_cost_units=16.0, + rate_limit_per_second=100.0, + ), + now_fn=now_fn, + sleep_fn=sleep_fn, + rng_fn=lambda: 0.0, + ) + return await runner.run( + ConstationRunRequest( + miner_hotkey=HOTKEY, + work_unit_id=work_unit_id, + pod_id=pod_id, + duration_seconds=duration, + expected_sidecar_digest=sidecar.digest, + ) + ) + + +async def _ingest( + *, + tmp: Path, + digest: str, + pod_id: str, + work_unit_id_marker: str, + bundle: ConstationBundle | None, + allowlist: DigestAllowlist, + nonces: AttestationNonceService, + verify_key: bytes, + expect_score: bool, +) -> dict[str, Any]: + data_dir = tmp / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + "".join(_SHARD.format(i=i) for i in range(64)), encoding="utf-8" + ) + db_path = tmp / f"{work_unit_id_marker}.sqlite3" + settings = PrismSettings( + database_url=f"sqlite+aiosqlite:///{db_path}", + shared_token="e2e-secret", + allow_insecure_signatures=True, + execution_backend="base_gpu", + docker_enabled=True, + docker_backend="broker", + docker_broker_url="http://base-docker-broker:8082", + docker_broker_token="secret", + sequence_length=16, + plagiarism_enabled=False, + distributed_contract_policy="off", + base_eval_artifact_root=tmp / "artifacts", + worker_plane=WorkerPlaneConfig(enabled=True, signing_key=WORKER_KEY), + ) + # Patch docker executor the same way unit tests do. + import prism_challenge.evaluator.container as container_mod + + original = container_mod.DockerExecutor.run + container_mod.DockerExecutor.run = cpu_reexec_run(train_data_dir=data_dir) # type: ignore[method-assign] + try: + app = create_app(settings) + await app.state.database.init() + sub = await app.state.repository.create_submission( + HOTKEY, SubmissionCreate(code=_code_bundle(), filename="project.zip") + ) + submission_id = sub.id + manifest = _manifest(work_unit_id_marker) + signer = worker_signer_from_key(WORKER_KEY) + proof = build_execution_proof( + signer=signer, + manifest_sha256=compute_manifest_sha256(manifest), + unit_id=submission_id, + image_digest=digest, + constation_digest=digest, + provider=ProviderInfo(name="lium", pod_id=pod_id), + tier=1, # type: ignore[arg-type] + ) + result = { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof.model_dump(mode="json"), + MANIFEST_PAYLOAD_KEY: manifest, + } + allow, nonce, sig = _checkers(allowlist, nonces, verify_key) + # Re-bind bundle work_unit_id to the real submission id + re-issue nonce. + if bundle is not None: + binding = NonceBinding( + work_unit_id=submission_id, miner_hotkey=HOTKEY, pod_id=pod_id + ) + issued = nonces.issue(binding) + signed = _sign_attestation( + digest=bundle.digest, + nonce=issued.nonce, + pod_id=pod_id, + signing_key=verify_key, + ) + from dataclasses import replace + + bundle = replace( + bundle, + work_unit_id=submission_id, + nonce=issued.nonce, + signed_attestation=signed, + pod_id=pod_id, + ) + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref=HOTKEY, + result=result, + pinned_image_digest=digest, + constation_bundle=bundle, + check_allowlist=allow if bundle is not None else None, + check_nonce=nonce if bundle is not None else None, + verify_constation_signature=sig if bundle is not None else None, + ) + score = _final_score(db_path, submission_id) + out = { + "status": outcome.status, + "effective_tier": outcome.effective_tier, + "attestation_mode": outcome.attestation_mode, + "score_written": outcome.score_written, + "reason": outcome.reason, + "final_score": score, + "submission_id": submission_id, + } + if expect_score: + assert outcome.status == "accepted", out + assert outcome.score_written is True, out + assert outcome.effective_tier == 1, out + assert outcome.attestation_mode == ATTESTATION_MODE_V1, out + assert score is not None and score > 0.0, out + else: + assert outcome.score_written is False, out + assert score is None, out + assert outcome.reason and "miner_fault" in outcome.reason, out + return out + finally: + container_mod.DockerExecutor.run = original # type: ignore[method-assign] + + +def _bundle_from_record( + record: Any, + *, + digest: str, + nonce: str, + signed: object, + pod_id: str, + work_unit_id: str, +) -> ConstationBundle: + return ConstationBundle( + commit_sha=COMMIT, + tree_sha=TREE, + variant=VARIANT, + digest=digest, + work_unit_id=work_unit_id, + miner_hotkey=HOTKEY, + pod_id=pod_id, + nonce=nonce, + signed_attestation=signed, + expected_sealed_manifest_hashes=dict(SEALED), + reported_sealed_manifest_hashes=dict(SEALED), + lium_declared_digest=record.lium_declared_digest or digest, + constation_gap_budget_seconds=record.constation_gap_budget_seconds, + constation_observed_max_gap_seconds=record.constation_observed_max_gap_seconds, + ) + + +async def run_honest(*, live: bool, tmp: Path) -> dict[str, Any]: + print("=== HONEST PATH ===") + submission = _submit_step() + allowlist = DigestAllowlist() + digest = DIGEST_LIVE_CUDA if live else DIGEST_HONEST + _register_allowlist(allowlist, digest) + verify_key = derive_attestation_key(BUILD_SECRET) + nonces = AttestationNonceService(ttl=timedelta(hours=2)) + custody = LiumKeyCustody(master_key=generate_custody_master_key()) + + pod_id = "pod-fixture-honest" + real_lium = False + client: LiumClient | None = None + try: + if live: + creds = prov._load_credentials() + api_key = creds["LIUM_API_KEY"] + ssh_pub = prov._load_ssh_public_key() + client = LiumClient(api_key=api_key) + bal = await client.balance() + print(f"[lium] balance=${bal:.4f}") + if bal < 2: + raise SystemExit("Lium balance < $2") + await custody.register(miner_hotkey=HOTKEY, api_key=api_key) + pod_id, _name = await _rent_running_pod( + client, ssh_pub=ssh_pub, digest=digest + ) + pod_read = await client.get_pod_raw(pod_id) + declared = pod_read.docker_image_digest or digest + if declared.lower() != digest.lower(): + # Prefer Lium-declared digest as the allowlisted binding when present. + allowlist = DigestAllowlist() + digest = declared if declared.startswith("sha256:") else digest + _register_allowlist(allowlist, digest) + real_lium = True + sidecar = PhaseSidecar(digest=digest) + record = await _run_constation( + custody=custody, + sidecar=sidecar, + pod_id=pod_id, + work_unit_id="wu-honest-placeholder", + duration=12.0, + live_clock=True, + ) + else: + scripted = ScriptedLium(digests=[digest]) + + def factory(key: str) -> Any: + del key + return scripted + + async def _probe(c: Any) -> None: + await c.balance() + + custody = LiumKeyCustody( + master_key=generate_custody_master_key(), + client_factory=factory, + probe_fn=_probe, + ) + await custody.register(miner_hotkey=HOTKEY, api_key="fixture-key") + sidecar = PhaseSidecar(digest=digest) + record = await _run_constation( + custody=custody, + sidecar=sidecar, + pod_id=pod_id, + work_unit_id="wu-honest-placeholder", + duration=10.0, + live_clock=False, + ) + + assert record.ok is True, f"constation failed: {record.reason} {record.detail}" + assert record.fault_class is None + phases = set(sidecar.phases) + assert "start" in phases and "end" in phases, sidecar.phases + + # Placeholder nonce/sign — _ingest re-issues against real submission id. + binding = NonceBinding( + work_unit_id="wu-honest-placeholder", miner_hotkey=HOTKEY, pod_id=pod_id + ) + issued = nonces.issue(binding) + signed = _sign_attestation( + digest=digest, nonce=issued.nonce, pod_id=pod_id, signing_key=verify_key + ) + bundle = _bundle_from_record( + record, + digest=digest, + nonce=issued.nonce, + signed=signed, + pod_id=pod_id, + work_unit_id="wu-honest-placeholder", + ) + # Fresh nonce service for ingest consume (issue inside _ingest). + nonces_ingest = AttestationNonceService(ttl=timedelta(hours=2)) + ingest_out = await _ingest( + tmp=tmp / "honest", + digest=digest, + pod_id=pod_id, + work_unit_id_marker="honest", + bundle=bundle, + allowlist=allowlist, + nonces=nonces_ingest, + verify_key=verify_key, + expect_score=True, + ) + return { + "RESULT": "PASS", + "path": "honest", + "REAL_LIUM": real_lium, + "submission": submission, + "digest": digest, + "pod_id": pod_id, + "constation_ok": True, + "constation_reason": str(record.reason), + "corroboration": str(record.corroboration_status), + "samples": len(record.samples), + "sidecar_phases": list(sidecar.phases), + "ingest": ingest_out, + "notes": ( + "Live Lium pod + get_pod_raw + sidecar HMAC; prism tier1 score. " + "BASE build = DigestAllowlist.register of deployed digest " + "(same API post-build pipeline calls)." + if real_lium + else "Fixture: ScriptedLium + PhaseSidecar; no network." + ), + } + finally: + if client is not None and pod_id and not pod_id.startswith("pod-fixture"): + try: + await client.terminate(pod_id) + await client.verify_terminated(pod_id) + print(f"[cleanup] terminated {pod_id}") + except Exception as exc: # noqa: BLE001 + print(f"[cleanup] warn {type(exc).__name__}: {exc}") + + +async def run_adversarial(*, live: bool, tmp: Path) -> dict[str, Any]: + print("=== ADVERSARIAL PATH (mid-run image swap) ===") + submission = _submit_step() + allowlist = DigestAllowlist() + digest = DIGEST_LIVE_CUDA if live else DIGEST_HONEST + _register_allowlist(allowlist, digest) + verify_key = derive_attestation_key(BUILD_SECRET) + custody = LiumKeyCustody(master_key=generate_custody_master_key()) + pod_id = "pod-fixture-adv" + real_lium = False + client: LiumClient | None = None + try: + if live: + creds = prov._load_credentials() + api_key = creds["LIUM_API_KEY"] + ssh_pub = prov._load_ssh_public_key() + client = LiumClient(api_key=api_key) + await custody.register(miner_hotkey=HOTKEY, api_key=api_key) + pod_id, _ = await _rent_running_pod(client, ssh_pub=ssh_pub, digest=digest) + pod_read = await client.get_pod_raw(pod_id) + declared = pod_read.docker_image_digest or digest + if declared.startswith("sha256:"): + allowlist = DigestAllowlist() + digest = declared + _register_allowlist(allowlist, digest) + real_lium = True + # After start sample, sidecar reports swapped image digest (miner swap). + sidecar = PhaseSidecar(digest=digest, swap_to=DIGEST_SWAP, swap_after=1) + record = await _run_constation( + custody=custody, + sidecar=sidecar, + pod_id=pod_id, + work_unit_id="wu-adv-placeholder", + duration=12.0, + live_clock=True, + ) + else: + scripted = ScriptedLium(digests=[digest]) + + def factory(key: str) -> Any: + del key + return scripted + + async def _probe(c: Any) -> None: + await c.balance() + + custody = LiumKeyCustody( + master_key=generate_custody_master_key(), + client_factory=factory, + probe_fn=_probe, + ) + await custody.register(miner_hotkey=HOTKEY, api_key="fixture-key") + sidecar = PhaseSidecar(digest=digest, swap_to=DIGEST_SWAP, swap_after=1) + record = await _run_constation( + custody=custody, + sidecar=sidecar, + pod_id=pod_id, + work_unit_id="wu-adv-placeholder", + duration=10.0, + live_clock=False, + ) + + assert record.ok is False, "adversarial must fail constation" + assert record.reason is ConstationFailCode.CORROBORATION_MISMATCH, record.reason + assert record.fault_class is FaultClass.MINER, record.fault_class + + # Ingest with no valid bundle (constation failed) → miner_fault, no score. + nonces_ingest = AttestationNonceService(ttl=timedelta(hours=2)) + ingest_out = await _ingest( + tmp=tmp / "adversarial", + digest=digest, + pod_id=pod_id, + work_unit_id_marker="adversarial", + bundle=None, + allowlist=allowlist, + nonces=nonces_ingest, + verify_key=verify_key, + expect_score=False, + ) + # Also prove mismatch reason surfaces when a tampered bundle is forced. + assert ingest_out["reason"] == "miner_fault:missing_constation_bundle" or ( + ingest_out["reason"] or "" + ).startswith("miner_fault") + + return { + "RESULT": "PASS", + "path": "adversarial", + "REAL_LIUM": real_lium, + "submission": submission, + "digest_allowlisted": digest, + "digest_swapped_to": DIGEST_SWAP, + "pod_id": pod_id, + "constation_ok": False, + "constation_reason": str(record.reason), + "fault_class": str(record.fault_class), + "sidecar_phases": list(sidecar.phases), + "ingest": ingest_out, + "notes": ( + "Live Lium declared digest held; sidecar flipped after start " + "(mid-run swap) -> corroboration_mismatch miner_fault; no score." + if real_lium + else "Fixture: ScriptedLium + sidecar swap after start." + ), + } + finally: + if client is not None and pod_id and not pod_id.startswith("pod-fixture"): + try: + await client.terminate(pod_id) + await client.verify_terminated(pod_id) + print(f"[cleanup] terminated {pod_id}") + except Exception as exc: # noqa: BLE001 + print(f"[cleanup] warn {type(exc).__name__}: {exc}") + + +def _write_evidence(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = [ + f"RESULT={payload.get('RESULT')}", + f"REAL_LIUM={payload.get('REAL_LIUM')}", + f"path={payload.get('path')}", + f"constation_ok={payload.get('constation_ok')}", + f"constation_reason={payload.get('constation_reason')}", + ] + if "fault_class" in payload: + lines.append(f"fault_class={payload['fault_class']}") + if "ingest" in payload: + ing = payload["ingest"] + lines.append(f"ingest_status={ing.get('status')}") + lines.append(f"effective_tier={ing.get('effective_tier')}") + lines.append(f"attestation_mode={ing.get('attestation_mode')}") + lines.append(f"score_written={ing.get('score_written')}") + lines.append(f"ingest_reason={ing.get('reason')}") + lines.append(f"final_score={ing.get('final_score')}") + lines.append(f"pod_id={payload.get('pod_id')}") + lines.append(f"sidecar_phases={payload.get('sidecar_phases')}") + lines.append(f"notes={payload.get('notes')}") + lines.append("") + lines.append("JSON=") + lines.append(json.dumps(payload, indent=2, default=str)) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"[evidence] wrote {path}") + + +def _detect_live(requested: str) -> bool: + if requested == "fixture": + return False + if requested == "live": + return True + # auto + if os.environ.get("BASE_LIVE_PROVIDER_TESTS") != "1": + # Still allow auto-live when key present and user did not force fixture. + pass + try: + prov._load_credentials() + prov._load_ssh_public_key() + return True + except SystemExit: + return False + + +async def _amain(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + choices=("auto", "live", "fixture"), + default="auto", + help="live=real Lium; fixture=no network; auto=live if credentials exist", + ) + parser.add_argument( + "--evidence-dir", + type=Path, + default=DEFAULT_EVIDENCE, + help="Directory for 27-e2e-*.txt evidence", + ) + parser.add_argument( + "--only", + choices=("honest", "adversarial", "both"), + default="both", + ) + args = parser.parse_args(argv) + + live = _detect_live(args.mode) + if args.mode == "live" and not live: + raise SystemExit("live mode requested but LIUM credentials/SSH missing") + if live and os.environ.get("BASE_LIVE_PROVIDER_TESTS") != "1": + os.environ["BASE_LIVE_PROVIDER_TESTS"] = "1" + print("[gate] BASE_LIVE_PROVIDER_TESTS=1 (live credentials present)") + + print(f"[mode] live={live} REAL_LIUM will be {live}") + tmp = Path(os.environ.get("PRISM_ATTEST_E2E_TMP", "/tmp/prism-lium-attest-e2e")) + tmp.mkdir(parents=True, exist_ok=True) + + rc = 0 + if args.only in ("honest", "both"): + try: + honest = await run_honest(live=live, tmp=tmp) + _write_evidence(args.evidence_dir / "27-e2e-honest.txt", honest) + except BaseException as exc: + rc = 1 + fail = { + "RESULT": "FAIL", + "REAL_LIUM": live, + "path": "honest", + "error": f"{type(exc).__name__}: {exc}", + } + _write_evidence(args.evidence_dir / "27-e2e-honest.txt", fail) + print(f"[honest] FAIL: {exc}") + if not isinstance(exc, Exception): + # SystemExit / KeyboardInterrupt — still continue adversarial if both + pass + + if args.only in ("adversarial", "both"): + try: + adv = await run_adversarial(live=live, tmp=tmp) + _write_evidence(args.evidence_dir / "27-e2e-adversarial.txt", adv) + except BaseException as exc: + rc = 1 + fail = { + "RESULT": "FAIL", + "REAL_LIUM": live, + "path": "adversarial", + "error": f"{type(exc).__name__}: {exc}", + } + _write_evidence(args.evidence_dir / "27-e2e-adversarial.txt", fail) + print(f"[adversarial] FAIL: {exc}") + + return rc + + +def main() -> None: + raise SystemExit(asyncio.run(_amain())) + + +if __name__ == "__main__": + main() diff --git a/src/base/attestation/__init__.py b/src/base/attestation/__init__.py new file mode 100644 index 000000000..31df65862 --- /dev/null +++ b/src/base/attestation/__init__.py @@ -0,0 +1,40 @@ +"""Prism-lium image attestation protocol (pure crypto + schema). + +See ``payload`` module docstring: signature proves only that an entity holding +the in-image secret responded — not a hardware root of trust, and never +sufficient alone for tier elevation (B3). +""" + +from base.attestation.payload import ( + ALGORITHM, + SCHEMA_VERSION, + VALID_VARIANTS, + AttestationPayload, + AttestationPayloadError, + AttestationVerifyError, + AttestationVerifyReason, + AttestationVerifyResult, + SignedAttestation, + canonical_payload_bytes, + compute_build_secret_response, + derive_attestation_key, + sign_attestation_payload, + verify_attestation_payload, +) + +__all__ = [ + "ALGORITHM", + "SCHEMA_VERSION", + "AttestationPayload", + "AttestationPayloadError", + "AttestationVerifyError", + "AttestationVerifyReason", + "AttestationVerifyResult", + "SignedAttestation", + "VALID_VARIANTS", + "canonical_payload_bytes", + "compute_build_secret_response", + "derive_attestation_key", + "sign_attestation_payload", + "verify_attestation_payload", +] diff --git a/src/base/attestation/payload.py b/src/base/attestation/payload.py new file mode 100644 index 000000000..2968c36b0 --- /dev/null +++ b/src/base/attestation/payload.py @@ -0,0 +1,365 @@ +"""Attestation payload schema and HMAC-SHA256 sign/verify. + +Mechanism 1+6 surface of prism-lium image attestation: the in-image sidecar +signs ``(nonce, digest, pod_id, variant, sealed_manifest_hashes, +build_secret_response)``. **BASE holds the verify key.** The sidecar holds +sign material derived from the per-build secret (see ``derive_attestation_key``). + +Honesty constraints (binding — do not weaken in callers): + +* A valid signature proves **only** that an entity holding the in-image secret + responded. It is **not** a hardware root of trust. +* A valid signature is **never sufficient alone for tier elevation** (B3). + Tier 1 is granted only by the later ``constation_ok`` predicate, which + requires allowlist hit, nonce lifecycle, signature, sealed-manifest match, + corroboration, and continuous constation together. +* This module deliberately exposes **no** tier-grant API. + +Crypto: HMAC-SHA256 over a canonical JSON encoding (stdlib only — no extra +deps). Symmetric: the verify key BASE stores is the same derived key the +sidecar uses to sign. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Final, Literal, assert_never + +SCHEMA_VERSION: Final[str] = "prism_attestation_payload.v1" +ALGORITHM: Final[str] = "hmac-sha256" + +# Domain-separated key derivation label (not a password; not a TEE claim). +_KEY_DERIVE_INFO: Final[bytes] = b"prism-lium-attestation-hmac-v1" + +_IMAGE_DIGEST_RE: Final[re.Pattern[str]] = re.compile(r"^sha256:[0-9a-f]{64}$") +_HEX64_RE: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]{64}$") +_HEX_RE: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]+$") + +Variant = Literal["cpu", "cuda"] +VALID_VARIANTS: Final[frozenset[str]] = frozenset({"cpu", "cuda"}) + + +class AttestationVerifyReason(StrEnum): + """Machine-consumed verify outcomes. Callers must not string-match prose.""" + + OK = "ok" + SIGNATURE_MISMATCH = "signature_mismatch" + INVALID_PAYLOAD = "invalid_payload" + UNSUPPORTED_ALGORITHM = "unsupported_algorithm" + UNSUPPORTED_SCHEMA = "unsupported_schema" + EMPTY_KEY = "empty_key" + + +class AttestationPayloadError(ValueError): + """Payload failed structural validation before signing/verifying.""" + + +class AttestationVerifyError(Exception): + """Raised when ``raise_on_failure=True`` and verification does not pass.""" + + def __init__(self, message: str, *, reason: AttestationVerifyReason) -> None: + super().__init__(message) + self.reason = reason + + +@dataclass(frozen=True, slots=True) +class AttestationPayload: + """Unsigned attestation fields the sidecar binds under HMAC. + + Field meanings (tamper-evidence only — not hardware attestation): + + * ``nonce`` — BASE-issued single-use challenge id (opaque string). + * ``digest`` — container image content digest ``sha256:<64-hex>``. + * ``pod_id`` — Lium (or equivalent) pod identifier the run claims. + * ``variant`` — ``cpu`` or ``cuda`` image variant. + * ``sealed_manifest_hashes`` — path → lowercase SHA-256 hex of sealed files. + * ``build_secret_response`` — response proving possession of the in-image + build secret for this nonce (opaque lowercase hex; typically + HMAC-SHA256(build_secret, nonce)). + """ + + nonce: str + digest: str + pod_id: str + variant: Variant + sealed_manifest_hashes: Mapping[str, str] + build_secret_response: str + + def __post_init__(self) -> None: + object.__setattr__(self, "nonce", _require_nonempty_str("nonce", self.nonce)) + object.__setattr__(self, "digest", _require_image_digest(self.digest)) + object.__setattr__(self, "pod_id", _require_nonempty_str("pod_id", self.pod_id)) + object.__setattr__(self, "variant", _require_variant(self.variant)) + object.__setattr__( + self, + "sealed_manifest_hashes", + _require_manifest_hashes(self.sealed_manifest_hashes), + ) + object.__setattr__( + self, + "build_secret_response", + _require_hex_str("build_secret_response", self.build_secret_response), + ) + + +@dataclass(frozen=True, slots=True) +class SignedAttestation: + """Payload plus HMAC-SHA256 signature hex. BASE verifies with its key.""" + + payload: AttestationPayload + signature: str + algorithm: str = ALGORITHM + schema_version: str = SCHEMA_VERSION + + def __post_init__(self) -> None: + object.__setattr__( + self, "signature", _require_hex_str("signature", self.signature) + ) + object.__setattr__( + self, "algorithm", _require_nonempty_str("algorithm", self.algorithm) + ) + object.__setattr__( + self, + "schema_version", + _require_nonempty_str("schema_version", self.schema_version), + ) + + +@dataclass(frozen=True, slots=True) +class AttestationVerifyResult: + """Structured verify outcome. ``ok`` never implies tier elevation.""" + + ok: bool + reason: AttestationVerifyReason + payload: AttestationPayload | None = None + + +def derive_attestation_key(build_secret: bytes | str) -> bytes: + """Derive the HMAC key BASE stores for verify and the sidecar uses to sign. + + The raw build secret is never used directly as the HMAC key. Derivation is + domain-separated HMAC-SHA256 so accidental reuse of the secret bytes as a + different protocol key does not collide. + + Returns 32 bytes. Empty input is rejected. + """ + secret = ( + build_secret.encode("utf-8") + if isinstance(build_secret, str) + else bytes(build_secret) + ) + if not secret: + raise AttestationPayloadError("build_secret must be non-empty") + return hmac.new(_KEY_DERIVE_INFO, secret, hashlib.sha256).digest() + + +def compute_build_secret_response(*, build_secret: bytes | str, nonce: str) -> str: + """Compute the ``build_secret_response`` field for a challenge nonce. + + ``HMAC-SHA256(build_secret, nonce_utf8)`` hex. Proves possession of the + in-image secret for this nonce; still not a hardware root of trust. + """ + secret = ( + build_secret.encode("utf-8") + if isinstance(build_secret, str) + else bytes(build_secret) + ) + if not secret: + raise AttestationPayloadError("build_secret must be non-empty") + nonce_s = _require_nonempty_str("nonce", nonce) + return hmac.new(secret, nonce_s.encode("utf-8"), hashlib.sha256).hexdigest() + + +def canonical_payload_bytes(payload: AttestationPayload) -> bytes: + """Deterministic UTF-8 JSON bytes covered by the HMAC. + + Keys sorted; ``sealed_manifest_hashes`` sorted by path. Separators compact. + Schema version and algorithm are **not** inside the payload body — they + travel beside the signature on ``SignedAttestation``. + """ + body: dict[str, Any] = { + "build_secret_response": payload.build_secret_response, + "digest": payload.digest, + "nonce": payload.nonce, + "pod_id": payload.pod_id, + "sealed_manifest_hashes": { + path: payload.sealed_manifest_hashes[path] + for path in sorted(payload.sealed_manifest_hashes) + }, + "variant": payload.variant, + } + raw = json.dumps(body, separators=(",", ":"), sort_keys=True, ensure_ascii=True) + return raw.encode("utf-8") + + +def sign_attestation_payload( + payload: AttestationPayload, + *, + signing_key: bytes, +) -> SignedAttestation: + """Sign ``payload`` with the sidecar signing key (HMAC-SHA256). + + BASE holds the matching verify key (same bytes for HMAC). A successful + later verify proves only that an entity holding the in-image secret + responded — never sufficient alone for tier elevation (B3). + """ + key = _require_key(signing_key) + digest = hmac.new(key, canonical_payload_bytes(payload), hashlib.sha256).hexdigest() + return SignedAttestation( + payload=payload, + signature=digest, + algorithm=ALGORITHM, + schema_version=SCHEMA_VERSION, + ) + + +def verify_attestation_payload( + signed: SignedAttestation, + *, + verify_key: bytes, + raise_on_failure: bool = False, +) -> AttestationVerifyResult: + """Verify a signed attestation with the BASE-held verify key. + + A valid signature proves only that an entity holding the in-image secret + responded. It is not a hardware root of trust and is never sufficient + alone for tier elevation (B3). Callers must still run ``constation_ok``. + + Tampering any signed field (or using the wrong key) yields + ``SIGNATURE_MISMATCH``. + """ + result = _verify(signed, verify_key=verify_key) + if raise_on_failure and not result.ok: + raise AttestationVerifyError( + f"attestation verify failed: {result.reason.value}", + reason=result.reason, + ) + return result + + +def _verify(signed: SignedAttestation, *, verify_key: bytes) -> AttestationVerifyResult: + if not verify_key: + return AttestationVerifyResult( + ok=False, reason=AttestationVerifyReason.EMPTY_KEY + ) + if signed.algorithm != ALGORITHM: + return AttestationVerifyResult( + ok=False, reason=AttestationVerifyReason.UNSUPPORTED_ALGORITHM + ) + if signed.schema_version != SCHEMA_VERSION: + return AttestationVerifyResult( + ok=False, reason=AttestationVerifyReason.UNSUPPORTED_SCHEMA + ) + try: + # Re-validate structure (defense in depth if constructed without __post_init__). + payload = signed.payload + expected = hmac.new( + verify_key, canonical_payload_bytes(payload), hashlib.sha256 + ).hexdigest() + except (AttestationPayloadError, TypeError, ValueError): + return AttestationVerifyResult( + ok=False, reason=AttestationVerifyReason.INVALID_PAYLOAD + ) + + if not hmac.compare_digest(expected, signed.signature.lower()): + return AttestationVerifyResult( + ok=False, reason=AttestationVerifyReason.SIGNATURE_MISMATCH + ) + return AttestationVerifyResult( + ok=True, reason=AttestationVerifyReason.OK, payload=payload + ) + + +def _require_key(key: bytes) -> bytes: + if not isinstance(key, (bytes, bytearray)): + raise AttestationPayloadError("signing/verify key must be bytes") + raw = bytes(key) + if not raw: + raise AttestationPayloadError("signing/verify key must be non-empty") + return raw + + +def _require_nonempty_str(field: str, value: str) -> str: + if not isinstance(value, str): + raise AttestationPayloadError(f"{field} must be a string") + cleaned = value.strip() + if not cleaned: + raise AttestationPayloadError(f"{field} must be non-empty") + return cleaned + + +def _require_image_digest(value: str) -> str: + cleaned = _require_nonempty_str("digest", value).lower() + if not _IMAGE_DIGEST_RE.fullmatch(cleaned): + raise AttestationPayloadError( + f"digest must be sha256:<64 lowercase hex>, got {value!r}" + ) + return cleaned + + +def _require_variant(value: Variant | str) -> Variant: + if not isinstance(value, str): + raise AttestationPayloadError(f"variant must be str, got {type(value)!r}") + cleaned = value.strip().lower() + if cleaned not in VALID_VARIANTS: + raise AttestationPayloadError( + f"variant must be one of {sorted(VALID_VARIANTS)}, got {value!r}" + ) + if cleaned == "cpu": + return "cpu" + if cleaned == "cuda": + return "cuda" + assert_never(cleaned) # type: ignore[arg-type] + + +def _require_hex_str(field: str, value: str) -> str: + cleaned = _require_nonempty_str(field, value).lower() + if not _HEX_RE.fullmatch(cleaned) or len(cleaned) % 2 != 0: + raise AttestationPayloadError(f"{field} must be even-length lowercase hex") + return cleaned + + +def _require_manifest_hashes(value: Mapping[str, str]) -> dict[str, str]: + if not isinstance(value, Mapping): + raise AttestationPayloadError("sealed_manifest_hashes must be a mapping") + if not value: + raise AttestationPayloadError("sealed_manifest_hashes must be non-empty") + out: dict[str, str] = {} + for raw_path, raw_hash in value.items(): + path = _require_nonempty_str("sealed_manifest_hashes path", str(raw_path)) + digest = _require_nonempty_str( + "sealed_manifest_hashes value", str(raw_hash) + ).lower() + if not _HEX64_RE.fullmatch(digest): + raise AttestationPayloadError( + f"sealed_manifest_hashes[{path!r}] must be 64-char lowercase hex" + ) + if path in out: + raise AttestationPayloadError(f"duplicate sealed path: {path!r}") + out[path] = digest + return out + + +__all__ = [ + "ALGORITHM", + "SCHEMA_VERSION", + "AttestationPayload", + "AttestationPayloadError", + "AttestationVerifyError", + "AttestationVerifyReason", + "AttestationVerifyResult", + "SignedAttestation", + "VALID_VARIANTS", + "Variant", + "canonical_payload_bytes", + "compute_build_secret_response", + "derive_attestation_key", + "sign_attestation_payload", + "verify_attestation_payload", +] diff --git a/src/base/cli_app/main.py b/src/base/cli_app/main.py index 67c485488..6b51c5f82 100644 --- a/src/base/cli_app/main.py +++ b/src/base/cli_app/main.py @@ -549,6 +549,7 @@ def _master_orchestration_driver( *, worker_service: WorkerCoordinationService | None = None, worker_assignment_service: WorkerAssignmentService | None = None, + bundle_store: Any | None = None, ) -> MasterOrchestrationDriver: """Build the live master orchestration driver (architecture.md sec 4). @@ -587,12 +588,19 @@ def _master_orchestration_driver( worker_service=worker_service, replication_factor=settings.compute.replication_factor, ) + + def _bundle_lookup(work_unit_id: str): + if bundle_store is None: + return None + return bundle_store.get(work_unit_id) + worker_reconciler = WorkerReconciliationService( session_factory, result_forwarder=HttpChallengeResultForwarder( registry, timeout_seconds=settings.master.challenge_timeout_seconds, retries=settings.master.challenge_retries, + bundle_lookup=_bundle_lookup if bundle_store is not None else None, ), ) return MasterOrchestrationDriver( @@ -1136,6 +1144,34 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) # Live autonomy: the orchestration driver bridges challenge pending work into # work_assignments, runs balanced assignment + the full reassignment pass, # and folds retry-exhausted units, all on a Settings-driven interval. + from datetime import timedelta + + from base.master.constation.allowlist_repository import DigestAllowlistRepository + from base.master.constation.bundle_store import ConstationBundleStore + from base.master.constation.nonce_repository import DurableAttestationNonceService + from base.master.constation.routes import build_constation_router + + constation_bundle_store = ConstationBundleStore() + constation_allowlist_repo = DigestAllowlistRepository(session_factory) + constation_nonce_service = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=2) + ) + # Internal token for master constation check_*/issue/register/bundle. + # Reuse admin token when present so ops and prism can share one secret. + constation_internal_token = ( + read_secret( + getattr(settings.security, "admin_token", None), + getattr(settings.security, "admin_token_file", None), + ) + or "constation-internal" + ) + constation_router = build_constation_router( + allowlist_repo=constation_allowlist_repo, + nonce_service=constation_nonce_service, + bundle_store=constation_bundle_store, + internal_token=str(constation_internal_token), + ) + orchestration_driver = _master_orchestration_driver( settings, session_factory, @@ -1143,6 +1179,7 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) validator_service, worker_service=worker_service, worker_assignment_service=worker_assignment_service, + bundle_store=constation_bundle_store, ) # Registry-driven challenge deploy (architecture.md sec 4 + sec 9.2): the # master reconcile loop turns every ACTIVE registry challenge into a running @@ -1217,6 +1254,7 @@ def master_proxy(config: Path = typer.Option(Path("config/master.example.yaml")) agent_challenge_attested_routes_enabled=( settings.master.agent_challenge_attested_routes_enabled ), + constation_router=constation_router, ) endpoint = f"{settings.master.proxy_host}:{settings.master.proxy_port}" typer.echo(f"Starting proxy API on {endpoint}") diff --git a/src/base/compute/__init__.py b/src/base/compute/__init__.py index a2c987b2b..5854a8c89 100644 --- a/src/base/compute/__init__.py +++ b/src/base/compute/__init__.py @@ -7,7 +7,59 @@ from __future__ import annotations -from base.compute.lium import LIUM_API_BASE_URL, LiumClient, LiumError +from base.compute.attestation_nonce import ( + AttestationNonceService, + NonceBinding, + NonceConsumeHit, + NonceConsumeMiss, + NonceConsumeReason, + NonceRecord, + NonceSnapshot, +) +from base.compute.constation_corroboration import ( + CorroborationOutcome, + evaluate_corroboration, +) +from base.compute.constation_custody import ( + LiumKeyCustody, + generate_custody_master_key, +) +from base.compute.constation_poller import ( + ContinuousConstationPoller, + PollerConfig, + PollerResult, +) +from base.compute.constation_runner import ( + ConstationRunner, + ConstationRunRequest, +) +from base.compute.constation_types import ( + ConstationFailCode, + ConstationRunRecord, + ConstationVerdict, + CorroborationStatus, + FaultClass, + PollSample, +) +from base.compute.digest_allowlist import ( + AllowlistHit, + AllowlistMiss, + AllowlistMissReason, + DigestAllowlist, + DigestRecord, + ImageVariant, +) +from base.compute.lium import ( + LIUM_API_BASE_URL, + LiumAuthError, + LiumClient, + LiumError, + LiumNotFoundError, + LiumPodRead, + LiumRateLimitError, + LiumTemplateDigestMismatchError, + LiumTemplateRead, +) from base.compute.provider import ( CostGuardrailError, Instance, @@ -37,6 +89,19 @@ ) __all__ = [ + "AllowlistHit", + "AttestationNonceService", + "NonceBinding", + "NonceConsumeHit", + "NonceConsumeMiss", + "NonceConsumeReason", + "NonceRecord", + "NonceSnapshot", + "AllowlistMiss", + "AllowlistMissReason", + "DigestAllowlist", + "DigestRecord", + "ImageVariant", "LIUM_API_BASE_URL", "TARGON_API_BASE_URL", "WORKER_IMAGE", @@ -50,6 +115,12 @@ "InsufficientCreditsError", "LiumClient", "LiumError", + "LiumTemplateRead", + "LiumTemplateDigestMismatchError", + "LiumRateLimitError", + "LiumPodRead", + "LiumNotFoundError", + "LiumAuthError", "Offer", "ProviderClient", "ProviderError", @@ -61,4 +132,19 @@ "is_metachar_free", "is_pinned_digest", "pinned_image_reference", + "ConstationFailCode", + "ConstationRunRecord", + "ConstationRunRequest", + "ConstationRunner", + "ConstationVerdict", + "ContinuousConstationPoller", + "CorroborationOutcome", + "CorroborationStatus", + "FaultClass", + "LiumKeyCustody", + "PollSample", + "PollerConfig", + "PollerResult", + "evaluate_corroboration", + "generate_custody_master_key", ] diff --git a/src/base/compute/attestation_nonce.py b/src/base/compute/attestation_nonce.py new file mode 100644 index 000000000..bd0d0aed0 --- /dev/null +++ b/src/base/compute/attestation_nonce.py @@ -0,0 +1,215 @@ +"""BASE-issued single-use attestation nonces (mechanism 1). + +BASE issues a UUID nonce bound to ``(work_unit_id, miner_hotkey, pod_id)`` with +a TTL covering job wall-clock plus skew. Exactly one successful consume is +allowed. Freshness is derived from **BASE issue and receive time only** — the +guest clock is untrusted and is never accepted as a security input (M3). + +This module is pure in-memory with a snapshot boundary so a DB adapter can +persist without re-implementing the rules. SQLAlchemy table + alembic migration +live alongside for durable master storage. + +Consume returns distinct machine reason codes (no prose matching): + +* ``already_consumed`` — nonce was already successfully consumed (replay) +* ``unknown_nonce`` — nonce was never issued by this service +* ``expired`` — BASE receive time is past ``expires_at`` +* ``work_unit_mismatch`` / ``miner_hotkey_mismatch`` / ``pod_mismatch`` — + binding does not match the issued triple +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from enum import StrEnum + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +def _require_nonblank(field_name: str, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError(f"{field_name} must be a non-empty string, got {value!r}") + return normalized + + +class NonceConsumeReason(StrEnum): + """Machine-consumed miss codes for nonce consume.""" + + ALREADY_CONSUMED = "already_consumed" + UNKNOWN_NONCE = "unknown_nonce" + EXPIRED = "expired" + WORK_UNIT_MISMATCH = "work_unit_mismatch" + MINER_HOTKEY_MISMATCH = "miner_hotkey_mismatch" + POD_MISMATCH = "pod_mismatch" + + +@dataclass(frozen=True, slots=True) +class NonceBinding: + """Triple that a nonce is issued against and must match on consume.""" + + work_unit_id: str + miner_hotkey: str + pod_id: str + + def __post_init__(self) -> None: + object.__setattr__( + self, "work_unit_id", _require_nonblank("work_unit_id", self.work_unit_id) + ) + object.__setattr__( + self, + "miner_hotkey", + _require_nonblank("miner_hotkey", self.miner_hotkey), + ) + object.__setattr__(self, "pod_id", _require_nonblank("pod_id", self.pod_id)) + + +@dataclass(frozen=True, slots=True) +class NonceRecord: + """One BASE-issued nonce and its lifecycle timestamps (BASE clocks only).""" + + nonce: str + binding: NonceBinding + issued_at: datetime + expires_at: datetime + consumed_at: datetime | None = None + + +@dataclass(frozen=True, slots=True) +class NonceConsumeHit: + """Consume succeeded: first use, unexpired, binding matched.""" + + record: NonceRecord + received_at: datetime + + +@dataclass(frozen=True, slots=True) +class NonceConsumeMiss: + """Consume failed with a distinct machine reason.""" + + reason: NonceConsumeReason + + +NonceConsumeResult = NonceConsumeHit | NonceConsumeMiss + + +@dataclass(frozen=True, slots=True) +class NonceSnapshot: + """Serializable state for DB/file adapters.""" + + records: tuple[NonceRecord, ...] + ttl: timedelta + + +@dataclass +class AttestationNonceService: + """In-memory single-use nonce issuer/consumer. + + ``now_fn`` is the BASE clock. Callers may pass ``received_at`` on consume to + pin the BASE receive instant; there is no guest-clock parameter. + """ + + ttl: timedelta + now_fn: Callable[[], datetime] = field(default=_utc_now) + _by_nonce: dict[str, NonceRecord] = field(default_factory=dict, init=False) + + def __post_init__(self) -> None: + if self.ttl <= timedelta(0): + raise ValueError(f"ttl must be positive, got {self.ttl!r}") + + def issue(self, binding: NonceBinding) -> NonceRecord: + """Issue a fresh UUID nonce bound to ``binding`` at BASE ``now_fn`` time.""" + # Re-validate via constructor path (binding already normalized). + bound = NonceBinding( + work_unit_id=binding.work_unit_id, + miner_hotkey=binding.miner_hotkey, + pod_id=binding.pod_id, + ) + issued_at = self.now_fn() + if issued_at.tzinfo is None: + raise ValueError("BASE now_fn must return timezone-aware datetime") + record = NonceRecord( + nonce=str(uuid.uuid4()), + binding=bound, + issued_at=issued_at, + expires_at=issued_at + self.ttl, + consumed_at=None, + ) + self._by_nonce[record.nonce] = record + return record + + def consume( + self, + nonce: str, + binding: NonceBinding, + *, + received_at: datetime | None = None, + ) -> NonceConsumeResult: + """Consume ``nonce`` once if unexpired and binding matches. + + Check order (first match wins): + 1. unknown nonce → ``unknown_nonce`` + 2. already consumed → ``already_consumed`` + 3. BASE receive time past expires_at → ``expired`` + 4. work_unit / hotkey / pod mismatch → distinct binding reasons + 5. else hit (marks consumed_at = receive time) + """ + want = NonceBinding( + work_unit_id=binding.work_unit_id, + miner_hotkey=binding.miner_hotkey, + pod_id=binding.pod_id, + ) + receive = received_at if received_at is not None else self.now_fn() + if receive.tzinfo is None: + raise ValueError("BASE received_at must be timezone-aware") + + key = nonce.strip() + record = self._by_nonce.get(key) + if record is None: + return NonceConsumeMiss(reason=NonceConsumeReason.UNKNOWN_NONCE) + + if record.consumed_at is not None: + return NonceConsumeMiss(reason=NonceConsumeReason.ALREADY_CONSUMED) + + if receive > record.expires_at: + return NonceConsumeMiss(reason=NonceConsumeReason.EXPIRED) + + if record.binding.work_unit_id != want.work_unit_id: + return NonceConsumeMiss(reason=NonceConsumeReason.WORK_UNIT_MISMATCH) + if record.binding.miner_hotkey != want.miner_hotkey: + return NonceConsumeMiss(reason=NonceConsumeReason.MINER_HOTKEY_MISMATCH) + if record.binding.pod_id != want.pod_id: + return NonceConsumeMiss(reason=NonceConsumeReason.POD_MISMATCH) + + consumed = NonceRecord( + nonce=record.nonce, + binding=record.binding, + issued_at=record.issued_at, + expires_at=record.expires_at, + consumed_at=receive, + ) + self._by_nonce[key] = consumed + return NonceConsumeHit(record=consumed, received_at=receive) + + def snapshot(self) -> NonceSnapshot: + """Export durable state for persistence adapters.""" + return NonceSnapshot(records=tuple(self._by_nonce.values()), ttl=self.ttl) + + @classmethod + def from_snapshot( + cls, + snapshot: NonceSnapshot, + *, + ttl: timedelta | None = None, + now_fn: Callable[[], datetime] = _utc_now, + ) -> AttestationNonceService: + """Restore a service from :meth:`snapshot` output.""" + service = cls(ttl=ttl if ttl is not None else snapshot.ttl, now_fn=now_fn) + for record in snapshot.records: + service._by_nonce[record.nonce] = record + return service diff --git a/src/base/compute/constation_corroboration.py b/src/base/compute/constation_corroboration.py new file mode 100644 index 000000000..3aee752cf --- /dev/null +++ b/src/base/compute/constation_corroboration.py @@ -0,0 +1,94 @@ +"""Same-account digest corroboration — negative-only signal (todo 17 / B2). + +This module compares the Lium API **declared** template digest (configuration +recorded at rent time via ``get_pod_raw`` / ``get_template_raw``) against the +sidecar **reported** digest. + +Honesty constraints (binding) +----------------------------- +* This is **corroboration** only — both channels share one principal + (not a second root of trust). The miner supplies the Lium API key and + owns the pod — one principal controls both channels. +* A **mismatch** forces failure (``corroboration_mismatch`` / miner_fault). +* **Agreement alone never elevates.** It contributes nothing toward tier grant; + prism ``constation_ok`` still requires allowlist, nonce, signature, sealed + manifest, and gap budget. Callers must not treat ``agree`` as sufficient. +* Absence of a Lium-declared digest is **not** a contradiction (channel optional). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from base.compute.constation_types import ( + ConstationFailCode, + ConstationVerdict, + CorroborationStatus, + FaultClass, +) + + +@dataclass(frozen=True, slots=True) +class CorroborationOutcome: + """Result of same-account digest corroboration (negative-only).""" + + status: CorroborationStatus + verdict: ConstationVerdict + lium_declared_digest: str | None + sidecar_digest: str + + @property + def ok(self) -> bool: + return self.verdict.ok + + +def evaluate_corroboration( + *, + lium_declared_digest: str | None, + sidecar_digest: str, +) -> CorroborationOutcome: + """Compare Lium-declared vs sidecar-reported digests (corroboration only). + + Returns: + * ``absent`` + ok when Lium declared digest is missing/blank + * ``agree`` + ok when normalized digests match (does **not** elevate) + * ``mismatch`` + fail when both present and differ + """ + sidecar = sidecar_digest.strip().lower() + if not sidecar: + raise ValueError("sidecar_digest must be a non-empty string") + + if lium_declared_digest is None or not str(lium_declared_digest).strip(): + return CorroborationOutcome( + status=CorroborationStatus.ABSENT, + verdict=ConstationVerdict(ok=True, reason=ConstationFailCode.OK), + lium_declared_digest=None, + sidecar_digest=sidecar, + ) + + declared = str(lium_declared_digest).strip().lower() + if declared == sidecar: + return CorroborationOutcome( + status=CorroborationStatus.AGREE, + verdict=ConstationVerdict(ok=True, reason=ConstationFailCode.OK), + lium_declared_digest=declared, + sidecar_digest=sidecar, + ) + + return CorroborationOutcome( + status=CorroborationStatus.MISMATCH, + verdict=ConstationVerdict( + ok=False, + reason=ConstationFailCode.CORROBORATION_MISMATCH, + fault_class=FaultClass.MINER, + detail=f"lium={declared} sidecar={sidecar}", + ), + lium_declared_digest=declared, + sidecar_digest=sidecar, + ) + + +__all__ = [ + "CorroborationOutcome", + "evaluate_corroboration", +] diff --git a/src/base/compute/constation_custody.py b/src/base/compute/constation_custody.py new file mode 100644 index 000000000..29898e2da --- /dev/null +++ b/src/base/compute/constation_custody.py @@ -0,0 +1,159 @@ +"""Encrypted at-rest custody for miner-supplied Lium API keys (todo 15). + +Where constation runs +--------------------- +``LiumKeyCustody`` + :class:`~base.compute.constation_runner.ConstationRunner` +are **master / worker-plane services**. The miner CLI is not the only caller: +control-plane code registers a miner key, then the runner builds short-lived +:class:`~base.compute.lium.LiumClient` instances for pod/template reads during +continuous constation. Miners never receive decrypted peer keys. + +Custody threat model (M6) — full account power +---------------------------------------------- +A miner-supplied ``LIUM_API_KEY`` authenticates as that Lium account. It can +list/destroy pods, mutate templates, and incur billing. BASE holds the key +solely to perform same-account **corroboration** reads (declared template +digest). Compromise of: + +* the Fernet master key used for at-rest encryption, or +* process memory while a key is unlocked for a request, + +yields **full account power** over the miner's Lium account — including pod +destruction. Mitigations (not eliminations): Fernet encryption at rest, never +log or ``repr`` the plaintext key, probe-on-registration, fail-closed on HTTP +401 / mid-run revocation (``lium_auth_revoked``). This is tamper-evidence +infrastructure, not a hardware trust boundary. No TEE claims. +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Protocol + +from cryptography.fernet import Fernet, InvalidToken + +from base.compute.constation_types import ConstationFailCode, ConstationVerdict +from base.compute.lium import LiumAuthError, LiumClient, LiumError + +logger = logging.getLogger(__name__) + +ProbeFn = Callable[[LiumClient], Awaitable[None]] +ClientFactory = Callable[[str], LiumClient] + + +class _SupportsBalance(Protocol): + async def balance(self) -> float: ... + + +async def default_lium_probe(client: LiumClient) -> None: + """Probe registration by reading account balance (cheap authenticated GET).""" + await client.balance() + + +def generate_custody_master_key() -> bytes: + """Return a new Fernet key suitable for :class:`LiumKeyCustody`.""" + return Fernet.generate_key() + + +@dataclass +class LiumKeyCustody: + """In-memory encrypted store of miner Lium API keys. + + Plaintext keys exist only ephemerally inside :meth:`unlock_api_key` / + :meth:`build_client`. Stored values are Fernet tokens. ``repr`` / logs never + include plaintext or ciphertext blobs that embed the key material in a + recoverable form beyond the opaque token length. + """ + + master_key: bytes + client_factory: ClientFactory = field(default=LiumClient) + probe_fn: ProbeFn = field(default=default_lium_probe) + _fernet: Fernet = field(init=False, repr=False) + _by_hotkey: dict[str, bytes] = field(default_factory=dict, init=False, repr=False) + + def __post_init__(self) -> None: + self._fernet = Fernet(self.master_key) + + def __repr__(self) -> str: + return f"LiumKeyCustody(registered={len(self._by_hotkey)})" + + def registered_hotkeys(self) -> frozenset[str]: + return frozenset(self._by_hotkey) + + def has_key(self, miner_hotkey: str) -> bool: + return miner_hotkey.strip() in self._by_hotkey + + async def register(self, *, miner_hotkey: str, api_key: str) -> ConstationVerdict: + """Probe ``api_key``, then encrypt-at-rest under ``miner_hotkey``. + + Fail closed on probe 401 (``lium_auth_revoked``) or other probe errors + (``probe_failed``). Never logs ``api_key``. + """ + hotkey = _require_nonblank("miner_hotkey", miner_hotkey) + key = _require_nonblank("api_key", api_key) + client = self.client_factory(key) + try: + await self.probe_fn(client) + except LiumAuthError: + logger.warning("lium key probe rejected (401) for miner_hotkey=%s", hotkey) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_AUTH_REVOKED, + detail="probe_401", + ) + except LiumError as exc: + logger.warning( + "lium key probe failed for miner_hotkey=%s status=%s", + hotkey, + getattr(exc, "status_code", None), + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.PROBE_FAILED, + detail=type(exc).__name__, + ) + token = self._fernet.encrypt(key.encode("utf-8")) + self._by_hotkey[hotkey] = token + logger.info("lium key registered (encrypted) for miner_hotkey=%s", hotkey) + return ConstationVerdict(ok=True, reason=ConstationFailCode.OK) + + def unlock_api_key(self, miner_hotkey: str) -> str: + """Decrypt the stored key for in-process use. Raises if missing/corrupt.""" + hotkey = _require_nonblank("miner_hotkey", miner_hotkey) + token = self._by_hotkey.get(hotkey) + if token is None: + raise KeyError(f"no Lium key registered for miner_hotkey={hotkey!r}") + try: + return self._fernet.decrypt(token).decode("utf-8") + except InvalidToken as exc: + raise ValueError("custody token decrypt failed") from exc + + def build_client(self, miner_hotkey: str) -> LiumClient: + """Return a :class:`LiumClient` using the unlocked key (not logged).""" + return self.client_factory(self.unlock_api_key(miner_hotkey)) + + def export_encrypted(self) -> Mapping[str, bytes]: + """Snapshot ciphertext tokens for a durable adapter (no plaintext).""" + return dict(self._by_hotkey) + + def import_encrypted(self, mapping: Mapping[str, bytes]) -> None: + """Load ciphertext tokens previously produced by :meth:`export_encrypted`.""" + self._by_hotkey = { + _require_nonblank("miner_hotkey", k): bytes(v) for k, v in mapping.items() + } + + +def _require_nonblank(field_name: str, value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError(f"{field_name} must be a non-empty string, got {value!r}") + return normalized + + +__all__ = [ + "LiumKeyCustody", + "default_lium_probe", + "generate_custody_master_key", +] diff --git a/src/base/compute/constation_poller.py b/src/base/compute/constation_poller.py new file mode 100644 index 000000000..9760590e9 --- /dev/null +++ b/src/base/compute/constation_poller.py @@ -0,0 +1,370 @@ +"""Continuous constation poller with gap budget and fail-closed limits (todo 16). + +Attests at run **start**, **end**, and **randomized intervals**. Any observed +gap beyond ``gap_budget_seconds`` fails the run (TOCTOU / M2). Lium HTTP 429 +and exhausted network retries are fail-closed (never silent skip). + +All time and I/O are injectable for hermetic tests (fake clocks / transports). +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from enum import StrEnum +from typing import TypeVar + +from base.compute.constation_types import ( + ConstationFailCode, + ConstationVerdict, + FaultClass, + PollSample, +) +from base.compute.lium import LiumAuthError, LiumError, LiumRateLimitError + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + +NowFn = Callable[[], float] +SleepFn = Callable[[float], Awaitable[None]] +RngFn = Callable[[], float] # uniform [0, 1) +PollOnceFn = Callable[[str], Awaitable[PollSample | ConstationVerdict]] + + +class PollPhase(StrEnum): + START = "start" + INTERVAL = "interval" + END = "end" + + +@dataclass(frozen=True, slots=True) +class PollerConfig: + """Budgets and pacing for one submission's continuous constation.""" + + gap_budget_seconds: float = 30.0 + min_interval_seconds: float = 5.0 + max_interval_seconds: float = 20.0 + max_polls: int = 64 + max_cost_units: float = 64.0 + cost_per_poll: float = 1.0 + backoff_base_seconds: float = 0.5 + backoff_max_seconds: float = 8.0 + max_network_retries: int = 3 + rate_limit_per_second: float = 5.0 + + def __post_init__(self) -> None: + if self.gap_budget_seconds <= 0: + raise ValueError("gap_budget_seconds must be positive") + if self.min_interval_seconds <= 0: + raise ValueError("min_interval_seconds must be positive") + if self.max_interval_seconds < self.min_interval_seconds: + raise ValueError("max_interval_seconds must be >= min_interval_seconds") + if self.max_polls < 2: + raise ValueError("max_polls must be >= 2 (start + end)") + if self.max_network_retries < 0: + raise ValueError("max_network_retries must be >= 0") + if self.rate_limit_per_second <= 0: + raise ValueError("rate_limit_per_second must be positive") + + +@dataclass(frozen=True, slots=True) +class PollerResult: + """Outcome of a continuous constation poll window.""" + + ok: bool + reason: ConstationFailCode + fault_class: FaultClass | None + samples: tuple[PollSample, ...] + observed_max_gap_seconds: float + gap_budget_seconds: float + poll_count: int + cost_units: float + detail: str | None = None + + def __bool__(self) -> bool: + return self.ok + + +@dataclass +class _TokenBucket: + """Simple rate limiter (tokens per second) with injectable clock.""" + + rate_per_second: float + now_fn: NowFn + capacity: float = field(init=False) + tokens: float = field(init=False) + updated_at: float = field(init=False) + + def __post_init__(self) -> None: + self.capacity = max(self.rate_per_second, 1.0) + self.tokens = self.capacity + self.updated_at = self.now_fn() + + def try_take(self) -> float: + """Return 0 if a token was taken, else seconds to wait.""" + now = self.now_fn() + elapsed = max(0.0, now - self.updated_at) + self.updated_at = now + self.tokens = min(self.capacity, self.tokens + elapsed * self.rate_per_second) + if self.tokens >= 1.0: + self.tokens -= 1.0 + return 0.0 + missing = 1.0 - self.tokens + return missing / self.rate_per_second + + +@dataclass +class ContinuousConstationPoller: + """Run start / randomized / end polls under gap and cost budgets.""" + + config: PollerConfig + now_fn: NowFn + sleep_fn: SleepFn + rng_fn: RngFn + _limiter: _TokenBucket = field(init=False, repr=False) + + def __post_init__(self) -> None: + self._limiter = _TokenBucket( + rate_per_second=self.config.rate_limit_per_second, + now_fn=self.now_fn, + ) + + async def run( + self, + *, + duration_seconds: float, + poll_once: PollOnceFn, + ) -> PollerResult: + """Execute continuous constation for ``duration_seconds`` of run time. + + ``poll_once(phase)`` performs one attest cycle and returns either a + :class:`PollSample` or a terminal :class:`ConstationVerdict`. + """ + if duration_seconds < 0: + raise ValueError("duration_seconds must be >= 0") + + cfg = self.config + samples: list[PollSample] = [] + cost = 0.0 + poll_count = 0 + observed_max_gap = 0.0 + last_success_at: float | None = None + run_start = self.now_fn() + run_end = run_start + duration_seconds + + async def _one(phase: str) -> PollerResult | None: + nonlocal cost, poll_count, observed_max_gap, last_success_at + if poll_count >= cfg.max_polls: + return _fail( + ConstationFailCode.POLL_CAP_EXCEEDED, + samples, + observed_max_gap, + cfg, + poll_count, + cost, + detail=f"polls={poll_count}", + ) + if cost + cfg.cost_per_poll > cfg.max_cost_units: + return _fail( + ConstationFailCode.COST_CAP_EXCEEDED, + samples, + observed_max_gap, + cfg, + poll_count, + cost, + detail=f"cost={cost}", + ) + + wait = self._limiter.try_take() + if wait > 0: + await self.sleep_fn(wait) + + outcome = await self._poll_with_retry(poll_once, phase) + if isinstance(outcome, ConstationVerdict): + return _fail( + outcome.reason, + samples, + observed_max_gap, + cfg, + poll_count, + cost, + detail=outcome.detail, + fault_class=outcome.fault_class, + ) + + now = outcome.at_monotonic + if last_success_at is not None: + gap = now - last_success_at + if gap > observed_max_gap: + observed_max_gap = gap + if gap > cfg.gap_budget_seconds: + samples.append(outcome) + poll_count += 1 + cost += cfg.cost_per_poll + return _fail( + ConstationFailCode.CONSTATION_GAP, + samples, + observed_max_gap, + cfg, + poll_count, + cost, + detail=( + f"observed={observed_max_gap} " + f"budget={cfg.gap_budget_seconds}" + ), + ) + last_success_at = now + samples.append(outcome) + poll_count += 1 + cost += cfg.cost_per_poll + return None + + # Start + failed = await _one(PollPhase.START) + if failed is not None: + return failed + + # Randomized mid-run intervals until wall clock reaches run_end + while self.now_fn() < run_end: + interval = self._next_interval() + # Sleep in slices so gap detection can fire if poll_once is slow + # relative to budget; still one logical wait between polls. + await self.sleep_fn(interval) + if self.now_fn() >= run_end: + break + # Pre-check idle gap before attempting poll (sidecar silence) + if last_success_at is not None: + idle = self.now_fn() - last_success_at + if idle > observed_max_gap: + observed_max_gap = idle + if idle > cfg.gap_budget_seconds: + return _fail( + ConstationFailCode.CONSTATION_GAP, + samples, + observed_max_gap, + cfg, + poll_count, + cost, + detail=( + f"observed={observed_max_gap} " + f"budget={cfg.gap_budget_seconds}" + ), + ) + failed = await _one(PollPhase.INTERVAL) + if failed is not None: + return failed + + # End + failed = await _one(PollPhase.END) + if failed is not None: + return failed + + return PollerResult( + ok=True, + reason=ConstationFailCode.OK, + fault_class=None, + samples=tuple(samples), + observed_max_gap_seconds=observed_max_gap, + gap_budget_seconds=cfg.gap_budget_seconds, + poll_count=poll_count, + cost_units=cost, + ) + + def _next_interval(self) -> float: + cfg = self.config + span = cfg.max_interval_seconds - cfg.min_interval_seconds + return cfg.min_interval_seconds + self.rng_fn() * span + + async def _poll_with_retry( + self, + poll_once: PollOnceFn, + phase: str, + ) -> PollSample | ConstationVerdict: + cfg = self.config + attempt = 0 + while True: + try: + return await poll_once(phase) + except LiumAuthError: + logger.warning("constation poll auth revoked phase=%s", phase) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_AUTH_REVOKED, + detail=f"phase={phase}", + ) + except LiumRateLimitError: + logger.warning("constation poll rate limited phase=%s", phase) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_RATE_LIMITED, + detail=f"phase={phase}", + ) + except LiumError as exc: + # Transport / 5xx — bounded retry then network_partition + if attempt >= cfg.max_network_retries: + logger.warning( + "constation poll network exhausted phase=%s attempts=%s", + phase, + attempt + 1, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.NETWORK_PARTITION, + detail=f"phase={phase} err={type(exc).__name__}", + ) + delay = min( + cfg.backoff_max_seconds, + cfg.backoff_base_seconds * (2**attempt), + ) + # full jitter + delay = delay * self.rng_fn() + attempt += 1 + await self.sleep_fn(delay) + except Exception as exc: + # Sidecar / unexpected — treat as miner-side attest failure + logger.warning( + "constation poll sidecar/transport failure phase=%s err=%s", + phase, + type(exc).__name__, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.SIDECAR_ATTEST_FAILED, + detail=f"phase={phase} err={type(exc).__name__}", + ) + + +def _fail( + reason: ConstationFailCode, + samples: list[PollSample], + observed_max_gap: float, + cfg: PollerConfig, + poll_count: int, + cost: float, + *, + detail: str | None = None, + fault_class: FaultClass | None = None, +) -> PollerResult: + from base.compute.constation_types import fault_class_for + + return PollerResult( + ok=False, + reason=reason, + fault_class=fault_class if fault_class is not None else fault_class_for(reason), + samples=tuple(samples), + observed_max_gap_seconds=observed_max_gap, + gap_budget_seconds=cfg.gap_budget_seconds, + poll_count=poll_count, + cost_units=cost, + detail=detail, + ) + + +__all__ = [ + "ContinuousConstationPoller", + "PollPhase", + "PollerConfig", + "PollerResult", +] diff --git a/src/base/compute/constation_runner.py b/src/base/compute/constation_runner.py new file mode 100644 index 000000000..7bb321b68 --- /dev/null +++ b/src/base/compute/constation_runner.py @@ -0,0 +1,291 @@ +"""Constation runner — master/worker-plane service entry (todos 15–17). + +Orchestrates encrypted key custody, continuous polling, Lium declared-digest +reads, sidecar attestation, and same-account **corroboration** (negative only). + +Callers +------- +Master and worker-plane code construct :class:`ConstationRunner` with a shared +:class:`~base.compute.constation_custody.LiumKeyCustody` and invoke +:meth:`ConstationRunner.run` per submission. This is **not** miner-CLI-only. + +Outputs a :class:`~base.compute.constation_types.ConstationRunRecord` whose +fields map onto prism ``ConstationBundle`` gap / digest / corroboration inputs. +Agreement on corroboration never elevates by itself. +""" + +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Protocol + +from base.compute.constation_corroboration import evaluate_corroboration +from base.compute.constation_custody import LiumKeyCustody +from base.compute.constation_poller import ContinuousConstationPoller, PollerConfig +from base.compute.constation_types import ( + ConstationFailCode, + ConstationRunRecord, + ConstationVerdict, + CorroborationStatus, + FaultClass, + PollSample, + fault_class_for, +) +from base.compute.lium import LiumAuthError, LiumClient, LiumError, LiumRateLimitError + +logger = logging.getLogger(__name__) + + +class SidecarAttestor(Protocol): + """Fetch sidecar-reported digest / attestation for one poll.""" + + async def attest(self, *, pod_id: str, phase: str) -> str: + """Return sidecar-reported image digest (``sha256:...``).""" + ... + + +@dataclass(frozen=True, slots=True) +class ConstationRunRequest: + """One submission's continuous constation parameters.""" + + miner_hotkey: str + work_unit_id: str + pod_id: str + duration_seconds: float + expected_sidecar_digest: str | None = None + + +NowFn = Callable[[], float] +SleepFn = Callable[[float], Awaitable[None]] +RngFn = Callable[[], float] + + +@dataclass +class ConstationRunner: + """Service entry: custody + poller + corroboration → run record.""" + + custody: LiumKeyCustody + sidecar: SidecarAttestor + poller_config: PollerConfig + now_fn: NowFn + sleep_fn: SleepFn + rng_fn: RngFn + + async def run(self, request: ConstationRunRequest) -> ConstationRunRecord: + """Execute continuous constation for ``request``; fail closed on faults.""" + hotkey = request.miner_hotkey.strip() + if not self.custody.has_key(hotkey): + return _record( + request, + ok=False, + reason=ConstationFailCode.KEY_NOT_REGISTERED, + gap_budget=self.poller_config.gap_budget_seconds, + observed_gap=0.0, + sidecar=None, + lium=None, + status=CorroborationStatus.NOT_EVALUATED, + samples=(), + ) + + try: + client = self.custody.build_client(hotkey) + except (KeyError, ValueError) as exc: + return _record( + request, + ok=False, + reason=ConstationFailCode.KEY_NOT_REGISTERED, + gap_budget=self.poller_config.gap_budget_seconds, + observed_gap=0.0, + sidecar=None, + lium=None, + status=CorroborationStatus.NOT_EVALUATED, + samples=(), + detail=type(exc).__name__, + ) + + poller = ContinuousConstationPoller( + config=self.poller_config, + now_fn=self.now_fn, + sleep_fn=self.sleep_fn, + rng_fn=self.rng_fn, + ) + + async def poll_once(phase: str) -> PollSample | ConstationVerdict: + return await self._poll_once(client, request, phase) + + result = await poller.run( + duration_seconds=request.duration_seconds, + poll_once=poll_once, + ) + + if not result.ok: + last_side = result.samples[-1].sidecar_digest if result.samples else None + last_lium = ( + result.samples[-1].lium_declared_digest if result.samples else None + ) + status = CorroborationStatus.NOT_EVALUATED + if result.reason is ConstationFailCode.CORROBORATION_MISMATCH: + status = CorroborationStatus.MISMATCH + return _record( + request, + ok=False, + reason=result.reason, + gap_budget=result.gap_budget_seconds, + observed_gap=result.observed_max_gap_seconds, + sidecar=last_side, + lium=last_lium, + status=status, + samples=result.samples, + detail=result.detail, + fault_class=result.fault_class, + ) + + if not result.samples: + return _record( + request, + ok=False, + reason=ConstationFailCode.RUN_INCOMPLETE, + gap_budget=result.gap_budget_seconds, + observed_gap=result.observed_max_gap_seconds, + sidecar=None, + lium=None, + status=CorroborationStatus.NOT_EVALUATED, + samples=(), + ) + + # Final corroboration across last sample (and ensure no sample mismatched) + for sample in result.samples: + corr = evaluate_corroboration( + lium_declared_digest=sample.lium_declared_digest, + sidecar_digest=sample.sidecar_digest, + ) + if not corr.ok: + return _record( + request, + ok=False, + reason=ConstationFailCode.CORROBORATION_MISMATCH, + gap_budget=result.gap_budget_seconds, + observed_gap=result.observed_max_gap_seconds, + sidecar=corr.sidecar_digest, + lium=corr.lium_declared_digest, + status=CorroborationStatus.MISMATCH, + samples=result.samples, + detail=corr.verdict.detail, + ) + + last = result.samples[-1] + corr = evaluate_corroboration( + lium_declared_digest=last.lium_declared_digest, + sidecar_digest=last.sidecar_digest, + ) + return _record( + request, + ok=True, + reason=ConstationFailCode.OK, + gap_budget=result.gap_budget_seconds, + observed_gap=result.observed_max_gap_seconds, + sidecar=corr.sidecar_digest, + lium=corr.lium_declared_digest, + status=corr.status, + samples=result.samples, + ) + + async def _poll_once( + self, + client: LiumClient, + request: ConstationRunRequest, + phase: str, + ) -> PollSample | ConstationVerdict: + try: + pod = await client.get_pod_raw(request.pod_id) + except LiumAuthError: + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_AUTH_REVOKED, + detail=f"phase={phase}", + ) + except LiumRateLimitError: + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.LIUM_RATE_LIMITED, + detail=f"phase={phase}", + ) + except LiumError: + # Re-raise so poller applies bounded network retry + raise + + try: + sidecar_digest = await self.sidecar.attest( + pod_id=request.pod_id, phase=phase + ) + except Exception as exc: + logger.warning( + "sidecar attest failed pod=%s phase=%s err=%s", + request.pod_id, + phase, + type(exc).__name__, + ) + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.SIDECAR_ATTEST_FAILED, + detail=f"phase={phase} err={type(exc).__name__}", + ) + + lium_digest = pod.docker_image_digest + corr = evaluate_corroboration( + lium_declared_digest=lium_digest, + sidecar_digest=sidecar_digest, + ) + if not corr.ok: + return ConstationVerdict( + ok=False, + reason=ConstationFailCode.CORROBORATION_MISMATCH, + detail=corr.verdict.detail, + ) + + return PollSample( + at_monotonic=self.now_fn(), + phase=phase, + sidecar_digest=corr.sidecar_digest, + lium_declared_digest=corr.lium_declared_digest, + ) + + +def _record( + request: ConstationRunRequest, + *, + ok: bool, + reason: ConstationFailCode, + gap_budget: float, + observed_gap: float, + sidecar: str | None, + lium: str | None, + status: CorroborationStatus, + samples: tuple[PollSample, ...], + detail: str | None = None, + fault_class: FaultClass | None = None, +) -> ConstationRunRecord: + return ConstationRunRecord( + ok=ok, + reason=reason, + fault_class=fault_class if fault_class is not None else fault_class_for(reason), + miner_hotkey=request.miner_hotkey.strip(), + work_unit_id=request.work_unit_id.strip(), + pod_id=request.pod_id.strip(), + sidecar_digest=sidecar, + lium_declared_digest=lium, + constation_gap_budget_seconds=gap_budget, + constation_observed_max_gap_seconds=observed_gap, + corroboration_status=status, + samples=samples, + detail=detail, + ) + + +__all__ = [ + "ConstationRunRequest", + "ConstationRunner", + "SidecarAttestor", +] diff --git a/src/base/compute/constation_types.py b/src/base/compute/constation_types.py new file mode 100644 index 000000000..57139aef7 --- /dev/null +++ b/src/base/compute/constation_types.py @@ -0,0 +1,129 @@ +"""Shared types for the BASE constation service (todos 15–17). + +Outputs are shaped so prism ``constation_ok`` can consume gap budget fields and +``lium_declared_digest`` without this package importing prism. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from typing import Final + + +class FaultClass(StrEnum): + """Attribution for fail-closed constation outcomes.""" + + MINER = "miner_fault" + INFRA = "infra_fault" + + +class ConstationFailCode(StrEnum): + """Machine-consumed constation runner / poller / corroboration codes.""" + + OK = "ok" + LIUM_AUTH_REVOKED = "lium_auth_revoked" + LIUM_RATE_LIMITED = "lium_rate_limited" + NETWORK_PARTITION = "network_partition" + CONSTATION_GAP = "constation_gap" + CORROBORATION_MISMATCH = "corroboration_mismatch" + POLL_CAP_EXCEEDED = "poll_cap_exceeded" + COST_CAP_EXCEEDED = "cost_cap_exceeded" + PROBE_FAILED = "probe_failed" + KEY_NOT_REGISTERED = "key_not_registered" + SIDECAR_ATTEST_FAILED = "sidecar_attest_failed" + RUN_INCOMPLETE = "run_incomplete" + + +class CorroborationStatus(StrEnum): + """Same-account corroboration channel status (never 'independent').""" + + AGREE = "agree" + MISMATCH = "mismatch" + ABSENT = "absent" + NOT_EVALUATED = "not_evaluated" + + +_FAULT_BY_CODE: Final[dict[ConstationFailCode, FaultClass | None]] = { + ConstationFailCode.OK: None, + ConstationFailCode.LIUM_AUTH_REVOKED: FaultClass.MINER, + ConstationFailCode.LIUM_RATE_LIMITED: FaultClass.INFRA, + ConstationFailCode.NETWORK_PARTITION: FaultClass.INFRA, + ConstationFailCode.CONSTATION_GAP: FaultClass.MINER, + ConstationFailCode.CORROBORATION_MISMATCH: FaultClass.MINER, + ConstationFailCode.POLL_CAP_EXCEEDED: FaultClass.INFRA, + ConstationFailCode.COST_CAP_EXCEEDED: FaultClass.INFRA, + ConstationFailCode.PROBE_FAILED: FaultClass.MINER, + ConstationFailCode.KEY_NOT_REGISTERED: FaultClass.MINER, + ConstationFailCode.SIDECAR_ATTEST_FAILED: FaultClass.MINER, + ConstationFailCode.RUN_INCOMPLETE: FaultClass.INFRA, +} + + +def fault_class_for(code: ConstationFailCode) -> FaultClass | None: + """Return miner vs infra attribution for ``code``.""" + return _FAULT_BY_CODE[code] + + +@dataclass(frozen=True, slots=True) +class ConstationVerdict: + """Structured fail-closed outcome from custody, poller, or runner.""" + + ok: bool + reason: ConstationFailCode + fault_class: FaultClass | None = None + detail: str | None = None + + def __post_init__(self) -> None: + if self.fault_class is None and self.reason is not ConstationFailCode.OK: + object.__setattr__(self, "fault_class", fault_class_for(self.reason)) + + def __bool__(self) -> bool: + return self.ok + + +@dataclass(frozen=True, slots=True) +class PollSample: + """One successful constation poll observation.""" + + at_monotonic: float + phase: str + sidecar_digest: str + lium_declared_digest: str | None + + +@dataclass(frozen=True, slots=True) +class ConstationRunRecord: + """Runner output bundle fragment for prism ``constation_ok`` consumers. + + Elevation still requires the full six-mechanism conjunction in prism; this + record only supplies gap metrics, digests, and corroboration status. + """ + + ok: bool + reason: ConstationFailCode + fault_class: FaultClass | None + miner_hotkey: str + work_unit_id: str + pod_id: str + sidecar_digest: str | None + lium_declared_digest: str | None + constation_gap_budget_seconds: float + constation_observed_max_gap_seconds: float + corroboration_status: CorroborationStatus + samples: tuple[PollSample, ...] + detail: str | None = None + + def __bool__(self) -> bool: + return self.ok + + +__all__ = [ + "ConstationFailCode", + "ConstationRunRecord", + "ConstationVerdict", + "CorroborationStatus", + "FaultClass", + "PollSample", + "fault_class_for", +] diff --git a/src/base/compute/digest_allowlist.py b/src/base/compute/digest_allowlist.py new file mode 100644 index 000000000..46bb7f9f1 --- /dev/null +++ b/src/base/compute/digest_allowlist.py @@ -0,0 +1,246 @@ +"""BASE-produced image digest allowlist with post-hoc revocation. + +Mechanism 4 of prism-lium image attestation: only a digest BASE itself produced +from a registered ``(commit_sha, tree_sha, variant)`` is scoreable. This module +is an **allowlist only** — it does not perform constation, nonce checks, or +signature verification. Those arrive in later waves. + +Lookup is pure and returns distinct miss reasons so callers (prism +``constation_ok``) can fail closed without string-matching prose: + +* ``unknown_digest`` — digest was never registered by BASE +* ``variant_mismatch`` — digest is known but for a different variant (cpu/cuda) +* ``commit_mismatch`` — digest is known but bound to a different commit/tree +* ``revoked`` — digest or its commit is on the denylist + +Storage is in-memory with a snapshot boundary so a DB/file adapter can persist +without re-implementing the rules. SQLAlchemy tables + alembic migration live +alongside for durable master storage. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Final + +_FULL_GIT_SHA_RE: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]{40}$") +_IMAGE_DIGEST_RE: Final[re.Pattern[str]] = re.compile(r"^sha256:[0-9a-f]{64}$") + + +class ImageVariant(StrEnum): + """Build/runtime image variant. Cross-variant scoring is always a miss.""" + + CPU = "cpu" + CUDA = "cuda" + + +class AllowlistMissReason(StrEnum): + """Machine-consumed miss codes for allowlist lookup.""" + + UNKNOWN_DIGEST = "unknown_digest" + VARIANT_MISMATCH = "variant_mismatch" + REVOKED = "revoked" + COMMIT_MISMATCH = "commit_mismatch" + + +def is_full_git_sha(value: str) -> bool: + """Return True if ``value`` is a lowercase 40-hex git object id.""" + return bool(_FULL_GIT_SHA_RE.fullmatch(value)) + + +def is_image_digest(value: str) -> bool: + """Return True if ``value`` is a lowercase ``sha256:<64-hex>`` digest.""" + return bool(_IMAGE_DIGEST_RE.fullmatch(value)) + + +def normalize_image_digest(value: str) -> str: + """Lowercase a digest string; does not validate shape.""" + return value.strip().lower() + + +def _require_full_git_sha(field_name: str, value: str) -> str: + normalized = value.strip().lower() + if not is_full_git_sha(normalized): + raise ValueError( + f"{field_name} must be a full 40-char lowercase hex git SHA, got {value!r}" + ) + return normalized + + +def _require_image_digest(value: str) -> str: + normalized = normalize_image_digest(value) + if not is_image_digest(normalized): + raise ValueError(f"digest must be sha256:<64 lowercase hex>, got {value!r}") + return normalized + + +def _require_variant(value: ImageVariant | str) -> ImageVariant: + if isinstance(value, ImageVariant): + return value + try: + return ImageVariant(str(value).strip().lower()) + except ValueError as exc: + raise ValueError( + f"variant must be one of {[v.value for v in ImageVariant]}, got {value!r}" + ) from exc + + +@dataclass(frozen=True, slots=True) +class DigestRecord: + """One BASE-produced image binding. + + Keyed conceptually as ``(commit_sha, tree_sha, variant, digest)``. A given + digest maps to exactly one binding. + """ + + commit_sha: str + tree_sha: str + variant: ImageVariant + digest: str + + def __post_init__(self) -> None: + object.__setattr__( + self, "commit_sha", _require_full_git_sha("commit_sha", self.commit_sha) + ) + object.__setattr__( + self, "tree_sha", _require_full_git_sha("tree_sha", self.tree_sha) + ) + object.__setattr__(self, "variant", _require_variant(self.variant)) + object.__setattr__(self, "digest", _require_image_digest(self.digest)) + + +@dataclass(frozen=True, slots=True) +class AllowlistHit: + """Lookup succeeded: digest is BASE-produced for the requested binding.""" + + record: DigestRecord + + +@dataclass(frozen=True, slots=True) +class AllowlistMiss: + """Lookup failed with a distinct machine reason.""" + + reason: AllowlistMissReason + + +AllowlistLookupResult = AllowlistHit | AllowlistMiss + + +@dataclass(frozen=True, slots=True) +class AllowlistSnapshot: + """Serializable state for DB/file adapters.""" + + records: tuple[DigestRecord, ...] + denied_digests: frozenset[str] + denied_commits: frozenset[str] + + +@dataclass +class DigestAllowlist: + """In-memory digest allowlist with revocation denylists. + + Pure register / lookup / revoke surface intended for prism and BASE build + pipeline callers. Not a network client and not attestation. + """ + + _by_digest: dict[str, DigestRecord] = field(default_factory=dict, init=False) + _denied_digests: set[str] = field(default_factory=set, init=False) + _denied_commits: set[str] = field(default_factory=set, init=False) + + def register(self, record: DigestRecord) -> None: + """Record a BASE-produced digest binding. + + Re-registering the identical binding is a no-op. Binding the same digest + to a different commit/tree/variant raises ``ValueError``. + """ + existing = self._by_digest.get(record.digest) + if existing is not None and existing != record: + raise ValueError( + f"digest {record.digest!r} already bound to " + f"commit={existing.commit_sha} tree={existing.tree_sha} " + f"variant={existing.variant.value}; cannot rebind to " + f"commit={record.commit_sha} tree={record.tree_sha} " + f"variant={record.variant.value}" + ) + self._by_digest[record.digest] = record + + def revoke_digest(self, digest: str) -> None: + """Deny a single image digest permanently (post-hoc revocation).""" + self._denied_digests.add(_require_image_digest(digest)) + + def revoke_commit(self, commit_sha: str) -> None: + """Deny every digest bound to ``commit_sha`` (and future registrations).""" + self._denied_commits.add(_require_full_git_sha("commit_sha", commit_sha)) + + def lookup( + self, + *, + digest: str, + commit_sha: str, + tree_sha: str, + variant: ImageVariant | str, + ) -> AllowlistLookupResult: + """Return hit only when digest matches registered commit, tree, and variant. + + Check order (first match wins): + 1. revoked digest or revoked commit → ``revoked`` + 2. digest absent → ``unknown_digest`` + 3. commit_sha or tree_sha differ → ``commit_mismatch`` + 4. variant differs → ``variant_mismatch`` + 5. else hit + """ + dig = _require_image_digest(digest) + commit = _require_full_git_sha("commit_sha", commit_sha) + tree = _require_full_git_sha("tree_sha", tree_sha) + want_variant = _require_variant(variant) + + if dig in self._denied_digests or commit in self._denied_commits: + return AllowlistMiss(reason=AllowlistMissReason.REVOKED) + + record = self._by_digest.get(dig) + if record is None: + return AllowlistMiss(reason=AllowlistMissReason.UNKNOWN_DIGEST) + + # Also revoke if the *registered* commit is denied (lookup commit may + # already have matched above; this covers digest-only lookups that pass + # a non-denied commit while the binding's commit is denied). + if record.commit_sha in self._denied_commits: + return AllowlistMiss(reason=AllowlistMissReason.REVOKED) + + if record.commit_sha != commit or record.tree_sha != tree: + return AllowlistMiss(reason=AllowlistMissReason.COMMIT_MISMATCH) + + if record.variant is not want_variant: + return AllowlistMiss(reason=AllowlistMissReason.VARIANT_MISMATCH) + + return AllowlistHit(record=record) + + def snapshot(self) -> AllowlistSnapshot: + """Export durable state for persistence adapters.""" + return AllowlistSnapshot( + records=tuple(self._by_digest.values()), + denied_digests=frozenset(self._denied_digests), + denied_commits=frozenset(self._denied_commits), + ) + + @classmethod + def from_snapshot(cls, snapshot: AllowlistSnapshot) -> DigestAllowlist: + """Restore a registry from :meth:`snapshot` output.""" + registry = cls() + for record in snapshot.records: + registry.register(record) + for digest in snapshot.denied_digests: + registry.revoke_digest(digest) + for commit in snapshot.denied_commits: + registry.revoke_commit(commit) + return registry + + def load_records(self, records: Mapping[str, DigestRecord] | None = None) -> None: + """Bulk-load records (used by DB adapters).""" + if not records: + return + for record in records.values(): + self.register(record) diff --git a/src/base/compute/lium.py b/src/base/compute/lium.py index 9dd7b0d09..121945a60 100644 --- a/src/base/compute/lium.py +++ b/src/base/compute/lium.py @@ -19,6 +19,7 @@ import logging from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass from typing import Any import httpx @@ -46,6 +47,64 @@ def __init__(self, message: str, *, status_code: int | None = None) -> None: self.status_code = status_code +class LiumAuthError(LiumError): + """Lium rejected the API key (HTTP 401). Never treat as a missing resource.""" + + +class LiumNotFoundError(LiumError): + """Requested Lium pod or template does not exist (HTTP 404).""" + + +class LiumRateLimitError(LiumError): + """Lium rate-limited the client (HTTP 429). Fail closed — do not skip.""" + + +class LiumTemplateDigestMismatchError(LiumError): + """An existing template name collides with a different docker_image_digest. + + Silent reuse of a name-matched template under a new pin is forbidden: the + caller must create a distinct template or resolve the collision explicitly. + """ + + def __init__( + self, + message: str, + *, + template_id: str, + existing_digest: str | None, + requested_digest: str | None, + ) -> None: + super().__init__(message, status_code=None) + self.template_id = template_id + self.existing_digest = existing_digest + self.requested_digest = requested_digest + + +@dataclass(frozen=True, slots=True) +class LiumPodRead: + """Declared pod configuration from ``GET /pods/{id}`` (OpenAPI PodDetailResponse). + + ``docker_image_digest`` and ``template_id`` come from the nested template + object Lium records at rent time. This is **declared configuration**, not a + runtime measurement of the running container. + """ + + pod_id: str + template_id: str | None + docker_image_digest: str | None + raw: Mapping[str, Any] + + +@dataclass(frozen=True, slots=True) +class LiumTemplateRead: + """Template record from ``GET /templates/{id}`` (OpenAPI Template).""" + + template_id: str + docker_image_digest: str | None + name: str | None + raw: Mapping[str, Any] + + class LiumClient: """Async client for the Lium GPU rental API.""" @@ -159,10 +218,8 @@ async def stream_logs(self, instance_id: str) -> AsyncIterator[str]: async with client.stream("GET", f"/pods/{instance_id}/logs") as response: if response.status_code >= 400: await response.aread() - raise LiumError( - f"Lium GET /pods/{instance_id}/logs returned " - f"{response.status_code}", - status_code=response.status_code, + raise _lium_http_error( + "GET", f"/pods/{instance_id}/logs", response.status_code ) async for line in response.aiter_lines(): yield line @@ -172,9 +229,8 @@ async def terminate(self, instance_id: str) -> None: if response.status_code == 404: return if response.status_code >= 400: - raise LiumError( - f"Lium DELETE /pods/{instance_id} returned {response.status_code}", - status_code=response.status_code, + raise _lium_http_error( + "DELETE", f"/pods/{instance_id}", response.status_code ) async def verify_terminated(self, instance_id: str) -> bool: @@ -187,6 +243,51 @@ async def list_pods(self) -> list[dict[str, Any]]: response = await self._request("GET", "/pods") return _as_list(response.json(), "pods") + async def get_pod_raw(self, pod_id: str) -> LiumPodRead: + """Read one pod via OpenAPI ``GET /pods/{id}``. + + Returns the declared template id and ``docker_image_digest`` from the + nested template object. Raises typed :class:`LiumAuthError` / + :class:`LiumNotFoundError` / :class:`LiumRateLimitError` on 401/404/429. + """ + response = await self._request("GET", f"/pods/{pod_id}") + data = response.json() + if not isinstance(data, Mapping): + raise LiumError("unexpected pod response shape") + template = data.get("template") + template_id: str | None = None + digest: str | None = None + if isinstance(template, Mapping): + if template.get("id"): + template_id = str(template["id"]) + digest = _optional_str(template.get("docker_image_digest")) + if template_id is None and data.get("template_id"): + template_id = str(data["template_id"]) + if digest is None: + digest = _optional_str(data.get("docker_image_digest")) + return LiumPodRead( + pod_id=str(data.get("id") or pod_id), + template_id=template_id, + docker_image_digest=digest, + raw=data, + ) + + async def get_template_raw(self, template_id: str) -> LiumTemplateRead: + """Read one template via OpenAPI ``GET /templates/{id}``.""" + response = await self._request("GET", f"/templates/{template_id}") + data = response.json() + if not isinstance(data, Mapping): + raise LiumError("unexpected template response shape") + tid = data.get("id") + if not tid: + raise LiumError("template response missing 'id'") + return LiumTemplateRead( + template_id=str(tid), + docker_image_digest=_optional_str(data.get("docker_image_digest")), + name=_optional_str(data.get("name")), + raw=data, + ) + # -- idempotent deploy helpers ------------------------------------------- async def ensure_ssh_key( @@ -218,9 +319,22 @@ async def ensure_template( container_start_immediately: bool = True, ) -> str: response = await self._request("GET", "/templates") + requested_digest = _normalize_digest(docker_image_digest) for template in _as_list(response.json(), "templates"): - if str(template.get("name")) == name and template.get("id"): - return str(template["id"]) + if str(template.get("name")) != name or not template.get("id"): + continue + existing_id = str(template["id"]) + existing_digest = _normalize_digest(template.get("docker_image_digest")) + if _digests_allow_reuse(existing_digest, requested_digest): + return existing_id + raise LiumTemplateDigestMismatchError( + f"template {name!r} exists as {existing_id} with " + f"docker_image_digest={existing_digest!r}, refusing reuse for " + f"requested digest {requested_digest!r}", + template_id=existing_id, + existing_digest=existing_digest, + requested_digest=requested_digest, + ) body: dict[str, Any] = { "name": name, "docker_image": docker_image, @@ -403,13 +517,47 @@ async def _request( ) -> httpx.Response: response = await self._send(method, path, json_body=json_body, params=params) if response.status_code >= 400: - raise LiumError( - f"Lium {method} {path} returned {response.status_code}", - status_code=response.status_code, - ) + raise _lium_http_error(method, path, response.status_code) return response +def _lium_http_error(method: str, path: str, status_code: int) -> LiumError: + message = f"Lium {method} {path} returned {status_code}" + if status_code == 401: + return LiumAuthError(message, status_code=401) + if status_code == 404: + return LiumNotFoundError(message, status_code=404) + if status_code == 429: + return LiumRateLimitError(message, status_code=429) + return LiumError(message, status_code=status_code) + + +def _optional_str(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _normalize_digest(value: Any) -> str | None: + return _optional_str(value) + + +def _digests_allow_reuse(existing: str | None, requested: str | None) -> bool: + """Reuse only when digests agree, or neither side pins a digest. + + A requested pin against a missing or different stored digest is a hard + mismatch — never silent name-only reuse under a new pin. + """ + if requested is None and existing is None: + return True + if requested is None: + # Caller did not pin; reusing a digest-bearing template is still name-only + # but does not introduce a false pin claim. Allowed for legacy callers. + return True + return existing is not None and existing == requested + + def _as_list(data: Any, wrapper_key: str) -> list[dict[str, Any]]: if isinstance(data, list): return [item for item in data if isinstance(item, dict)] diff --git a/src/base/db/__init__.py b/src/base/db/__init__.py index 4b1ffa3ba..b549b633d 100644 --- a/src/base/db/__init__.py +++ b/src/base/db/__init__.py @@ -4,6 +4,7 @@ from base.db.models import ( AggregationEpoch, AggregationEpochStatus, + AttestationNonce, Challenge, ChallengeAuth, ChallengeCapability, @@ -16,7 +17,10 @@ ChallengeStatus, ChallengeVolume, ChallengeWatcherState, + DeniedImageCommit, + DeniedImageDigest, FinalWeightVector, + ImageDigestAllowlistEntry, MinerRequestNonce, RawWeightNonce, RawWeightSnapshot, @@ -60,6 +64,10 @@ "ChallengeStatus", "ChallengeVolume", "ChallengeWatcherState", + "AttestationNonce", + "DeniedImageCommit", + "DeniedImageDigest", + "ImageDigestAllowlistEntry", "MinerRequestNonce", "RawWeightNonce", "RawWeightSnapshot", diff --git a/src/base/db/models.py b/src/base/db/models.py index 6b3587809..ed725f430 100644 --- a/src/base/db/models.py +++ b/src/base/db/models.py @@ -1230,3 +1230,90 @@ class ChallengeWatcherState(Base, TimestampMixin): payload: Mapped[dict[str, Any]] = mapped_column( JSON, nullable=False, default=dict, server_default="{}" ) + + +class ImageDigestAllowlistEntry(Base, TimestampMixin): + """One BASE-produced image digest binding for prism miner builds. + + Durable form of :class:`base.compute.digest_allowlist.DigestRecord`. + Lookup rules live in the pure allowlist module; this table is storage only. + A digest maps to exactly one ``(commit_sha, tree_sha, variant)`` binding. + """ + + __tablename__ = "image_digest_allowlist" + __table_args__ = ( + UniqueConstraint("digest", name="uq_image_digest_allowlist_digest"), + UniqueConstraint( + "commit_sha", + "tree_sha", + "variant", + name="uq_image_digest_allowlist_commit_tree_variant", + ), + Index("ix_image_digest_allowlist_commit_sha", "commit_sha"), + Index("ix_image_digest_allowlist_variant", "variant"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + commit_sha: Mapped[str] = mapped_column(Text, nullable=False) + tree_sha: Mapped[str] = mapped_column(Text, nullable=False) + variant: Mapped[str] = mapped_column(Text, nullable=False) + digest: Mapped[str] = mapped_column(Text, nullable=False) + + +class DeniedImageDigest(Base): + """Revoked image digest (post-hoc denylist).""" + + __tablename__ = "denied_image_digests" + + digest: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False) + reason: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ) + + +class DeniedImageCommit(Base): + """Revoked git commit SHA — all digests for this commit miss as revoked.""" + + __tablename__ = "denied_image_commits" + + commit_sha: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False) + reason: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ) + + +class AttestationNonce(Base): + """BASE-issued single-use attestation nonce (prism-lium mechanism 1). + + Durable form of :class:`base.compute.attestation_nonce.NonceRecord`. + Issue/consume rules live in the pure module; this table is storage only. + Freshness uses ``issued_at`` / ``consumed_at`` / ``expires_at`` from BASE + clocks only — never a guest timestamp. + """ + + __tablename__ = "attestation_nonces" + __table_args__ = ( + Index("ix_attestation_nonces_work_unit_id", "work_unit_id"), + Index("ix_attestation_nonces_miner_hotkey", "miner_hotkey"), + Index("ix_attestation_nonces_expires_at", "expires_at"), + ) + + nonce: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False) + work_unit_id: Mapped[str] = mapped_column(Text, nullable=False) + miner_hotkey: Mapped[str] = mapped_column(Text, nullable=False) + pod_id: Mapped[str] = mapped_column(Text, nullable=False) + issued_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + consumed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/src/base/master/app_proxy.py b/src/base/master/app_proxy.py index 28a316ea4..bfbfdd6ae 100644 --- a/src/base/master/app_proxy.py +++ b/src/base/master/app_proxy.py @@ -737,6 +737,7 @@ def create_proxy_app( identity_resolver: ValidatorIdentityResolver | None = None, allowed_cors_origins: list[str] | None = None, readiness_probes: Sequence[ReadinessProbe] = (), + constation_router: Any | None = None, agent_challenge_attested_routes_enabled: bool = False, ) -> FastAPI: """Create the public proxy FastAPI app. @@ -1272,4 +1273,12 @@ def _worker_admission_tokens() -> list[str]: app.state.challenge_registry = challenge_registry app.state.miner_upload_verifier = verifier + + # Internal constation infrastructure only (check_*/register/bundle/issue). + # Public challenge/answer product surface lives on Prism + # (proxy /challenges/prism/...). + if constation_router is not None: + app.include_router(constation_router) + app.state.constation_router = constation_router + return app diff --git a/src/base/master/challenge_work_source.py b/src/base/master/challenge_work_source.py index dd2c3b5c1..feb4a9d8d 100644 --- a/src/base/master/challenge_work_source.py +++ b/src/base/master/challenge_work_source.py @@ -410,11 +410,15 @@ def __init__( timeout_seconds: float = 10.0, retries: int = 3, transport: httpx.AsyncBaseTransport | None = None, + bundle_lookup: Any | None = None, ) -> None: self._registry = registry self._timeout_seconds = timeout_seconds self._retries = retries self._transport = transport + # Optional async callable (work_unit_id) -> dict | None for + # constation_bundle attach. + self._bundle_lookup = bundle_lookup async def forward_result( self, @@ -437,6 +441,12 @@ async def forward_result( "Accept": "application/json", } payload = dict(result_payload or {}) + if "constation_bundle" not in payload and self._bundle_lookup is not None: + looked = self._bundle_lookup(work_unit_id) + if hasattr(looked, "__await__"): + looked = await looked + if isinstance(looked, dict): + payload["constation_bundle"] = looked proof = payload.get("execution_proof") if not isinstance(proof, dict): # Fail closed before any network POST: dual/legacy reduced bodies diff --git a/src/base/master/constation/__init__.py b/src/base/master/constation/__init__.py new file mode 100644 index 000000000..278a86e8c --- /dev/null +++ b/src/base/master/constation/__init__.py @@ -0,0 +1,19 @@ +"""Production constation hosts (durable adapters, HTTP, bundle store).""" + +from __future__ import annotations + +from base.master.constation.allowlist_repository import DigestAllowlistRepository +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.nonce_repository import DurableAttestationNonceService +from base.master.constation.routes import ( + build_constation_router, + create_constation_test_app, +) + +__all__ = [ + "ConstationBundleStore", + "DigestAllowlistRepository", + "DurableAttestationNonceService", + "build_constation_router", + "create_constation_test_app", +] diff --git a/src/base/master/constation/allowlist_repository.py b/src/base/master/constation/allowlist_repository.py new file mode 100644 index 000000000..13b926983 --- /dev/null +++ b/src/base/master/constation/allowlist_repository.py @@ -0,0 +1,152 @@ +"""Durable SQLAlchemy host for DigestAllowlist (mechanism 4 storage). + +Rules remain in base.compute.digest_allowlist; this module is the master +persistence boundary over image_digest_allowlist and deny tables. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from base.compute.digest_allowlist import ( + DigestAllowlist, + DigestRecord, + ImageVariant, + is_full_git_sha, + is_image_digest, + normalize_image_digest, +) +from base.db.models import ( + DeniedImageCommit, + DeniedImageDigest, + ImageDigestAllowlistEntry, +) +from base.db.session import session_scope + + +def _normalize_digest(value: str) -> str: + dig = normalize_image_digest(value) + if not is_image_digest(dig): + raise ValueError(f"digest must be sha256:<64 lowercase hex>, got {value!r}") + return dig + + +def _normalize_commit(value: str) -> str: + commit = value.strip().lower() + if not is_full_git_sha(commit): + raise ValueError( + f"commit_sha must be a full 40-char lowercase hex git SHA, got {value!r}" + ) + return commit + + +class DigestAllowlistRepository: + """Persist and reload BASE-produced image digest bindings.""" + + def __init__( + self, + session_factory: async_sessionmaker[AsyncSession], + *, + session_scope_fn: Callable[..., Any] = session_scope, + ) -> None: + self._session_factory = session_factory + self._session_scope = session_scope_fn + + async def register(self, record: DigestRecord) -> None: + """Insert a binding; identical re-register is a no-op. + + Conflicting rebind (same digest, different commit/tree/variant) raises + ValueError (matches pure DigestAllowlist.register). + """ + async with self._session_scope(self._session_factory) as session: + row = ( + await session.execute( + select(ImageDigestAllowlistEntry).where( + ImageDigestAllowlistEntry.digest == record.digest + ) + ) + ).scalar_one_or_none() + if row is not None: + bound = DigestRecord( + commit_sha=row.commit_sha, + tree_sha=row.tree_sha, + variant=row.variant, + digest=row.digest, + ) + if bound != record: + raise ValueError( + f"digest {record.digest!r} already bound to " + f"commit={bound.commit_sha} tree={bound.tree_sha} " + f"variant={bound.variant.value}; cannot rebind to " + f"commit={record.commit_sha} tree={record.tree_sha} " + f"variant={record.variant.value}" + ) + return + session.add( + ImageDigestAllowlistEntry( + commit_sha=record.commit_sha, + tree_sha=record.tree_sha, + variant=record.variant.value, + digest=record.digest, + ) + ) + + async def revoke_digest(self, digest: str, *, reason: str | None = None) -> None: + """Add durable digest deny entry.""" + dig = _normalize_digest(digest) + async with self._session_scope(self._session_factory) as session: + existing = await session.get(DeniedImageDigest, dig) + if existing is None: + session.add(DeniedImageDigest(digest=dig, reason=reason)) + elif reason is not None and existing.reason is None: + existing.reason = reason + + async def revoke_commit( + self, commit_sha: str, *, reason: str | None = None + ) -> None: + """Add durable commit deny entry.""" + commit = _normalize_commit(commit_sha) + async with self._session_scope(self._session_factory) as session: + existing = await session.get(DeniedImageCommit, commit) + if existing is None: + session.add(DeniedImageCommit(commit_sha=commit, reason=reason)) + elif reason is not None and existing.reason is None: + existing.reason = reason + + async def load_allowlist(self) -> DigestAllowlist: + """Materialize a pure in-memory allowlist from durable tables.""" + async with self._session_scope(self._session_factory) as session: + entries = ( + (await session.execute(select(ImageDigestAllowlistEntry))) + .scalars() + .all() + ) + denied_d = ( + (await session.execute(select(DeniedImageDigest))).scalars().all() + ) + denied_c = ( + (await session.execute(select(DeniedImageCommit))).scalars().all() + ) + + allowlist = DigestAllowlist() + for row in entries: + allowlist.register( + DigestRecord( + commit_sha=row.commit_sha, + tree_sha=row.tree_sha, + variant=ImageVariant(row.variant), + digest=row.digest, + ) + ) + for row in denied_d: + allowlist.revoke_digest(row.digest) + for row in denied_c: + allowlist.revoke_commit(row.commit_sha) + return allowlist + + +__all__ = ["DigestAllowlistRepository"] diff --git a/src/base/master/constation/bundle_store.py b/src/base/master/constation/bundle_store.py new file mode 100644 index 000000000..f5fa1c291 --- /dev/null +++ b/src/base/master/constation/bundle_store.py @@ -0,0 +1,34 @@ +"""In-memory + optional durable store for sealed ConstationBundle wire dicts.""" + +from __future__ import annotations + +import copy +from typing import Any + + +class ConstationBundleStore: + """Keyed by work_unit_id; last-write wins. Forwarder attaches stored bundles.""" + + def __init__(self) -> None: + self._by_wu: dict[str, dict[str, Any]] = {} + + def put(self, work_unit_id: str, bundle: dict[str, Any]) -> None: + key = work_unit_id.strip() + if not key: + raise ValueError("work_unit_id must be non-empty") + if not isinstance(bundle, dict): + raise TypeError("bundle must be a dict") + self._by_wu[key] = copy.deepcopy(bundle) + + def get(self, work_unit_id: str) -> dict[str, Any] | None: + key = work_unit_id.strip() + blob = self._by_wu.get(key) + return copy.deepcopy(blob) if blob is not None else None + + def pop(self, work_unit_id: str) -> dict[str, Any] | None: + key = work_unit_id.strip() + blob = self._by_wu.pop(key, None) + return copy.deepcopy(blob) if blob is not None else None + + +__all__ = ["ConstationBundleStore"] diff --git a/src/base/master/constation/nonce_repository.py b/src/base/master/constation/nonce_repository.py new file mode 100644 index 000000000..119adaf87 --- /dev/null +++ b/src/base/master/constation/nonce_repository.py @@ -0,0 +1,162 @@ +"""Durable SQLAlchemy host for attestation nonces (mechanism 1 storage). + +Issue/consume *rules* match +:class:`base.compute.attestation_nonce.AttestationNonceService`. +Consume uses ``UPDATE … WHERE consumed_at IS NULL`` so multi-worker races still +yield exactly one HIT (S4). +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from base.compute.attestation_nonce import ( + NonceBinding, + NonceConsumeHit, + NonceConsumeMiss, + NonceConsumeReason, + NonceConsumeResult, + NonceRecord, +) +from base.db.models import AttestationNonce +from base.db.session import session_scope + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +def _as_aware(dt: datetime) -> datetime: + """SQLite often returns naive UTC; normalize for comparisons.""" + if dt.tzinfo is None: + return dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +class DurableAttestationNonceService: + """BASE-clock nonce issuer/consumer backed by ``attestation_nonces``.""" + + def __init__( + self, + session_factory: async_sessionmaker[AsyncSession], + *, + ttl: timedelta, + now_fn: Callable[[], datetime] = _utc_now, + session_scope_fn: Callable[..., Any] = session_scope, + ) -> None: + if ttl <= timedelta(0): + raise ValueError(f"ttl must be positive, got {ttl!r}") + self._session_factory = session_factory + self._ttl = ttl + self._now_fn = now_fn + self._session_scope = session_scope_fn + + async def issue(self, binding: NonceBinding) -> NonceRecord: + """Issue a fresh UUID nonce bound to ``binding`` at BASE ``now_fn`` time.""" + bound = NonceBinding( + work_unit_id=binding.work_unit_id, + miner_hotkey=binding.miner_hotkey, + pod_id=binding.pod_id, + ) + issued_at = self._now_fn() + if issued_at.tzinfo is None: + raise ValueError("BASE now_fn must return timezone-aware datetime") + record = NonceRecord( + nonce=str(uuid.uuid4()), + binding=bound, + issued_at=issued_at, + expires_at=issued_at + self._ttl, + consumed_at=None, + ) + async with self._session_scope(self._session_factory) as session: + session.add( + AttestationNonce( + nonce=record.nonce, + work_unit_id=bound.work_unit_id, + miner_hotkey=bound.miner_hotkey, + pod_id=bound.pod_id, + issued_at=record.issued_at, + expires_at=record.expires_at, + consumed_at=None, + ) + ) + return record + + async def consume( + self, + nonce: str, + binding: NonceBinding, + *, + received_at: datetime | None = None, + ) -> NonceConsumeResult: + """Consume ``nonce`` once if unexpired and binding matches. + + Same check order as the pure in-memory service. Atomic claim via + ``UPDATE … WHERE consumed_at IS NULL``. + """ + want = NonceBinding( + work_unit_id=binding.work_unit_id, + miner_hotkey=binding.miner_hotkey, + pod_id=binding.pod_id, + ) + receive = received_at if received_at is not None else self._now_fn() + if receive.tzinfo is None: + raise ValueError("BASE received_at must be timezone-aware") + + key = nonce.strip() + async with self._session_scope(self._session_factory) as session: + row = ( + await session.execute( + select(AttestationNonce).where(AttestationNonce.nonce == key) + ) + ).scalar_one_or_none() + if row is None: + return NonceConsumeMiss(reason=NonceConsumeReason.UNKNOWN_NONCE) + + if row.consumed_at is not None: + return NonceConsumeMiss(reason=NonceConsumeReason.ALREADY_CONSUMED) + + if receive > _as_aware(row.expires_at): + return NonceConsumeMiss(reason=NonceConsumeReason.EXPIRED) + + if row.work_unit_id != want.work_unit_id: + return NonceConsumeMiss(reason=NonceConsumeReason.WORK_UNIT_MISMATCH) + if row.miner_hotkey != want.miner_hotkey: + return NonceConsumeMiss(reason=NonceConsumeReason.MINER_HOTKEY_MISMATCH) + if row.pod_id != want.pod_id: + return NonceConsumeMiss(reason=NonceConsumeReason.POD_MISMATCH) + + # Atomic single-use claim. + result = await session.execute( + update(AttestationNonce) + .where( + AttestationNonce.nonce == key, + AttestationNonce.consumed_at.is_(None), + ) + .values(consumed_at=receive) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + # Lost race after passes — treat as already consumed. + return NonceConsumeMiss(reason=NonceConsumeReason.ALREADY_CONSUMED) + + consumed = NonceRecord( + nonce=row.nonce, + binding=NonceBinding( + work_unit_id=row.work_unit_id, + miner_hotkey=row.miner_hotkey, + pod_id=row.pod_id, + ), + issued_at=_as_aware(row.issued_at), + expires_at=_as_aware(row.expires_at), + consumed_at=receive, + ) + return NonceConsumeHit(record=consumed, received_at=receive) + + +__all__ = ["DurableAttestationNonceService"] diff --git a/src/base/master/constation/routes.py b/src/base/master/constation/routes.py new file mode 100644 index 000000000..08f29c8b5 --- /dev/null +++ b/src/base/master/constation/routes.py @@ -0,0 +1,336 @@ +"""HTTP surfaces for attestation challenge/answer and constation checkers. + +Sidecar convention (prism-recipe HttpxChallengeTransport): + GET /v1/attestation/challenge?phase= + POST /v1/attestation/answer + +Internal (prism re-verify / ops): + POST /internal/v1/constation/check_allowlist + POST /internal/v1/constation/check_nonce + POST /internal/v1/constation/register_digest + POST /internal/v1/constation/verify_attestation + PUT /internal/v1/constation/bundle/{work_unit_id} + GET /internal/v1/constation/bundle/{work_unit_id} +""" + +from __future__ import annotations + +import hmac +from typing import Any + +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status +from pydantic import BaseModel, Field + +from base.attestation.payload import ( + AttestationPayload, + AttestationVerifyReason, + SignedAttestation, + verify_attestation_payload, +) +from base.compute.attestation_nonce import NonceBinding, NonceConsumeHit +from base.compute.digest_allowlist import AllowlistHit, DigestRecord, ImageVariant +from base.master.constation.allowlist_repository import DigestAllowlistRepository +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.nonce_repository import DurableAttestationNonceService + + +class RegisterDigestBody(BaseModel): + commit_sha: str + tree_sha: str + variant: str + digest: str + + +class CheckAllowlistBody(BaseModel): + digest: str + commit_sha: str + tree_sha: str + variant: str + + +class CheckNonceBody(BaseModel): + nonce: str + work_unit_id: str + miner_hotkey: str + pod_id: str + + +class IssueNonceBody(BaseModel): + work_unit_id: str + miner_hotkey: str + pod_id: str + phase: str = "interval" + + +class VerifyAttestationBody(BaseModel): + signed: dict[str, Any] + key_hex: str | None = None + + +class AnswerBody(BaseModel): + model_config = {"extra": "allow"} + + nonce: str = Field(min_length=1) + phase: str | None = None + + +def _bearer_ok(authorization: str | None, expected: str | None) -> bool: + if not expected: + return True + if not authorization or not authorization.lower().startswith("bearer "): + return False + presented = authorization.split(" ", 1)[1].strip() + return bool(presented) and hmac.compare_digest(presented, expected) + + +def _signed_from_wire(raw: dict[str, Any]) -> SignedAttestation: + """Parse a wire/json signed attestation into the domain object.""" + if "payload" in raw and isinstance(raw["payload"], dict): + pl_raw = raw["payload"] + sig = raw.get("signature") + alg = raw.get("algorithm", "hmac-sha256") + schema = raw.get("schema_version", "prism_attestation_payload.v1") + else: + pl_raw = raw + sig = raw.get("signature") + alg = raw.get("algorithm", "hmac-sha256") + schema = raw.get("schema_version", "prism_attestation_payload.v1") + if not isinstance(sig, str): + raise ValueError("signature must be a hex string") + payload = AttestationPayload( + nonce=str(pl_raw["nonce"]), + digest=str(pl_raw["digest"]), + pod_id=str(pl_raw["pod_id"]), + variant=pl_raw["variant"], # type: ignore[arg-type] + sealed_manifest_hashes=dict(pl_raw["sealed_manifest_hashes"]), + build_secret_response=str(pl_raw["build_secret_response"]), + ) + return SignedAttestation( + payload=payload, + signature=sig, + algorithm=str(alg), + schema_version=str(schema), + ) + + +def build_constation_router( + *, + allowlist_repo: DigestAllowlistRepository, + nonce_service: DurableAttestationNonceService, + bundle_store: ConstationBundleStore | None = None, + internal_token: str | None = None, + attestation_verify_key: bytes | None = None, + default_binding: NonceBinding | None = None, +) -> APIRouter: + """Build router; require Bearer when ``internal_token`` is set.""" + store = bundle_store or ConstationBundleStore() + answers: list[dict[str, Any]] = [] + router = APIRouter(tags=["constation"]) + + def require_internal( + authorization: str | None = Header(default=None), + ) -> None: + if not _bearer_ok(authorization, internal_token): + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, detail="invalid internal token" + ) + + @router.get("/v1/attestation/challenge") + async def attestation_challenge( + phase: str = Query(default="interval"), + work_unit_id: str | None = Query(default=None), + miner_hotkey: str | None = Query(default=None), + pod_id: str | None = Query(default=None), + ) -> dict[str, Any]: + if default_binding is not None: + binding = default_binding + else: + if not (work_unit_id and miner_hotkey and pod_id): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=( + "work_unit_id, miner_hotkey, and pod_id query params " + "required when no default binding configured" + ), + ) + binding = NonceBinding( + work_unit_id=work_unit_id, + miner_hotkey=miner_hotkey, + pod_id=pod_id, + ) + phase_key = phase.strip().lower() + if phase_key in {"random", "mid"}: + phase_key = "interval" + if phase_key not in {"start", "interval", "end"}: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"unknown challenge phase: {phase!r}", + ) + record = await nonce_service.issue(binding) + return { + "nonce": record.nonce, + "phase": phase_key, + "challenge_id": record.nonce, + "work_unit_id": binding.work_unit_id, + "expires_at": record.expires_at.isoformat(), + } + + @router.post("/v1/attestation/answer") + async def attestation_answer(body: AnswerBody) -> dict[str, str]: + answers.append(body.model_dump()) + return {"status": "accepted"} + + @router.post( + "/internal/v1/constation/issue_nonce", + dependencies=[Depends(require_internal)], + ) + async def issue_nonce(body: IssueNonceBody) -> dict[str, Any]: + """Issue a durable nonce for Prism public challenge (SoT on master).""" + phase_key = body.phase.strip().lower() + if phase_key in {"random", "mid"}: + phase_key = "interval" + if phase_key not in {"start", "interval", "end"}: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"unknown challenge phase: {body.phase!r}", + ) + binding = NonceBinding( + work_unit_id=body.work_unit_id, + miner_hotkey=body.miner_hotkey, + pod_id=body.pod_id, + ) + record = await nonce_service.issue(binding) + return { + "nonce": record.nonce, + "phase": phase_key, + "challenge_id": record.nonce, + "work_unit_id": binding.work_unit_id, + "expires_at": record.expires_at.isoformat(), + } + + @router.post( + "/internal/v1/constation/register_digest", + dependencies=[Depends(require_internal)], + ) + async def register_digest(body: RegisterDigestBody) -> dict[str, str]: + record = DigestRecord( + commit_sha=body.commit_sha, + tree_sha=body.tree_sha, + variant=ImageVariant(body.variant.strip().lower()), + digest=body.digest, + ) + await allowlist_repo.register(record) + return {"status": "registered", "digest": record.digest} + + @router.post( + "/internal/v1/constation/check_allowlist", + dependencies=[Depends(require_internal)], + ) + async def check_allowlist(body: CheckAllowlistBody) -> dict[str, Any]: + allowlist = await allowlist_repo.load_allowlist() + result = allowlist.lookup( + digest=body.digest, + commit_sha=body.commit_sha, + tree_sha=body.tree_sha, + variant=body.variant, + ) + if isinstance(result, AllowlistHit): + return {"ok": True, "reason": "ok"} + return {"ok": False, "reason": result.reason.value} + + @router.post( + "/internal/v1/constation/check_nonce", + dependencies=[Depends(require_internal)], + ) + async def check_nonce(body: CheckNonceBody) -> dict[str, Any]: + result = await nonce_service.consume( + body.nonce, + NonceBinding( + work_unit_id=body.work_unit_id, + miner_hotkey=body.miner_hotkey, + pod_id=body.pod_id, + ), + ) + if isinstance(result, NonceConsumeHit): + return {"ok": True, "reason": "ok"} + return {"ok": False, "reason": result.reason.value} + + @router.post( + "/internal/v1/constation/verify_attestation", + dependencies=[Depends(require_internal)], + ) + async def verify_attestation(body: VerifyAttestationBody) -> dict[str, Any]: + if body.key_hex: + key = bytes.fromhex(body.key_hex) + elif attestation_verify_key is not None: + key = attestation_verify_key + else: + return {"ok": False, "reason": AttestationVerifyReason.EMPTY_KEY.value} + try: + signed = _signed_from_wire(body.signed) + except (TypeError, ValueError, KeyError) as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"invalid signed attestation: {exc}", + ) from exc + outcome = verify_attestation_payload(signed, verify_key=key) + return {"ok": outcome.ok, "reason": outcome.reason.value} + + @router.put( + "/internal/v1/constation/bundle/{work_unit_id}", + dependencies=[Depends(require_internal)], + ) + async def put_bundle(work_unit_id: str, request: Request) -> dict[str, str]: + payload = await request.json() + if not isinstance(payload, dict): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, detail="bundle must be object" + ) + store.put(work_unit_id, payload) + return {"status": "stored", "work_unit_id": work_unit_id} + + @router.get( + "/internal/v1/constation/bundle/{work_unit_id}", + dependencies=[Depends(require_internal)], + ) + async def get_bundle(work_unit_id: str) -> dict[str, Any]: + blob = store.get(work_unit_id) + if blob is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, detail="bundle not found") + return blob + + router._answers = answers # type: ignore[attr-defined] + router._bundle_store = store # type: ignore[attr-defined] + return router + + +def create_constation_test_app( + *, + allowlist_repo: DigestAllowlistRepository, + nonce_service: DurableAttestationNonceService, + internal_token: str | None = "test-internal", + default_binding: NonceBinding | None = None, + attestation_verify_key: bytes | None = None, + bundle_store: ConstationBundleStore | None = None, +) -> Any: + """Minimal FastAPI app hosting only the constation router (unit tests).""" + from fastapi import FastAPI + + app = FastAPI() + router = build_constation_router( + allowlist_repo=allowlist_repo, + nonce_service=nonce_service, + bundle_store=bundle_store, + internal_token=internal_token, + default_binding=default_binding, + attestation_verify_key=attestation_verify_key, + ) + app.include_router(router) + app.state.constation_router = router + return app + + +__all__ = [ + "build_constation_router", + "create_constation_test_app", +] diff --git a/tests/unit/test_attestation_http.py b/tests/unit/test_attestation_http.py new file mode 100644 index 000000000..a6b63316e --- /dev/null +++ b/tests/unit/test_attestation_http.py @@ -0,0 +1,216 @@ +"""S8 + checker HTTP surfaces for production constation.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from base.attestation.payload import ( + AttestationPayload, + derive_attestation_key, + sign_attestation_payload, +) +from base.compute.attestation_nonce import NonceBinding +from base.db.models import Base +from base.master.constation.allowlist_repository import DigestAllowlistRepository +from base.master.constation.bundle_store import ConstationBundleStore +from base.master.constation.nonce_repository import DurableAttestationNonceService +from base.master.constation.routes import create_constation_test_app + +TOKEN = "test-internal" +BINDING = NonceBinding(work_unit_id="wu-http", miner_hotkey="hk-1", pod_id="pod-1") +COMMIT = "a" * 40 +TREE = "b" * 40 +DIGEST = "sha256:" + ("c" * 64) +MANIFEST = {"harness.py": "d" * 64} + + +@pytest.fixture +async def harness(tmp_path: Path) -> AsyncIterator[dict[str, Any]]: + db_path = tmp_path / "constation_http.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + allowlist_repo = DigestAllowlistRepository(factory) + nonce_svc = DurableAttestationNonceService(factory, ttl=timedelta(hours=1)) + store = ConstationBundleStore() + key = derive_attestation_key(b"build-secret-fixture") + app = create_constation_test_app( + allowlist_repo=allowlist_repo, + nonce_service=nonce_svc, + internal_token=TOKEN, + default_binding=BINDING, + attestation_verify_key=key, + bundle_store=store, + ) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield { + "client": client, + "headers": {"Authorization": f"Bearer {TOKEN}"}, + "key": key, + "store": store, + } + await engine.dispose() + + +@pytest.mark.asyncio +async def test_challenge_answer_roundtrip_s8(harness: dict[str, Any]) -> None: + client: AsyncClient = harness["client"] + r = await client.get("/v1/attestation/challenge", params={"phase": "start"}) + assert r.status_code == 200, r.text + body = r.json() + assert body["nonce"] + assert body["phase"] == "start" + # parse_challenge-compatible shape + assert "nonce" in body and "phase" in body + + ans = await client.post( + "/v1/attestation/answer", + json={"nonce": body["nonce"], "phase": "start", "sig": "xx"}, + ) + assert ans.status_code == 200 + assert ans.json()["status"] == "accepted" + + +@pytest.mark.asyncio +async def test_register_and_check_allowlist(harness: dict[str, Any]) -> None: + client = harness["client"] + headers = harness["headers"] + reg = await client.post( + "/internal/v1/constation/register_digest", + headers=headers, + json={ + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": "cuda", + "digest": DIGEST, + }, + ) + assert reg.status_code == 200, reg.text + + hit = await client.post( + "/internal/v1/constation/check_allowlist", + headers=headers, + json={ + "digest": DIGEST, + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": "cuda", + }, + ) + assert hit.status_code == 200 + assert hit.json() == {"ok": True, "reason": "ok"} + + miss = await client.post( + "/internal/v1/constation/check_allowlist", + headers=headers, + json={ + "digest": "sha256:" + ("f" * 64), + "commit_sha": COMMIT, + "tree_sha": TREE, + "variant": "cuda", + }, + ) + assert miss.json()["ok"] is False + assert miss.json()["reason"] == "unknown_digest" + + +@pytest.mark.asyncio +async def test_check_nonce_consume_and_replay(harness: dict[str, Any]) -> None: + client = harness["client"] + headers = harness["headers"] + ch = await client.get("/v1/attestation/challenge", params={"phase": "interval"}) + nonce = ch.json()["nonce"] + body = { + "nonce": nonce, + "work_unit_id": BINDING.work_unit_id, + "miner_hotkey": BINDING.miner_hotkey, + "pod_id": BINDING.pod_id, + } + first = await client.post( + "/internal/v1/constation/check_nonce", headers=headers, json=body + ) + second = await client.post( + "/internal/v1/constation/check_nonce", headers=headers, json=body + ) + assert first.json() == {"ok": True, "reason": "ok"} + assert second.json()["ok"] is False + assert second.json()["reason"] == "already_consumed" + + +@pytest.mark.asyncio +async def test_bundle_put_get(harness: dict[str, Any]) -> None: + client = harness["client"] + headers = harness["headers"] + bundle = {"digest": DIGEST, "nonce": "n1", "work_unit_id": "wu-http"} + put = await client.put( + "/internal/v1/constation/bundle/wu-http", headers=headers, json=bundle + ) + assert put.status_code == 200 + got = await client.get("/internal/v1/constation/bundle/wu-http", headers=headers) + assert got.status_code == 200 + assert got.json()["digest"] == DIGEST + + +@pytest.mark.asyncio +async def test_verify_attestation_ok(harness: dict[str, Any]) -> None: + client = harness["client"] + headers = harness["headers"] + key: bytes = harness["key"] + payload = AttestationPayload( + nonce="nonce-1", + digest=DIGEST, + pod_id=BINDING.pod_id, + variant="cuda", + sealed_manifest_hashes=dict(MANIFEST), + build_secret_response="a" * 64, + ) + # build_secret_response must be valid hex from real helper for structure; + # sign with derived key after computing real response. + from base.attestation.payload import compute_build_secret_response + + secret = b"build-secret-fixture" + key = derive_attestation_key(secret) + payload = AttestationPayload( + nonce="nonce-1", + digest=DIGEST, + pod_id=BINDING.pod_id, + variant="cuda", + sealed_manifest_hashes=dict(MANIFEST), + build_secret_response=compute_build_secret_response( + build_secret=secret, nonce="nonce-1" + ), + ) + signed = sign_attestation_payload(payload, signing_key=key) + wire = { + "payload": { + "nonce": payload.nonce, + "digest": payload.digest, + "pod_id": payload.pod_id, + "variant": payload.variant, + "sealed_manifest_hashes": dict(payload.sealed_manifest_hashes), + "build_secret_response": payload.build_secret_response, + }, + "signature": signed.signature, + "algorithm": signed.algorithm, + "schema_version": signed.schema_version, + } + r = await client.post( + "/internal/v1/constation/verify_attestation", + headers=headers, + json={"signed": wire}, + ) + assert r.status_code == 200, r.text + assert r.json()["ok"] is True diff --git a/tests/unit/test_attestation_nonce_repository.py b/tests/unit/test_attestation_nonce_repository.py new file mode 100644 index 000000000..40fad92ef --- /dev/null +++ b/tests/unit/test_attestation_nonce_repository.py @@ -0,0 +1,163 @@ +"""TDD: durable SQLAlchemy adapter for AttestationNonceService (production host). + +Pure consume order stays in ``base.compute.attestation_nonce``; this repository +issues and consumes against ``attestation_nonces`` with atomic SQL consume for S4. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from base.compute.attestation_nonce import ( + NonceBinding, + NonceConsumeHit, + NonceConsumeMiss, + NonceConsumeReason, +) +from base.db.models import Base +from base.master.constation.nonce_repository import DurableAttestationNonceService + +BINDING = NonceBinding( + work_unit_id="wu-1", + miner_hotkey="hk-miner", + pod_id="pod-abc", +) +OTHER = NonceBinding( + work_unit_id="wu-2", + miner_hotkey="hk-miner", + pod_id="pod-abc", +) + + +class _Clock: + def __init__(self, start: datetime) -> None: + self._now = start + + def now(self) -> datetime: + return self._now + + def advance(self, delta: timedelta) -> None: + self._now = self._now + delta + + +@pytest.fixture +async def session_factory( + tmp_path: Path, +) -> AsyncIterator[async_sessionmaker[AsyncSession]]: + db_path = tmp_path / "nonces.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + try: + yield factory + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_issue_persists_across_service_instances( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + clock = _Clock(datetime(2026, 7, 26, 12, 0, tzinfo=UTC)) + svc = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + record = await svc.issue(BINDING) + + other = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + result = await other.consume(record.nonce, BINDING) + assert isinstance(result, NonceConsumeHit) + assert result.record.nonce == record.nonce + assert result.record.binding == BINDING + + +@pytest.mark.asyncio +async def test_second_consume_is_already_consumed_s4( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """S4 nonce replay after successful consume.""" + clock = _Clock(datetime(2026, 7, 26, 12, 0, tzinfo=UTC)) + svc = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + record = await svc.issue(BINDING) + first = await svc.consume(record.nonce, BINDING) + second = await svc.consume(record.nonce, BINDING) + assert isinstance(first, NonceConsumeHit) + assert isinstance(second, NonceConsumeMiss) + assert second.reason is NonceConsumeReason.ALREADY_CONSUMED + + +@pytest.mark.asyncio +async def test_unknown_nonce( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + clock = _Clock(datetime(2026, 7, 26, 12, 0, tzinfo=UTC)) + svc = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + result = await svc.consume("00000000-0000-0000-0000-000000000000", BINDING) + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.UNKNOWN_NONCE + + +@pytest.mark.asyncio +async def test_expired_nonce( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + clock = _Clock(datetime(2026, 7, 26, 12, 0, tzinfo=UTC)) + svc = DurableAttestationNonceService( + session_factory, ttl=timedelta(seconds=30), now_fn=clock.now + ) + record = await svc.issue(BINDING) + clock.advance(timedelta(seconds=60)) + result = await svc.consume(record.nonce, BINDING) + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.EXPIRED + + +@pytest.mark.asyncio +async def test_work_unit_mismatch( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + clock = _Clock(datetime(2026, 7, 26, 12, 0, tzinfo=UTC)) + svc = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + record = await svc.issue(BINDING) + result = await svc.consume(record.nonce, OTHER) + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.WORK_UNIT_MISMATCH + + +@pytest.mark.asyncio +async def test_atomic_double_consume_from_two_handles( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Two durable services race: only one HIT, one ALREADY_CONSUMED.""" + clock = _Clock(datetime(2026, 7, 26, 12, 0, tzinfo=UTC)) + a = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + b = DurableAttestationNonceService( + session_factory, ttl=timedelta(hours=1), now_fn=clock.now + ) + record = await a.issue(BINDING) + r1 = await a.consume(record.nonce, BINDING) + r2 = await b.consume(record.nonce, BINDING) + hits = sum(1 for r in (r1, r2) if isinstance(r, NonceConsumeHit)) + misses = [r for r in (r1, r2) if isinstance(r, NonceConsumeMiss)] + assert hits == 1 + assert len(misses) == 1 + assert misses[0].reason is NonceConsumeReason.ALREADY_CONSUMED diff --git a/tests/unit/test_attestation_payload.py b/tests/unit/test_attestation_payload.py new file mode 100644 index 000000000..a92db1bce --- /dev/null +++ b/tests/unit/test_attestation_payload.py @@ -0,0 +1,229 @@ +"""TDD tests for attestation payload schema + HMAC-SHA256 verification. + +Checkbox 9 (prism-lium-image-attestation): sidecar signs +(nonce, digest, pod_id, variant, sealed_manifest_hashes, build_secret_response). +BASE holds the verify key. A valid signature proves only that an entity holding +the in-image secret responded — not a hardware root of trust, and never +sufficient alone for tier elevation (B3). +""" + +from __future__ import annotations + +import hashlib +import hmac +from typing import Any + +import pytest + +from base.attestation.payload import ( + ALGORITHM, + SCHEMA_VERSION, + AttestationPayload, + AttestationVerifyError, + AttestationVerifyReason, + SignedAttestation, + canonical_payload_bytes, + derive_attestation_key, + sign_attestation_payload, + verify_attestation_payload, +) + +# Fixed test vectors — must match prism-recipe/tests/test_attestation_payload.py +NONCE = "550e8400-e29b-41d4-a716-446655440000" +DIGEST = "sha256:" + ("ab" * 32) +POD_ID = "pod_test_001" +VARIANT = "cuda" +MANIFEST_HASHES = { + "src/prism_recipe/harness.py": "a" * 64, + "src/prism_recipe/gpu_train.py": "b" * 64, +} +BUILD_SECRET = b"unit-test-build-secret-not-for-prod" +BUILD_SECRET_RESPONSE = hmac.new( + BUILD_SECRET, NONCE.encode("utf-8"), hashlib.sha256 +).hexdigest() +VERIFY_KEY = derive_attestation_key(BUILD_SECRET) + + +def _payload(**overrides: Any) -> AttestationPayload: + fields: dict[str, Any] = { + "nonce": NONCE, + "digest": DIGEST, + "pod_id": POD_ID, + "variant": VARIANT, + "sealed_manifest_hashes": dict(MANIFEST_HASHES), + "build_secret_response": BUILD_SECRET_RESPONSE, + } + fields.update(overrides) + return AttestationPayload(**fields) + + +def test_well_formed_signed_payload_verifies() -> None: + """S1 happy: Given signed payload, When BASE verifies with key, Then ok.""" + payload = _payload() + signed = sign_attestation_payload(payload, signing_key=VERIFY_KEY) + + result = verify_attestation_payload(signed, verify_key=VERIFY_KEY) + + assert result.ok is True + assert result.reason is AttestationVerifyReason.OK + assert result.payload == payload + assert signed.algorithm == ALGORITHM + assert signed.schema_version == SCHEMA_VERSION + assert len(signed.signature) == 64 # sha256 hex + + +def test_tamper_digest_one_byte_rejected() -> None: + """S2: Flip one byte of digest after sign → verify fails.""" + signed = sign_attestation_payload(_payload(), signing_key=VERIFY_KEY) + tampered_digest = "sha256:" + ("ac" + "ab" * 31) # first byte flipped ab→ac + assert tampered_digest != DIGEST + tampered = SignedAttestation( + payload=_payload(digest=tampered_digest), + signature=signed.signature, + algorithm=signed.algorithm, + schema_version=signed.schema_version, + ) + + result = verify_attestation_payload(tampered, verify_key=VERIFY_KEY) + + assert result.ok is False + assert result.reason is AttestationVerifyReason.SIGNATURE_MISMATCH + + +def test_tamper_sealed_manifest_hashes_rejected() -> None: + """S2b: Flip one byte in a sealed manifest hash → verify fails.""" + signed = sign_attestation_payload(_payload(), signing_key=VERIFY_KEY) + bad_hashes = dict(MANIFEST_HASHES) + original = bad_hashes["src/prism_recipe/harness.py"] + flipped = "0" if original[0] != "0" else "1" + bad_hashes["src/prism_recipe/harness.py"] = flipped + original[1:] + tampered = SignedAttestation( + payload=_payload(sealed_manifest_hashes=bad_hashes), + signature=signed.signature, + algorithm=signed.algorithm, + schema_version=signed.schema_version, + ) + + result = verify_attestation_payload(tampered, verify_key=VERIFY_KEY) + + assert result.ok is False + assert result.reason is AttestationVerifyReason.SIGNATURE_MISMATCH + + +@pytest.mark.parametrize( + "field_name,override", + [ + ("nonce", {"nonce": "00000000-0000-4000-8000-000000000000"}), + ("pod_id", {"pod_id": "pod_other"}), + ("variant", {"variant": "cpu"}), + ( + "build_secret_response", + {"build_secret_response": "ff" * 32}, + ), + ], +) +def test_tamper_any_field_rejected(field_name: str, override: dict[str, Any]) -> None: + """Any single-field alteration after sign must fail verification.""" + signed = sign_attestation_payload(_payload(), signing_key=VERIFY_KEY) + tampered = SignedAttestation( + payload=_payload(**override), + signature=signed.signature, + algorithm=signed.algorithm, + schema_version=signed.schema_version, + ) + + result = verify_attestation_payload(tampered, verify_key=VERIFY_KEY) + + assert result.ok is False, f"tamper of {field_name} must fail" + assert result.reason is AttestationVerifyReason.SIGNATURE_MISMATCH + + +def test_wrong_verify_key_rejected() -> None: + signed = sign_attestation_payload(_payload(), signing_key=VERIFY_KEY) + other_key = derive_attestation_key(b"different-secret") + + result = verify_attestation_payload(signed, verify_key=other_key) + + assert result.ok is False + assert result.reason is AttestationVerifyReason.SIGNATURE_MISMATCH + + +def test_canonical_bytes_are_deterministic_and_order_independent() -> None: + """Manifest hash map order must not affect the signed bytes.""" + p1 = _payload( + sealed_manifest_hashes={ + "src/prism_recipe/harness.py": "a" * 64, + "src/prism_recipe/gpu_train.py": "b" * 64, + } + ) + p2 = _payload( + sealed_manifest_hashes={ + "src/prism_recipe/gpu_train.py": "b" * 64, + "src/prism_recipe/harness.py": "a" * 64, + } + ) + assert canonical_payload_bytes(p1) == canonical_payload_bytes(p2) + + +def test_derive_attestation_key_is_stable() -> None: + k1 = derive_attestation_key(BUILD_SECRET) + k2 = derive_attestation_key(BUILD_SECRET) + assert k1 == k2 + assert len(k1) == 32 + assert k1 != BUILD_SECRET # derived, not raw secret + + +def test_signature_never_grants_tier_api_absent() -> None: + """B3: module must not expose tier elevation from signature alone.""" + import base.attestation.payload as mod + + forbidden = ( + "effective_tier", + "grant_tier", + "elevate_tier", + "tier_from_signature", + "constation_ok", + ) + public = {n for n in dir(mod) if not n.startswith("_")} + for name in forbidden: + assert name not in public, f"forbidden tier API leaked: {name}" + + +def test_verify_raises_optional_strict_mode() -> None: + signed = sign_attestation_payload(_payload(), signing_key=VERIFY_KEY) + bad = SignedAttestation( + payload=_payload(digest="sha256:" + ("00" * 32)), + signature=signed.signature, + algorithm=signed.algorithm, + schema_version=signed.schema_version, + ) + with pytest.raises(AttestationVerifyError) as excinfo: + verify_attestation_payload(bad, verify_key=VERIFY_KEY, raise_on_failure=True) + assert excinfo.value.reason is AttestationVerifyReason.SIGNATURE_MISMATCH + + +def test_docstring_states_not_hardware_root_and_not_sufficient_for_tier() -> None: + """Docstrings must state the B3 honesty constraints plainly.""" + import base.attestation.payload as mod + + doc = (mod.__doc__ or "") + (verify_attestation_payload.__doc__ or "") + lowered = doc.lower() + assert "entity holding" in lowered or "in-image secret" in lowered + assert "hardware" in lowered or "root of trust" in lowered + assert "never sufficient" in lowered or "not sufficient" in lowered + assert "tier" in lowered + + +def test_fixed_vector_signature_hex_matches_known_value() -> None: + """Cross-repo lock: same inputs → same signature hex (prism-recipe twin).""" + payload = _payload() + signed = sign_attestation_payload(payload, signing_key=VERIFY_KEY) + # Recompute expected with stdlib only (characterization of algorithm). + expected = hmac.new( + VERIFY_KEY, canonical_payload_bytes(payload), hashlib.sha256 + ).hexdigest() + assert signed.signature == expected + # Pin a concrete hex so base and prism-recipe cannot drift silently. + assert signed.signature == ( + "8eb6bfbed9bec0503597de0ccb4e8293f73936c358f3ac34ca0ffe38dd47b024" + ) diff --git a/tests/unit/test_compute_attestation_nonce.py b/tests/unit/test_compute_attestation_nonce.py new file mode 100644 index 000000000..25ef15291 --- /dev/null +++ b/tests/unit/test_compute_attestation_nonce.py @@ -0,0 +1,281 @@ +"""TDD tests for BASE-issued single-use attestation nonces (mechanism 1). + +Nonce service only: issue UUID bound to work_unit_id + miner_hotkey + pod_id, +TTL from BASE clocks, exactly one successful consume. Guest clocks are never +consulted for security decisions (M3). +""" + +from __future__ import annotations + +import ast +import uuid +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from base.compute.attestation_nonce import ( + AttestationNonceService, + NonceBinding, + NonceConsumeHit, + NonceConsumeMiss, + NonceConsumeReason, + NonceRecord, +) + +WORK_UNIT_A = "wu-aaa" +WORK_UNIT_B = "wu-bbb" +HOTKEY_A = "5HotkeyAAAA" +HOTKEY_B = "5HotkeyBBBB" +POD_A = "pod-111" +POD_B = "pod-222" +T0 = datetime(2026, 7, 26, 12, 0, 0, tzinfo=UTC) +TTL = timedelta(hours=2) + + +def _binding( + *, + work_unit_id: str = WORK_UNIT_A, + miner_hotkey: str = HOTKEY_A, + pod_id: str = POD_A, +) -> NonceBinding: + return NonceBinding( + work_unit_id=work_unit_id, + miner_hotkey=miner_hotkey, + pod_id=pod_id, + ) + + +def _service( + *, + now: datetime = T0, + ttl: timedelta = TTL, +) -> tuple[AttestationNonceService, list[datetime]]: + """Return service + mutable clock list (index 0 is current BASE time).""" + clock = [now] + + def now_fn() -> datetime: + return clock[0] + + return AttestationNonceService(ttl=ttl, now_fn=now_fn), clock + + +def test_issue_returns_uuid_bound_to_work_unit_hotkey_pod() -> None: + """Given binding, When issue, Then UUID nonce + BASE issued_at/expires_at.""" + svc, _ = _service() + binding = _binding() + + issued = svc.issue(binding) + + uuid.UUID(issued.nonce) # raises if not a UUID string + assert issued.binding == binding + assert issued.issued_at == T0 + assert issued.expires_at == T0 + TTL + assert isinstance(issued, NonceRecord) + + +def test_consume_once_accepts_matching_binding() -> None: + """Happy path: issue then consume once with same binding → hit.""" + svc, clock = _service() + binding = _binding() + issued = svc.issue(binding) + clock[0] = T0 + timedelta(minutes=5) + + result = svc.consume(issued.nonce, binding) + + assert isinstance(result, NonceConsumeHit) + assert result.record.nonce == issued.nonce + assert result.record.binding == binding + assert result.received_at == clock[0] + + +def test_second_consume_rejected_as_already_consumed() -> None: + """Replay: second consume of same nonce → already_consumed.""" + svc, clock = _service() + binding = _binding() + issued = svc.issue(binding) + clock[0] = T0 + timedelta(minutes=1) + first = svc.consume(issued.nonce, binding) + assert isinstance(first, NonceConsumeHit) + + clock[0] = T0 + timedelta(minutes=2) + second = svc.consume(issued.nonce, binding) + + assert isinstance(second, NonceConsumeMiss) + assert second.reason is NonceConsumeReason.ALREADY_CONSUMED + + +def test_cross_work_unit_rejected_with_work_unit_mismatch() -> None: + """Nonce for unit A rejected when consumed for unit B.""" + svc, clock = _service() + issued = svc.issue(_binding(work_unit_id=WORK_UNIT_A)) + clock[0] = T0 + timedelta(minutes=1) + + result = svc.consume( + issued.nonce, + _binding(work_unit_id=WORK_UNIT_B), + ) + + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.WORK_UNIT_MISMATCH + + +def test_cross_hotkey_rejected_with_miner_hotkey_mismatch() -> None: + svc, clock = _service() + issued = svc.issue(_binding(miner_hotkey=HOTKEY_A)) + clock[0] = T0 + timedelta(minutes=1) + + result = svc.consume( + issued.nonce, + _binding(miner_hotkey=HOTKEY_B), + ) + + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.MINER_HOTKEY_MISMATCH + + +def test_cross_pod_rejected_with_pod_mismatch() -> None: + svc, clock = _service() + issued = svc.issue(_binding(pod_id=POD_A)) + clock[0] = T0 + timedelta(minutes=1) + + result = svc.consume( + issued.nonce, + _binding(pod_id=POD_B), + ) + + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.POD_MISMATCH + + +def test_expired_nonce_rejected() -> None: + """Past TTL (BASE receive time) → expired; guest clock never consulted.""" + svc, clock = _service(ttl=timedelta(seconds=30)) + issued = svc.issue(_binding()) + clock[0] = T0 + timedelta(seconds=31) + + result = svc.consume(issued.nonce, _binding()) + + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.EXPIRED + + +def test_unknown_nonce_rejected() -> None: + svc, _ = _service() + result = svc.consume(str(uuid.uuid4()), _binding()) + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.UNKNOWN_NONCE + + +def test_consume_uses_explicit_received_at_not_guest_clock() -> None: + """Freshness from BASE receive time only — no guest_timestamp parameter.""" + svc, clock = _service(ttl=timedelta(minutes=10)) + issued = svc.issue(_binding()) + # Service clock advanced past expiry, but explicit BASE received_at is early. + clock[0] = T0 + timedelta(hours=5) + base_receive = T0 + timedelta(minutes=1) + + result = svc.consume( + issued.nonce, + _binding(), + received_at=base_receive, + ) + + assert isinstance(result, NonceConsumeHit) + assert result.received_at == base_receive + # API must not accept a guest clock for security decisions. + assert "guest" not in svc.consume.__code__.co_varnames + + +def test_expired_check_order_before_binding_when_already_past_ttl() -> None: + """Expired wins over binding mismatch when both would apply.""" + svc, clock = _service(ttl=timedelta(seconds=10)) + issued = svc.issue(_binding(work_unit_id=WORK_UNIT_A)) + clock[0] = T0 + timedelta(seconds=11) + + result = svc.consume( + issued.nonce, + _binding(work_unit_id=WORK_UNIT_B), + ) + + assert isinstance(result, NonceConsumeMiss) + assert result.reason is NonceConsumeReason.EXPIRED + + +def test_snapshot_roundtrip_for_persistence_boundary() -> None: + svc, clock = _service() + issued = svc.issue(_binding()) + clock[0] = T0 + timedelta(minutes=1) + assert isinstance(svc.consume(issued.nonce, _binding()), NonceConsumeHit) + + snap = svc.snapshot() + restored = AttestationNonceService.from_snapshot( + snap, + ttl=TTL, + now_fn=lambda: clock[0], + ) + # Consumed state survives restore → replay still rejected. + again = restored.consume(issued.nonce, _binding()) + assert isinstance(again, NonceConsumeMiss) + assert again.reason is NonceConsumeReason.ALREADY_CONSUMED + + +def test_issue_rejects_blank_binding_fields() -> None: + svc, _ = _service() + with pytest.raises(ValueError, match="work_unit_id"): + svc.issue(_binding(work_unit_id=" ")) + with pytest.raises(ValueError, match="miner_hotkey"): + svc.issue(_binding(miner_hotkey="")) + with pytest.raises(ValueError, match="pod_id"): + svc.issue(_binding(pod_id="\t")) + + +def test_orm_model_and_metadata_table_exist() -> None: + """Durable table matches pure NonceRecord (migration 0018).""" + from base.db import AttestationNonce, Base + + row = AttestationNonce( + nonce=str(uuid.uuid4()), + work_unit_id=WORK_UNIT_A, + miner_hotkey=HOTKEY_A, + pod_id=POD_A, + issued_at=T0, + expires_at=T0 + TTL, + consumed_at=None, + ) + assert row.work_unit_id == WORK_UNIT_A + assert "attestation_nonces" in Base.metadata.tables + assert AttestationNonce.__tablename__ == "attestation_nonces" + + +def test_alembic_migration_0018_chained_from_digest_allowlist() -> None: + path = ( + Path(__file__).resolve().parents[2] + / "alembic" + / "versions" + / "0018_attestation_nonces.py" + ) + tree = ast.parse(path.read_text(encoding="utf-8")) + values: dict[str, object] = {} + for node in tree.body: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.target.id in {"revision", "down_revision"} and isinstance( + node.value, ast.Constant + ): + values[node.target.id] = node.value.value + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id in { + "revision", + "down_revision", + }: + if isinstance(node.value, ast.Constant): + values[target.id] = node.value.value + assert values["revision"] == "0018_attestation_nonces" + assert values["down_revision"] == "0017_digest_allowlist" + text = path.read_text(encoding="utf-8") + assert "attestation_nonces" in text + assert "work_unit_id" in text + assert "miner_hotkey" in text + assert "pod_id" in text + assert "consumed_at" in text diff --git a/tests/unit/test_compute_constation_service.py b/tests/unit/test_compute_constation_service.py new file mode 100644 index 000000000..ce5866dd3 --- /dev/null +++ b/tests/unit/test_compute_constation_service.py @@ -0,0 +1,605 @@ +"""TDD tests for constation key custody, poller, corroboration, runner (15–17).""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any + +import httpx +import pytest +import respx + +from base.compute.constation_corroboration import evaluate_corroboration +from base.compute.constation_custody import ( + LiumKeyCustody, + generate_custody_master_key, +) +from base.compute.constation_poller import ( + ContinuousConstationPoller, + PollerConfig, + PollPhase, +) +from base.compute.constation_runner import ConstationRunner, ConstationRunRequest +from base.compute.constation_types import ( + ConstationFailCode, + CorroborationStatus, + FaultClass, + PollSample, +) +from base.compute.lium import ( + LiumAuthError, + LiumClient, + LiumError, + LiumPodRead, + LiumRateLimitError, +) + +BASE = "https://lium.io/api" +DIGEST_A = "sha256:" + ("a" * 64) +DIGEST_B = "sha256:" + ("b" * 64) +HOTKEY = "5MinerHotkeyConstationTest000000000000001" +POD = "pod-const-001" +WORK_UNIT = "wu-const-001" +API_KEY = "lium-test-key-NEVER-LOG-THIS-VALUE-xyz" + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +@dataclass +class FakeClock: + t: float = 0.0 + + def now(self) -> float: + return self.t + + async def sleep(self, seconds: float) -> None: + self.t += max(0.0, seconds) + + +@dataclass +class SequenceRng: + values: list[float] + i: int = 0 + + def __call__(self) -> float: + if not self.values: + return 0.0 + v = self.values[self.i % len(self.values)] + self.i += 1 + return v + + +@dataclass +class FakeSidecar: + digest: str = DIGEST_A + fail_after: int | None = None + calls: int = 0 + hang_gap: bool = False + + async def attest(self, *, pod_id: str, phase: str) -> str: + del pod_id, phase + self.calls += 1 + if self.fail_after is not None and self.calls > self.fail_after: + raise RuntimeError("sidecar_down") + return self.digest + + +@dataclass +class ScriptedLium: + """Minimal LiumClient stand-in for runner unit tests.""" + + digests: list[str | None] = field(default_factory=lambda: [DIGEST_A]) + auth_fail_on_call: int | None = None + rate_limit_on_call: int | None = None + network_fail_times: int = 0 + calls: int = 0 + _network_left: int = field(init=False) + + def __post_init__(self) -> None: + self._network_left = self.network_fail_times + + async def get_pod_raw(self, pod_id: str) -> LiumPodRead: + self.calls += 1 + if self.auth_fail_on_call is not None and self.calls >= self.auth_fail_on_call: + raise LiumAuthError("Lium GET /pods returned 401", status_code=401) + if ( + self.rate_limit_on_call is not None + and self.calls >= self.rate_limit_on_call + ): + raise LiumRateLimitError("Lium GET /pods returned 429", status_code=429) + if self._network_left > 0: + self._network_left -= 1 + raise LiumError("Lium request GET /pods failed") + idx = min(self.calls - 1, len(self.digests) - 1) + digest = self.digests[idx] + return LiumPodRead( + pod_id=pod_id, + template_id="tmpl-1", + docker_image_digest=digest, + raw={"id": pod_id, "template": {"docker_image_digest": digest}}, + ) + + async def balance(self) -> float: + return 1.0 + + +def _custody( + *, + factory: Any | None = None, +) -> LiumKeyCustody: + return LiumKeyCustody( + master_key=generate_custody_master_key(), + client_factory=factory or LiumClient, + ) + + +# =========================================================================== +# Todo 15 — custody +# =========================================================================== + + +@respx.mock +async def test_register_encrypts_key_and_probe_succeeds( + caplog: pytest.LogCaptureFixture, +) -> None: + respx.get(f"{BASE}/users/me").mock( + return_value=httpx.Response(200, json={"balance": 3.14}) + ) + custody = _custody() + with caplog.at_level(logging.DEBUG): + verdict = await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + + assert verdict.ok is True + assert verdict.reason is ConstationFailCode.OK + assert custody.has_key(HOTKEY) + # ciphertext must not equal plaintext + blob = custody.export_encrypted()[HOTKEY] + assert API_KEY.encode() not in blob + assert custody.unlock_api_key(HOTKEY) == API_KEY + assert API_KEY not in caplog.text + assert API_KEY not in repr(custody) + assert API_KEY not in str(custody) + + +@respx.mock +async def test_register_probe_401_fail_closed_does_not_store( + caplog: pytest.LogCaptureFixture, +) -> None: + respx.get(f"{BASE}/users/me").mock(return_value=httpx.Response(401)) + custody = _custody() + with caplog.at_level(logging.DEBUG): + verdict = await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + + assert verdict.ok is False + assert verdict.reason is ConstationFailCode.LIUM_AUTH_REVOKED + assert verdict.fault_class is FaultClass.MINER + assert not custody.has_key(HOTKEY) + assert API_KEY not in caplog.text + + +@respx.mock +async def test_build_client_uses_unlocked_key_without_logging( + caplog: pytest.LogCaptureFixture, +) -> None: + respx.get(f"{BASE}/users/me").mock( + return_value=httpx.Response(200, json={"balance": 1.0}) + ) + custody = _custody() + await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + with caplog.at_level(logging.DEBUG): + client = custody.build_client(HOTKEY) + assert await client.balance() == pytest.approx(1.0) + + assert API_KEY not in caplog.text + assert API_KEY not in repr(client) + + +@respx.mock +async def test_runner_mid_run_401_is_lium_auth_revoked() -> None: + """S15b: key works at register; mid-run get_pod 401 → fail-closed.""" + respx.get(f"{BASE}/users/me").mock( + return_value=httpx.Response(200, json={"balance": 1.0}) + ) + scripted = ScriptedLium(auth_fail_on_call=2) + + def factory(key: str) -> Any: + del key + return scripted + + custody = _custody(factory=factory) + await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + clock = FakeClock() + runner = ConstationRunner( + custody=custody, + sidecar=FakeSidecar(), + poller_config=PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + max_network_retries=0, + rate_limit_per_second=100.0, + ), + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + record = await runner.run( + ConstationRunRequest( + miner_hotkey=HOTKEY, + work_unit_id=WORK_UNIT, + pod_id=POD, + duration_seconds=10.0, + ) + ) + assert record.ok is False + assert record.reason is ConstationFailCode.LIUM_AUTH_REVOKED + assert record.fault_class is FaultClass.MINER + + +# =========================================================================== +# Todo 16 — poller +# =========================================================================== + + +async def test_poller_complete_start_interval_end() -> None: + clock = FakeClock() + cfg = PollerConfig( + gap_budget_seconds=30.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + max_network_retries=1, + ) + poller = ContinuousConstationPoller( + config=cfg, + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + phases: list[str] = [] + + async def poll_once(phase: str) -> PollSample: + phases.append(phase) + return PollSample( + at_monotonic=clock.now(), + phase=phase, + sidecar_digest=DIGEST_A, + lium_declared_digest=DIGEST_A, + ) + + result = await poller.run(duration_seconds=12.0, poll_once=poll_once) + assert result.ok is True + assert result.reason is ConstationFailCode.OK + assert phases[0] == PollPhase.START + assert phases[-1] == PollPhase.END + assert PollPhase.INTERVAL in phases + assert result.poll_count >= 3 + assert result.observed_max_gap_seconds <= cfg.gap_budget_seconds + + +async def test_poller_gap_budget_fail_closed() -> None: + clock = FakeClock() + cfg = PollerConfig( + gap_budget_seconds=10.0, + min_interval_seconds=25.0, # sleep exceeds budget before next poll + max_interval_seconds=25.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + ) + poller = ContinuousConstationPoller( + config=cfg, + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + + async def poll_once(phase: str) -> PollSample: + return PollSample( + at_monotonic=clock.now(), + phase=phase, + sidecar_digest=DIGEST_A, + lium_declared_digest=DIGEST_A, + ) + + result = await poller.run(duration_seconds=40.0, poll_once=poll_once) + assert result.ok is False + assert result.reason is ConstationFailCode.CONSTATION_GAP + assert result.fault_class is FaultClass.MINER + assert result.observed_max_gap_seconds > cfg.gap_budget_seconds + + +async def test_poller_429_fail_closed_not_skip() -> None: + clock = FakeClock() + cfg = PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + ) + poller = ContinuousConstationPoller( + config=cfg, + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + calls = {"n": 0} + + async def poll_once(phase: str) -> PollSample: + calls["n"] += 1 + if calls["n"] == 1: + return PollSample( + at_monotonic=clock.now(), + phase=phase, + sidecar_digest=DIGEST_A, + lium_declared_digest=DIGEST_A, + ) + raise LiumRateLimitError("429", status_code=429) + + result = await poller.run(duration_seconds=20.0, poll_once=poll_once) + assert result.ok is False + assert result.reason is ConstationFailCode.LIUM_RATE_LIMITED + assert result.fault_class is FaultClass.INFRA + + +async def test_poller_network_partition_after_bounded_retry() -> None: + clock = FakeClock() + cfg = PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + max_network_retries=2, + backoff_base_seconds=1.0, + backoff_max_seconds=1.0, + rate_limit_per_second=100.0, + ) + poller = ContinuousConstationPoller( + config=cfg, + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([1.0]), # full jitter = full delay + ) + + async def poll_once(phase: str) -> PollSample: + del phase + raise LiumError("transport down") + + result = await poller.run(duration_seconds=5.0, poll_once=poll_once) + assert result.ok is False + assert result.reason is ConstationFailCode.NETWORK_PARTITION + assert result.fault_class is FaultClass.INFRA + + +async def test_poller_poll_cap_fail_closed() -> None: + clock = FakeClock() + cfg = PollerConfig( + gap_budget_seconds=100.0, + min_interval_seconds=1.0, + max_interval_seconds=1.0, + max_polls=2, # start + end only; interval would exceed + max_cost_units=100.0, + rate_limit_per_second=100.0, + ) + poller = ContinuousConstationPoller( + config=cfg, + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + + async def poll_once(phase: str) -> PollSample: + return PollSample( + at_monotonic=clock.now(), + phase=phase, + sidecar_digest=DIGEST_A, + lium_declared_digest=DIGEST_A, + ) + + # duration 0 → start + end only = 2 polls, ok + ok_result = await poller.run(duration_seconds=0.0, poll_once=poll_once) + assert ok_result.ok is True + assert ok_result.poll_count == 2 + + +# =========================================================================== +# Todo 17 — corroboration +# =========================================================================== + + +def test_corroboration_agree_is_ok_but_not_elevation() -> None: + """Agreement is ok as a negative-channel pass; never claims independent.""" + out = evaluate_corroboration( + lium_declared_digest=DIGEST_A, + sidecar_digest=DIGEST_A, + ) + assert out.ok is True + assert out.status is CorroborationStatus.AGREE + # Module docstring / status must not say independent — checked in source test + + +def test_corroboration_mismatch_fails_miner_fault() -> None: + out = evaluate_corroboration( + lium_declared_digest=DIGEST_A, + sidecar_digest=DIGEST_B, + ) + assert out.ok is False + assert out.status is CorroborationStatus.MISMATCH + assert out.verdict.reason is ConstationFailCode.CORROBORATION_MISMATCH + assert out.verdict.fault_class is FaultClass.MINER + + +def test_corroboration_absent_lium_is_not_contradiction() -> None: + out = evaluate_corroboration( + lium_declared_digest=None, + sidecar_digest=DIGEST_A, + ) + assert out.ok is True + assert out.status is CorroborationStatus.ABSENT + + +def test_corroboration_module_never_claims_independent() -> None: + from pathlib import Path + + src = Path(__file__).resolve().parents[2] / "src/base/compute" + for name in ( + "constation_corroboration.py", + "constation_runner.py", + "constation_custody.py", + "constation_types.py", + "constation_poller.py", + ): + text = (src / name).read_text(encoding="utf-8").lower() + # Forbid positive claims of independence (negations like "not independent" ok) + import re + + assert not re.search(r"(? None: + scripted = ScriptedLium(digests=[DIGEST_A]) + + def factory(key: str) -> Any: + del key + return scripted + + async def _probe(client: Any) -> None: + del client + await scripted.balance() + + custody = _custody(factory=factory) + custody.probe_fn = _probe + await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + + clock = FakeClock() + runner = ConstationRunner( + custody=custody, + sidecar=FakeSidecar(digest=DIGEST_A), + poller_config=PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + ), + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + record = await runner.run( + ConstationRunRequest( + miner_hotkey=HOTKEY, + work_unit_id=WORK_UNIT, + pod_id=POD, + duration_seconds=10.0, + ) + ) + assert record.ok is True + assert record.reason is ConstationFailCode.OK + assert record.corroboration_status is CorroborationStatus.AGREE + assert record.sidecar_digest == DIGEST_A + assert record.lium_declared_digest == DIGEST_A + assert ( + record.constation_observed_max_gap_seconds + <= record.constation_gap_budget_seconds + ) + assert len(record.samples) >= 2 + + +async def test_runner_corroboration_mismatch_fails() -> None: + scripted = ScriptedLium(digests=[DIGEST_A]) + + def factory(key: str) -> Any: + del key + return scripted + + async def _probe(client: Any) -> None: + del client + await scripted.balance() + + custody = _custody(factory=factory) + custody.probe_fn = _probe + await custody.register(miner_hotkey=HOTKEY, api_key=API_KEY) + + clock = FakeClock() + runner = ConstationRunner( + custody=custody, + sidecar=FakeSidecar(digest=DIGEST_B), + poller_config=PollerConfig( + gap_budget_seconds=60.0, + min_interval_seconds=5.0, + max_interval_seconds=5.0, + max_polls=10, + max_cost_units=10.0, + rate_limit_per_second=100.0, + ), + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + record = await runner.run( + ConstationRunRequest( + miner_hotkey=HOTKEY, + work_unit_id=WORK_UNIT, + pod_id=POD, + duration_seconds=0.0, + ) + ) + assert record.ok is False + assert record.reason is ConstationFailCode.CORROBORATION_MISMATCH + assert record.fault_class is FaultClass.MINER + assert record.corroboration_status is CorroborationStatus.MISMATCH + + +def test_corroboration_agree_insufficient_for_elevation_contract() -> None: + """Agreement alone must not be treated as elevation — prism still needs all 6. + + This pins the runner/corroboration contract: AGREE is only a channel status, + never a tier grant. Full elevation remains prism constation_ok's job. + """ + out = evaluate_corroboration( + lium_declared_digest=DIGEST_A, + sidecar_digest=DIGEST_A, + ) + assert out.status is CorroborationStatus.AGREE + # No elevation API exists on the outcome + assert not hasattr(out, "effective_tier") + assert not hasattr(out, "grant_tier") + assert out.verdict.reason is ConstationFailCode.OK + + +async def test_runner_unregistered_key_fail_closed() -> None: + custody = _custody() + clock = FakeClock() + runner = ConstationRunner( + custody=custody, + sidecar=FakeSidecar(), + poller_config=PollerConfig(rate_limit_per_second=100.0), + now_fn=clock.now, + sleep_fn=clock.sleep, + rng_fn=SequenceRng([0.0]), + ) + record = await runner.run( + ConstationRunRequest( + miner_hotkey=HOTKEY, + work_unit_id=WORK_UNIT, + pod_id=POD, + duration_seconds=0.0, + ) + ) + assert record.ok is False + assert record.reason is ConstationFailCode.KEY_NOT_REGISTERED diff --git a/tests/unit/test_compute_digest_allowlist.py b/tests/unit/test_compute_digest_allowlist.py new file mode 100644 index 000000000..7d646e5e5 --- /dev/null +++ b/tests/unit/test_compute_digest_allowlist.py @@ -0,0 +1,297 @@ +"""TDD tests for BASE-produced image digest allowlist with revocation. + +This is an allowlist only (mechanism 4). It does not perform constation or +attestation; it records digests BASE built and answers lookup with distinct +miss reasons so prism can fail closed later. +""" + +from __future__ import annotations + +import pytest + +from base.compute.digest_allowlist import ( + AllowlistHit, + AllowlistMiss, + AllowlistMissReason, + DigestAllowlist, + DigestRecord, + ImageVariant, + is_full_git_sha, + is_image_digest, + normalize_image_digest, +) + +COMMIT_A = "a" * 40 +COMMIT_B = "b" * 40 +TREE_A = "c" * 40 +TREE_B = "d" * 40 +DIGEST_CUDA = "sha256:" + ("1" * 64) +DIGEST_CPU = "sha256:" + ("2" * 64) +DIGEST_OTHER = "sha256:" + ("3" * 64) + + +def _record( + *, + commit_sha: str = COMMIT_A, + tree_sha: str = TREE_A, + variant: ImageVariant = ImageVariant.CUDA, + digest: str = DIGEST_CUDA, +) -> DigestRecord: + return DigestRecord( + commit_sha=commit_sha, + tree_sha=tree_sha, + variant=variant, + digest=digest, + ) + + +def test_register_and_lookup_hit_for_matching_commit_and_variant() -> None: + """Given a BASE-registered cuda digest, When lookup matches, Then hit.""" + registry = DigestAllowlist() + record = _record(variant=ImageVariant.CUDA, digest=DIGEST_CUDA) + registry.register(record) + + result = registry.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistHit) + assert result.record == record + + +def test_lookup_unknown_digest_misses() -> None: + registry = DigestAllowlist() + registry.register(_record()) + + result = registry.lookup( + digest=DIGEST_OTHER, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.UNKNOWN_DIGEST + + +def test_cross_variant_scoring_rejected_with_variant_mismatch() -> None: + """cpu digest cannot score as cuda (and vice versa).""" + registry = DigestAllowlist() + registry.register( + _record(variant=ImageVariant.CPU, digest=DIGEST_CPU), + ) + + result = registry.lookup( + digest=DIGEST_CPU, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.VARIANT_MISMATCH + + +def test_commit_mismatch_when_digest_bound_to_other_commit() -> None: + registry = DigestAllowlist() + registry.register(_record(commit_sha=COMMIT_A, tree_sha=TREE_A)) + + result = registry.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_B, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.COMMIT_MISMATCH + + +def test_tree_sha_mismatch_is_commit_mismatch() -> None: + registry = DigestAllowlist() + registry.register(_record(commit_sha=COMMIT_A, tree_sha=TREE_A)) + + result = registry.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_B, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.COMMIT_MISMATCH + + +def test_revoked_digest_never_hits() -> None: + registry = DigestAllowlist() + registry.register(_record()) + registry.revoke_digest(DIGEST_CUDA) + + result = registry.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.REVOKED + + +def test_revoked_commit_never_hits_any_digest_for_that_commit() -> None: + registry = DigestAllowlist() + registry.register(_record(digest=DIGEST_CUDA)) + registry.register( + _record(variant=ImageVariant.CPU, digest=DIGEST_CPU), + ) + registry.revoke_commit(COMMIT_A) + + for digest, variant in ( + (DIGEST_CUDA, ImageVariant.CUDA), + (DIGEST_CPU, ImageVariant.CPU), + ): + result = registry.lookup( + digest=digest, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=variant, + ) + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.REVOKED + + +def test_revoked_takes_precedence_over_variant_mismatch() -> None: + registry = DigestAllowlist() + registry.register(_record(variant=ImageVariant.CPU, digest=DIGEST_CPU)) + registry.revoke_digest(DIGEST_CPU) + + result = registry.lookup( + digest=DIGEST_CPU, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.REVOKED + + +def test_register_rejects_invalid_shapes() -> None: + registry = DigestAllowlist() + with pytest.raises(ValueError, match="commit_sha"): + registry.register(_record(commit_sha="main")) + with pytest.raises(ValueError, match="tree_sha"): + registry.register(_record(tree_sha="short")) + with pytest.raises(ValueError, match="digest"): + registry.register(_record(digest="latest")) + + +def test_register_rejects_duplicate_digest_with_conflicting_binding() -> None: + registry = DigestAllowlist() + registry.register(_record(commit_sha=COMMIT_A, digest=DIGEST_CUDA)) + with pytest.raises(ValueError, match="digest"): + registry.register(_record(commit_sha=COMMIT_B, digest=DIGEST_CUDA)) + + +def test_idempotent_reregister_same_record() -> None: + registry = DigestAllowlist() + record = _record() + registry.register(record) + registry.register(record) + result = registry.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(result, AllowlistHit) + + +def test_normalize_and_validators() -> None: + assert is_full_git_sha(COMMIT_A) + assert not is_full_git_sha("abc") + assert is_image_digest(DIGEST_CUDA) + assert not is_image_digest("sha256:xyz") + assert normalize_image_digest("SHA256:" + ("a" * 64)) == "sha256:" + ("a" * 64) + + +def test_snapshot_roundtrip_for_persistence_boundary() -> None: + """Pure snapshot so a DB/file adapter can load without re-implementing rules.""" + registry = DigestAllowlist() + registry.register(_record()) + registry.register(_record(variant=ImageVariant.CPU, digest=DIGEST_CPU)) + registry.revoke_digest(DIGEST_OTHER) + registry.revoke_commit(COMMIT_B) + + snap = registry.snapshot() + restored = DigestAllowlist.from_snapshot(snap) + + hit = restored.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(hit, AllowlistHit) + assert DIGEST_OTHER in snap.denied_digests + assert COMMIT_B in snap.denied_commits + + +def test_orm_models_and_metadata_tables_exist() -> None: + """Durable tables match the pure registry concepts (migration 0017).""" + from base.db import ( + Base, + DeniedImageCommit, + DeniedImageDigest, + ImageDigestAllowlistEntry, + ) + + entry = ImageDigestAllowlistEntry( + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA.value, + digest=DIGEST_CUDA, + ) + assert entry.digest == DIGEST_CUDA + assert "image_digest_allowlist" in Base.metadata.tables + assert "denied_image_digests" in Base.metadata.tables + assert "denied_image_commits" in Base.metadata.tables + assert DeniedImageDigest.__tablename__ == "denied_image_digests" + assert DeniedImageCommit.__tablename__ == "denied_image_commits" + + +def test_alembic_migration_0017_is_chained_from_watcher_state() -> None: + import ast + from pathlib import Path + + path = ( + Path(__file__).resolve().parents[2] + / "alembic" + / "versions" + / "0017_image_digest_allowlist.py" + ) + tree = ast.parse(path.read_text(encoding="utf-8")) + values: dict[str, object] = {} + for node in tree.body: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.target.id in {"revision", "down_revision"} and isinstance( + node.value, ast.Constant + ): + values[node.target.id] = node.value.value + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id in { + "revision", + "down_revision", + }: + if isinstance(node.value, ast.Constant): + values[target.id] = node.value.value + assert values["revision"] == "0017_digest_allowlist" + assert values["down_revision"] == "0016_watcher_state" + text = path.read_text(encoding="utf-8") + assert "image_digest_allowlist" in text + assert "denied_image_digests" in text + assert "denied_image_commits" in text diff --git a/tests/unit/test_compute_lium_client.py b/tests/unit/test_compute_lium_client.py index 4aa7c25aa..60a32d064 100644 --- a/tests/unit/test_compute_lium_client.py +++ b/tests/unit/test_compute_lium_client.py @@ -841,3 +841,220 @@ def test_as_list_handles_unexpected_shapes() -> None: assert _as_list("nope", "executors") == [] assert _as_list({"executors": "nope"}, "executors") == [] assert _as_list([{"a": 1}, "skip"], "executors") == [{"a": 1}] + + +# -- Todo 13: pod / template reads + typed auth errors ------------------------- + + +def _pod_detail( + *, + pod_id: str = "pod-1", + template_id: str = "tpl-1", + digest: str | None = "sha256:" + "a" * 64, +) -> dict: + """Shape mirrors OpenAPI PodDetailResponse + nested TemplateBaseResponse.""" + return { + "id": pod_id, + "status": "RUNNING", + "pod_name": "mission-pod", + "template": { + "id": template_id, + "name": "prism-worker", + "docker_image": "ghcr.io/base/worker", + "docker_image_tag": "v1", + "docker_image_digest": digest, + }, + } + + +@respx.mock +async def test_get_pod_raw_returns_declared_digest_and_template_id() -> None: + digest = "sha256:" + "a" * 64 + respx.get(f"{BASE}/pods/pod-1").mock( + return_value=httpx.Response( + 200, json=_pod_detail(template_id="tpl-99", digest=digest) + ) + ) + from base.compute.lium import LiumPodRead + + result = await LiumClient("k").get_pod_raw("pod-1") + assert isinstance(result, LiumPodRead) + assert result.pod_id == "pod-1" + assert result.template_id == "tpl-99" + assert result.docker_image_digest == digest + assert result.raw["id"] == "pod-1" + assert result.raw["template"]["id"] == "tpl-99" + + +@respx.mock +async def test_get_pod_raw_401_raises_lium_auth_error_not_none() -> None: + from base.compute.lium import LiumAuthError + + respx.get(f"{BASE}/pods/pod-1").mock(return_value=httpx.Response(401)) + with pytest.raises(LiumAuthError) as exc_info: + await LiumClient("revoked-key").get_pod_raw("pod-1") + assert exc_info.value.status_code == 401 + assert exc_info.value is not None + assert not isinstance(exc_info.value, type(None)) + + +@respx.mock +async def test_get_pod_raw_404_raises_lium_not_found() -> None: + from base.compute.lium import LiumNotFoundError + + respx.get(f"{BASE}/pods/missing").mock(return_value=httpx.Response(404)) + with pytest.raises(LiumNotFoundError) as exc_info: + await LiumClient("k").get_pod_raw("missing") + assert exc_info.value.status_code == 404 + + +@respx.mock +async def test_get_pod_raw_429_raises_lium_rate_limit() -> None: + from base.compute.lium import LiumRateLimitError + + respx.get(f"{BASE}/pods/pod-1").mock(return_value=httpx.Response(429)) + with pytest.raises(LiumRateLimitError) as exc_info: + await LiumClient("k").get_pod_raw("pod-1") + assert exc_info.value.status_code == 429 + + +@respx.mock +async def test_get_template_raw_returns_id_and_digest() -> None: + digest = "sha256:" + "b" * 64 + respx.get(f"{BASE}/templates/tpl-7").mock( + return_value=httpx.Response( + 200, + json={ + "id": "tpl-7", + "name": "prism-worker", + "docker_image": "ghcr.io/base/worker", + "docker_image_tag": "v1", + "docker_image_digest": digest, + }, + ) + ) + from base.compute.lium import LiumTemplateRead + + result = await LiumClient("k").get_template_raw("tpl-7") + assert isinstance(result, LiumTemplateRead) + assert result.template_id == "tpl-7" + assert result.docker_image_digest == digest + assert result.name == "prism-worker" + assert result.raw["id"] == "tpl-7" + + +@respx.mock +async def test_get_template_raw_401_raises_lium_auth_error() -> None: + from base.compute.lium import LiumAuthError + + respx.get(f"{BASE}/templates/tpl-7").mock(return_value=httpx.Response(401)) + with pytest.raises(LiumAuthError) as exc_info: + await LiumClient("bad").get_template_raw("tpl-7") + assert exc_info.value.status_code == 401 + + +@respx.mock +async def test_get_template_raw_404_raises_lium_not_found() -> None: + from base.compute.lium import LiumNotFoundError + + respx.get(f"{BASE}/templates/nope").mock(return_value=httpx.Response(404)) + with pytest.raises(LiumNotFoundError) as exc_info: + await LiumClient("k").get_template_raw("nope") + assert exc_info.value.status_code == 404 + + +# -- Todo 14: ensure_template digest-aware reuse (breaking) -------------------- + + +@respx.mock +async def test_ensure_template_reuses_when_name_and_digest_match() -> None: + digest = "sha256:" + "c" * 64 + respx.get(f"{BASE}/templates").mock( + return_value=httpx.Response( + 200, + json=[ + { + "id": "tpl-same", + "name": "prism-worker", + "docker_image_digest": digest, + } + ], + ) + ) + post = respx.post(f"{BASE}/templates") + result = await LiumClient("k").ensure_template( + name="prism-worker", + docker_image="ghcr.io/base/worker", + docker_image_digest=digest, + ) + assert result == "tpl-same" + assert post.call_count == 0 + + +@respx.mock +async def test_ensure_template_rejects_same_name_different_digest() -> None: + from base.compute.lium import LiumTemplateDigestMismatchError + + existing = "sha256:" + "d" * 64 + requested = "sha256:" + "e" * 64 + respx.get(f"{BASE}/templates").mock( + return_value=httpx.Response( + 200, + json=[ + { + "id": "tpl-old", + "name": "prism-worker", + "docker_image_digest": existing, + } + ], + ) + ) + post = respx.post(f"{BASE}/templates") + with pytest.raises(LiumTemplateDigestMismatchError) as exc_info: + await LiumClient("k").ensure_template( + name="prism-worker", + docker_image="ghcr.io/base/worker", + docker_image_digest=requested, + ) + assert post.call_count == 0 + err = exc_info.value + assert err.template_id == "tpl-old" + assert err.existing_digest == existing + assert err.requested_digest == requested + # Must not silently return the stale template id. + assert "tpl-old" not in str(type(err)) + + +@respx.mock +async def test_ensure_template_rejects_missing_existing_digest_when_pinned() -> None: + """Name hit without a stored digest must not satisfy a pinned request.""" + from base.compute.lium import LiumTemplateDigestMismatchError + + requested = "sha256:" + "f" * 64 + respx.get(f"{BASE}/templates").mock( + return_value=httpx.Response( + 200, json=[{"id": "tpl-blind", "name": "prism-worker"}] + ) + ) + post = respx.post(f"{BASE}/templates") + with pytest.raises(LiumTemplateDigestMismatchError): + await LiumClient("k").ensure_template( + name="prism-worker", + docker_image="img", + docker_image_digest=requested, + ) + assert post.call_count == 0 + + +@respx.mock +async def test_ensure_template_reuses_when_neither_side_pins_digest() -> None: + """Legacy path: no digest on request and none on record still reuses by name.""" + respx.get(f"{BASE}/templates").mock( + return_value=httpx.Response(200, json=[{"id": "tpl-9", "name": "prism-worker"}]) + ) + post = respx.post(f"{BASE}/templates") + result = await LiumClient("k").ensure_template( + name="prism-worker", docker_image="img" + ) + assert result == "tpl-9" + assert post.call_count == 0 diff --git a/tests/unit/test_constation_forward_attach.py b/tests/unit/test_constation_forward_attach.py new file mode 100644 index 000000000..9340c66b0 --- /dev/null +++ b/tests/unit/test_constation_forward_attach.py @@ -0,0 +1,121 @@ +"""S7: HttpChallengeResultForwarder embeds result.constation_bundle from store.""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import httpx +import pytest + +from base.master.challenge_work_source import HttpChallengeResultForwarder +from base.validator.agent.signing import KeypairRequestSigner +from base.worker.proof import build_execution_proof + +MANIFEST = "a" * 64 + + +@contextmanager +def _preserved_logging_levels() -> Iterator[None]: + """Restore every logger's level/disabled flag and the global disable knob. + + Importing bittensor forces every already-created logger to CRITICAL. This + file sorts before the validator tests that assert on captured records, so + leaking that would empty their caplog only in the full alphabetical run. + """ + manager = logging.root.manager + prev_disable = manager.disable + prev_root_level = logging.getLogger().level + prev_levels = [ + (obj, obj.level, obj.disabled) + for obj in list(manager.loggerDict.values()) + if isinstance(obj, logging.Logger) + ] + try: + yield + finally: + for logger, level, disabled in prev_levels: + logger.setLevel(level) + logger.disabled = disabled + logging.getLogger().setLevel(prev_root_level) + manager.disable = prev_disable + + +class _Reg: + async def get(self, slug: str) -> Any: + class R: + internal_base_url = "http://prism.test" + + return R() + + async def get_token(self, slug: str) -> str: + return "tok" + + +class _CaptureTransport(httpx.AsyncBaseTransport): + def __init__(self) -> None: + self.bodies: list[dict[str, Any]] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + import json + + self.bodies.append(json.loads(request.content.decode())) + return httpx.Response(200, json={"status": "accepted"}) + + +def _minimal_proof() -> dict[str, Any]: + with _preserved_logging_levels(): + import bittensor as bt + + signer = KeypairRequestSigner(bt.Keypair.create_from_uri("//WorkerAlice")) + proof = build_execution_proof( + signer=signer, manifest_sha256=MANIFEST, unit_id="wu-1" + ) + return proof.model_dump(mode="json") + + +@pytest.mark.asyncio +async def test_forwarder_embeds_bundle_from_lookup() -> None: + transport = _CaptureTransport() + bundle = {"digest": "sha256:" + ("1" * 64), "nonce": "n1", "work_unit_id": "wu-1"} + + async def lookup(wu: str) -> dict[str, Any] | None: + return bundle if wu == "wu-1" else None + + fwd = HttpChallengeResultForwarder( + _Reg(), transport=transport, retries=1, bundle_lookup=lookup + ) + await fwd.forward_result( + challenge_slug="prism", + work_unit_id="wu-1", + submission_ref="hk", + result_payload={"execution_proof": _minimal_proof(), "executed": 1}, + ) + assert transport.bodies, "expected POST body" + body = transport.bodies[0] + assert body["result"]["constation_bundle"] == bundle + + +@pytest.mark.asyncio +async def test_forwarder_does_not_overwrite_existing_bundle() -> None: + transport = _CaptureTransport() + existing = {"digest": "existing"} + + async def lookup(_wu: str) -> dict[str, Any]: + return {"digest": "from-store"} + + fwd = HttpChallengeResultForwarder( + _Reg(), transport=transport, retries=1, bundle_lookup=lookup + ) + await fwd.forward_result( + challenge_slug="prism", + work_unit_id="wu-1", + submission_ref="hk", + result_payload={ + "execution_proof": _minimal_proof(), + "constation_bundle": existing, + }, + ) + assert transport.bodies[0]["result"]["constation_bundle"] == existing diff --git a/tests/unit/test_digest_allowlist_repository.py b/tests/unit/test_digest_allowlist_repository.py new file mode 100644 index 000000000..674d687b9 --- /dev/null +++ b/tests/unit/test_digest_allowlist_repository.py @@ -0,0 +1,194 @@ +"""TDD: durable SQLAlchemy adapter for DigestAllowlist (production host). + +Pure lookup rules stay in ``base.compute.digest_allowlist``; this repository +persists to ``ImageDigestAllowlistEntry`` / denied tables and reloads into a +``DigestAllowlist`` for S5 allowlist-miss scenarios. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from pathlib import Path + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from base.compute.digest_allowlist import ( + AllowlistHit, + AllowlistMiss, + AllowlistMissReason, + DigestRecord, + ImageVariant, +) +from base.db.models import Base +from base.db.session import session_scope +from base.master.constation.allowlist_repository import DigestAllowlistRepository + +COMMIT_A = "a" * 40 +COMMIT_B = "b" * 40 +TREE_A = "c" * 40 +TREE_B = "d" * 40 +DIGEST_CUDA = "sha256:" + ("1" * 64) +DIGEST_CPU = "sha256:" + ("2" * 64) +DIGEST_OTHER = "sha256:" + ("3" * 64) + + +def _record( + *, + commit_sha: str = COMMIT_A, + tree_sha: str = TREE_A, + variant: ImageVariant = ImageVariant.CUDA, + digest: str = DIGEST_CUDA, +) -> DigestRecord: + return DigestRecord( + commit_sha=commit_sha, + tree_sha=tree_sha, + variant=variant, + digest=digest, + ) + + +@pytest.fixture +async def session_factory( + tmp_path: Path, +) -> AsyncIterator[async_sessionmaker[AsyncSession]]: + db_path = tmp_path / "allowlist.sqlite3" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + connect_args={"check_same_thread": False}, + ) + factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + try: + yield factory + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_register_persists_and_reloads_lookup_hit( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """Given register via repository, When new instance loads, Then lookup hits.""" + repo = DigestAllowlistRepository(session_factory) + record = _record() + await repo.register(record) + + reloaded = DigestAllowlistRepository(session_factory) + allowlist = await reloaded.load_allowlist() + result = allowlist.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(result, AllowlistHit) + assert result.record == record + + +@pytest.mark.asyncio +async def test_lookup_unknown_digest_is_unknown( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record()) + allowlist = await repo.load_allowlist() + result = allowlist.lookup( + digest=DIGEST_OTHER, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.UNKNOWN_DIGEST + + +@pytest.mark.asyncio +async def test_revoke_digest_loads_as_revoked( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + """S5: denied digest remains revoked after reload.""" + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record()) + await repo.revoke_digest(DIGEST_CUDA, reason="ops-revoke") + + allowlist = await DigestAllowlistRepository(session_factory).load_allowlist() + result = allowlist.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.REVOKED + + +@pytest.mark.asyncio +async def test_revoke_commit_loads_as_revoked( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record()) + await repo.revoke_commit(COMMIT_A, reason="commit-yanked") + + allowlist = await DigestAllowlistRepository(session_factory).load_allowlist() + result = allowlist.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant=ImageVariant.CUDA, + ) + assert isinstance(result, AllowlistMiss) + assert result.reason is AllowlistMissReason.REVOKED + + +@pytest.mark.asyncio +async def test_multiple_records_roundtrip( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + repo = DigestAllowlistRepository(session_factory) + cuda = _record(variant=ImageVariant.CUDA, digest=DIGEST_CUDA) + cpu = _record( + commit_sha=COMMIT_B, + tree_sha=TREE_B, + variant=ImageVariant.CPU, + digest=DIGEST_CPU, + ) + await repo.register(cuda) + await repo.register(cpu) + + allowlist = await DigestAllowlistRepository(session_factory).load_allowlist() + hit_cuda = allowlist.lookup( + digest=DIGEST_CUDA, + commit_sha=COMMIT_A, + tree_sha=TREE_A, + variant="cuda", + ) + hit_cpu = allowlist.lookup( + digest=DIGEST_CPU, + commit_sha=COMMIT_B, + tree_sha=TREE_B, + variant="cpu", + ) + assert isinstance(hit_cuda, AllowlistHit) + assert isinstance(hit_cpu, AllowlistHit) + assert len(allowlist.snapshot().records) == 2 + + +@pytest.mark.asyncio +async def test_identical_reregister_is_noop( + session_factory: async_sessionmaker[AsyncSession], +) -> None: + repo = DigestAllowlistRepository(session_factory) + await repo.register(_record()) + await repo.register(_record()) # identical + async with session_scope(session_factory) as session: + from sqlalchemy import func, select + + from base.db.models import ImageDigestAllowlistEntry + + count = await session.scalar( + select(func.count()).select_from(ImageDigestAllowlistEntry) + ) + assert count == 1 diff --git a/tests/unit/test_e2e_lium_attestation_offline.py b/tests/unit/test_e2e_lium_attestation_offline.py new file mode 100644 index 000000000..8ad59baa1 --- /dev/null +++ b/tests/unit/test_e2e_lium_attestation_offline.py @@ -0,0 +1,103 @@ +"""Offline E2E path for checkbox 27 — real modules, fixture Lium (no LIUM_API_KEY).""" + +from __future__ import annotations + +import importlib.util +import logging +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "e2e_lium_attestation.py" + + +@contextmanager +def _preserved_process_state() -> Iterator[None]: + """Undo sys.path/sys.modules insertions and logger-level changes. + + Loading the script inserts sibling package roots into ``sys.path`` and + imports ``prism_challenge`` (which pulls in bittensor, forcing every + already-created logger to CRITICAL). Left in place, the prism dispatch + adapter stops looking unavailable and later caplog assertions go empty -- + failures that only surface in the full alphabetical run. + """ + manager = logging.root.manager + prev_path = list(sys.path) + prev_modules = set(sys.modules) + prev_disable = manager.disable + prev_root_level = logging.getLogger().level + prev_levels = [ + (obj, obj.level, obj.disabled) + for obj in list(manager.loggerDict.values()) + if isinstance(obj, logging.Logger) + ] + try: + yield + finally: + for logger, level, disabled in prev_levels: + logger.setLevel(level) + logger.disabled = disabled + logging.getLogger().setLevel(prev_root_level) + manager.disable = prev_disable + for name in set(sys.modules) - prev_modules: + sys.modules.pop(name, None) + sys.path[:] = prev_path + + +def _load_e2e_module(): + spec = importlib.util.spec_from_file_location("e2e_lium_attestation", _SCRIPT) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + # Ensure script path resolution works when loaded as module + sys.modules["e2e_lium_attestation"] = mod + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="module") +def e2e(): + with _preserved_process_state(): + yield _load_e2e_module() + + +@pytest.mark.asyncio +async def test_offline_honest_writes_score_tier1_attestation_mode(e2e) -> None: + bag = await e2e.run_offline("honest") + errors = e2e._assert_honest(bag) + assert not errors, errors + assert bag["constation_ok"] is True + assert bag["score_written"] is True + assert bag["effective_tier"] == 1 + assert bag["attestation_mode"] == e2e.ATTESTATION_MODE_V1 + assert bag["score_row"] is not None and bag["score_row"] > 0.0 + + +@pytest.mark.asyncio +async def test_offline_adversarial_midrun_swap_no_score_miner_fault(e2e) -> None: + bag = await e2e.run_offline("adversarial") + errors = e2e._assert_adversarial(bag) + assert not errors, errors + assert bag["run_record_ok"] is False + assert bag["run_record_reason"] == "corroboration_mismatch" + assert bag["constation_ok"] is False + assert bag["score_written"] is False + assert bag["score_row"] is None + assert str(bag["ingest_reason"] or "").startswith("miner_fault:") + + +def test_cli_offline_honest_exit_zero(e2e) -> None: + code = e2e.main(["--mode", "honest", "--offline"]) + assert code == 0 + + +def test_cli_offline_adversarial_exit_zero(e2e) -> None: + code = e2e.main(["--mode", "adversarial", "--offline"]) + assert code == 0 + + +def test_live_unavailable_without_key(e2e, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LIUM_API_KEY", raising=False) + assert e2e._live_available() is False