diff --git a/apps/loopover-ui/content/docs/self-hosting-operations.mdx b/apps/loopover-ui/content/docs/self-hosting-operations.mdx index 786e9e14b3..0dd2c8ece3 100644 --- a/apps/loopover-ui/content/docs/self-hosting-operations.mdx +++ b/apps/loopover-ui/content/docs/self-hosting-operations.mdx @@ -627,6 +627,35 @@ captured event. ]} /> +## AI observability (`$ai_generation`) + +When PostHog is configured (the same `POSTHOG_API_KEY`/`POSTHOG_HOST` as error tracking above — no +separate flag), every self-host AI provider attempt is also captured as a PostHog LLM-analytics +event, so spend/latency/failure are visible per provider+model+effort without opening raw logs. +This covers the full self-host AI surface: the configured review chain (`AI_PROVIDER` — +ollama/openai/openai-compatible/anthropic, and the `claude-code`/`codex` subscription CLIs) and the +three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`). + + + ## PostHog alert classes and runbook Tune PostHog insight alerts for **persistent failure classes**, not one-off diff --git a/apps/loopover-ui/src/lib/ams-env-reference.ts b/apps/loopover-ui/src/lib/ams-env-reference.ts index c940b04ca9..67f29e1f9a 100644 --- a/apps/loopover-ui/src/lib/ams-env-reference.ts +++ b/apps/loopover-ui/src/lib/ams-env-reference.ts @@ -161,6 +161,16 @@ export const AMS_ENV_REFERENCE_ROWS: MinerEnvReferenceRow[] = [ firstReference: "lib/portfolio-queue.ts", defaultValue: null, }, + { + name: "LOOPOVER_MINER_POSTHOG_API_KEY", + firstReference: "lib/egress-allowlist.ts", + defaultValue: null, + }, + { + name: "LOOPOVER_MINER_POSTHOG_HOST", + firstReference: "lib/egress-allowlist.ts", + defaultValue: null, + }, { name: "LOOPOVER_MINER_PREDICTION_LEDGER_DB", firstReference: "lib/prediction-ledger.ts", @@ -271,6 +281,8 @@ export const AMS_ENV_REFERENCE_MARKDOWN = [ "| `LOOPOVER_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.ts` | (none) |", "| `LOOPOVER_MINER_POLICY_VERDICT_CACHE_DB` | `lib/policy-verdict-cache.ts` | (none) |", "| `LOOPOVER_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.ts` | (none) |", + "| `LOOPOVER_MINER_POSTHOG_API_KEY` | `lib/egress-allowlist.ts` | (none) |", + "| `LOOPOVER_MINER_POSTHOG_HOST` | `lib/egress-allowlist.ts` | (none) |", "| `LOOPOVER_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.ts` | (none) |", "| `LOOPOVER_MINER_RANKED_CANDIDATES_DB` | `lib/ranked-candidates.ts` | (none) |", "| `LOOPOVER_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.ts` | (none) |", diff --git a/packages/loopover-miner/DEPLOYMENT.md b/packages/loopover-miner/DEPLOYMENT.md index dc914aa4ac..a8327f8c66 100644 --- a/packages/loopover-miner/DEPLOYMENT.md +++ b/packages/loopover-miner/DEPLOYMENT.md @@ -218,6 +218,7 @@ and `LOOPOVER_MINER_CONFIG_DIR` are covered above under the fleet/state notes; t | `LOOPOVER_MINER_REPO_CLONE_DIR` | `dist/lib/repo-clone.js` | Base directory for cloned target repos. | | `LOOPOVER_MINER_WORKTREE_DIR` | `dist/lib/worktree-allocator.js` | Base directory for per-attempt git worktrees. | | `LOOPOVER_MINER_SENTRY_DSN`, `LOOPOVER_MINER_SENTRY_ENVIRONMENT` | `dist/lib/sentry.js` | Optional Sentry error reporting (DSN + environment tag). | +| `LOOPOVER_MINER_POSTHOG_API_KEY`, `LOOPOVER_MINER_POSTHOG_HOST` | `dist/lib/posthog.js` | Optional PostHog error + AI-generation reporting ([#8292](https://github.com/JSONbored/loopover/issues/8292), epic [#8286](https://github.com/JSONbored/loopover/issues/8286)) — runs alongside Sentry during the parallel-run window; API key + optional ingestion host override (defaults to PostHog's US-cloud host). | | `LOOPOVER_MINER_VERSION` | `dist/lib/version.js` | Overrides the reported release id (e.g. for a custom build). | ## Optional hosted discovery plane (opt-in) diff --git a/packages/loopover-miner/bin/loopover-miner-mcp.ts b/packages/loopover-miner/bin/loopover-miner-mcp.ts index 92048056d2..57bede90b4 100755 --- a/packages/loopover-miner/bin/loopover-miner-mcp.ts +++ b/packages/loopover-miner/bin/loopover-miner-mcp.ts @@ -24,6 +24,7 @@ import { initPredictionLedger, type PredictionLedgerEntry } from "../lib/predict import { loadMinerFileSecrets } from "../lib/env-file-indirection.js"; import { installCliSignalHandlers } from "../lib/process-lifecycle.js"; import { captureMinerErrorAndFlush, initMinerSentry } from "../lib/sentry.js"; +import { captureMinerPostHogErrorAndFlush, initMinerPostHog } from "../lib/posthog.js"; // MCP stdio server for @loopover/miner (scaffold #5153). Mirrors the packages/loopover-mcp // harness (MCP SDK server + stdio transport). Tools: @@ -443,16 +444,23 @@ if (invokedPath && invokedPath === realpathSync(fileURLToPath(import.meta.url))) console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } - await initMinerSentry(process.env); - installCliSignalHandlers({ captureError: captureMinerErrorAndFlush }); + await Promise.all([initMinerSentry(process.env), initMinerPostHog(process.env)]); + installCliSignalHandlers({ + captureError: (error, context) => + Promise.all([captureMinerErrorAndFlush(error, context), captureMinerPostHogErrorAndFlush(error, context)]).then(() => undefined), + }); createMinerMcpServer() .connect(new StdioServerTransport()) .catch(async (error) => { console.error(error); - // Awaited so the captured event has a chance to actually reach Sentry before exit() tears the process - // down -- a bare synchronous capture only queues it (#6011 follow-up). - await captureMinerErrorAndFlush(error, { kind: "mcp_startup_connect_failed" }); + // Awaited so the captured event has a chance to actually reach each configured sink before exit() tears + // the process down -- a bare synchronous capture only queues it (#6011 follow-up, extended to PostHog + // per #8292's parallel-run posture). + await Promise.all([ + captureMinerErrorAndFlush(error, { kind: "mcp_startup_connect_failed" }), + captureMinerPostHogErrorAndFlush(error, { kind: "mcp_startup_connect_failed" }), + ]); process.exit(1); }); } diff --git a/packages/loopover-miner/bin/loopover-miner.ts b/packages/loopover-miner/bin/loopover-miner.ts index 6694929fe9..fcaee91318 100755 --- a/packages/loopover-miner/bin/loopover-miner.ts +++ b/packages/loopover-miner/bin/loopover-miner.ts @@ -21,6 +21,7 @@ import { runOrbExportCli } from "../lib/orb-export.js"; import { runTenantCli } from "../lib/tenant-cli.js"; import { installCliSignalHandlers } from "../lib/process-lifecycle.js"; import { captureMinerErrorAndFlush, initMinerSentry } from "../lib/sentry.js"; +import { captureMinerPostHogErrorAndFlush, initMinerPostHog } from "../lib/posthog.js"; import { runStateCli } from "../lib/run-state-cli.js"; import { runInit } from "../lib/laptop-init.js"; import { createWizardIo, runInteractiveInit } from "../lib/init-wizard.js"; @@ -48,20 +49,27 @@ try { process.exit(2); } -// Opt-in Sentry (#6011): a complete no-op unless the operator sets LOOPOVER_MINER_SENTRY_DSN themselves. Must -// run AFTER loadMinerFileSecrets (so a `_FILE`-mounted DSN resolves first) and BEFORE installCliSignalHandlers -// (so a startup crash is still captured). +// Opt-in Sentry (#6011) + opt-in PostHog (#8292, epic #8286 -- parallel-run alongside Sentry, both capture +// when both configured): each a complete no-op unless its own env var is set. Must run AFTER +// loadMinerFileSecrets (so a `_FILE`-mounted DSN/key resolves first) and BEFORE installCliSignalHandlers +// (so a startup crash is still captured by whichever sink(s) are configured). /* v8 ignore start -- process entry point (this bin's top level runs unconditionally, every invocation); * exercised by the real --help/--version subprocess spawn in miner-package-skeleton.test.ts (a different * Node process, invisible to this test run's own coverage instrumentation), not unit-coverable here. The - * functions themselves (initMinerSentry, installCliSignalHandlers) are fully unit-tested in isolation - * (miner-sentry.test.ts, miner-process-lifecycle.test.ts) -- this is only the top-level wiring that calls - * them, mirroring src/server.ts's identical, already-established exemption in codecov.yml. */ -await initMinerSentry(process.env); + * functions themselves (initMinerSentry, initMinerPostHog, installCliSignalHandlers) are fully unit-tested + * in isolation (miner-sentry.test.ts, miner-posthog.test.ts, miner-process-lifecycle.test.ts) -- this is + * only the top-level wiring that calls them, mirroring src/server.ts's identical, already-established + * exemption in codecov.yml. */ +await Promise.all([initMinerSentry(process.env), initMinerPostHog(process.env)]); // Register signal + crash handlers once, before any command runs, so an interrupted run closes its open ledgers -// cleanly instead of dying mid-write (#4826). Covers every subcommand below, including the local ones. -installCliSignalHandlers({ captureError: captureMinerErrorAndFlush }); +// cleanly instead of dying mid-write (#4826). Covers every subcommand below, including the local ones. Both +// opt-in sinks capture the crash independently -- neither call awaits the other, so a slow/hung flush on one +// sink can never delay the other's delivery. +installCliSignalHandlers({ + captureError: (error, context) => + Promise.all([captureMinerErrorAndFlush(error, context), captureMinerPostHogErrorAndFlush(error, context)]).then(() => undefined), +}); /* v8 ignore stop */ // Peel the global logging flags (--quiet/--verbose/--log-level) off the front of argv and configure the diff --git a/packages/loopover-miner/docs/env-reference.md b/packages/loopover-miner/docs/env-reference.md index 794869abdf..8d25d3fda0 100644 --- a/packages/loopover-miner/docs/env-reference.md +++ b/packages/loopover-miner/docs/env-reference.md @@ -35,6 +35,8 @@ Generated by `npm run miner:env-reference`. Do not edit manually. | `LOOPOVER_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.ts` | (none) | | `LOOPOVER_MINER_POLICY_VERDICT_CACHE_DB` | `lib/policy-verdict-cache.ts` | (none) | | `LOOPOVER_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.ts` | (none) | +| `LOOPOVER_MINER_POSTHOG_API_KEY` | `lib/egress-allowlist.ts` | (none) | +| `LOOPOVER_MINER_POSTHOG_HOST` | `lib/egress-allowlist.ts` | (none) | | `LOOPOVER_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.ts` | (none) | | `LOOPOVER_MINER_RANKED_CANDIDATES_DB` | `lib/ranked-candidates.ts` | (none) | | `LOOPOVER_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.ts` | (none) | diff --git a/packages/loopover-miner/lib/coding-agent-construction.ts b/packages/loopover-miner/lib/coding-agent-construction.ts index 894389209d..1fe9d68bfa 100644 --- a/packages/loopover-miner/lib/coding-agent-construction.ts +++ b/packages/loopover-miner/lib/coding-agent-construction.ts @@ -9,6 +9,7 @@ import { spawn as nodeSpawn } from "node:child_process"; import { + CODING_AGENT_DRIVER_CONFIG_ENV, createCodingAgentDriver, resolveFirstConfiguredCodingAgentDriverName, type AgentSdkHooks, @@ -21,6 +22,7 @@ import { type HouseRulesConfig, type HouseRulesOptions, } from "./coding-agent-house-rules.js"; +import { captureMinerPostHogAiGeneration } from "./posthog.js"; /** * Real `child_process.spawn`-backed implementation of the engine's `CliSubprocessSpawnFn` contract. Captures @@ -71,6 +73,46 @@ export function createRealCliSubprocessSpawn(): CliSubprocessSpawnFn { }); } +/** Wrap a real `CodingAgentDriver` with PostHog `$ai_generation` capture (#8296 AMS follow-up, epic #8286). + * Lives here, not in `@loopover/engine` (the pure driver package cannot depend on `posthog-node` or any + * vendor client -- the same "no cross-package import" boundary this file's own header already documents + * for its spawn implementation): this is the miner CLI's own host-bound construction site, exactly where + * `src/selfhost/`'s equivalent ORB-side wrapper (`withAiGenerationCapture`, ai.ts) lives relative to its + * own chain. `CodingAgentDriverResult` carries a single blended `costUsd`/`tokensUsed` (no input/output + * split, unlike ORB's `AiUsage`) -- captureMinerPostHogAiGeneration is deliberately built for that exact + * shape, never fabricating a split its source data doesn't have. A driver failure is reported via + * `result.ok === false` (the real, observed contract every shipped driver follows -- none of them throw + * for an ordinary task failure), with a genuine thrown exception handled defensively on top. */ +export function withCodingAgentAiGenerationCapture(providerName: string, model: string, driver: CodingAgentDriver): CodingAgentDriver { + return { + async run(task) { + const startedAtMs = Date.now(); + try { + const result = await driver.run(task); + captureMinerPostHogAiGeneration({ + provider: providerName, + model, + latencyMs: Date.now() - startedAtMs, + isError: !result.ok, + totalTokens: result.tokensUsed, + totalCostUsd: result.costUsd, + error: result.ok ? undefined : result.error, + }); + return result; + } catch (error) { + captureMinerPostHogAiGeneration({ + provider: providerName, + model, + latencyMs: Date.now() - startedAtMs, + isError: true, + error, + }); + throw error; + } + }, + }; +} + export type ConstructProductionCodingAgentDriverOptions = { spawn?: CliSubprocessSpawnFn; query?: AgentSdkQueryFn; @@ -108,7 +150,7 @@ export function constructProductionCodingAgentDriver( (providerName.trim().toLowerCase() === "agent-sdk" ? buildHouseRulesAgentSdkHooks(options.houseRulesConfig, options.houseRulesOptions) : undefined); - return createCodingAgentDriver({ + const driver = createCodingAgentDriver({ providerName, env, spawn: options.spawn ?? createRealCliSubprocessSpawn(), @@ -116,4 +158,11 @@ export function constructProductionCodingAgentDriver( ...(hooks !== undefined ? { hooks } : {}), ...(options.listChangedFiles !== undefined ? { listChangedFiles: options.listChangedFiles } : {}), }); + // #8296 AMS follow-up: the configured model env var (CODING_AGENT_DRIVER_CONFIG_ENV), when this + // provider declares one -- agent-sdk declares none (its session uses the account/CLI default), so it + // falls back to the provider name itself, mirroring ai.ts's own "unconfigured -> falls back to a + // sensible default" convention. + const modelEnvKey = CODING_AGENT_DRIVER_CONFIG_ENV[providerName as keyof typeof CODING_AGENT_DRIVER_CONFIG_ENV]?.model; + const model = (modelEnvKey ? env[modelEnvKey] : undefined) || providerName; + return withCodingAgentAiGenerationCapture(providerName, model, driver); } diff --git a/packages/loopover-miner/lib/egress-allowlist.ts b/packages/loopover-miner/lib/egress-allowlist.ts index b3281375d3..af48d948f8 100644 --- a/packages/loopover-miner/lib/egress-allowlist.ts +++ b/packages/loopover-miner/lib/egress-allowlist.ts @@ -31,6 +31,8 @@ // (`ORB_ENROLLMENT_SECRET` or #8202/#8246's `LOOPOVER_TENANT_SECRET_TOKEN` is set) // - the discovery-index plane (`LOOPOVER_MINER_DISCOVERY_INDEX_URL`) -- only if set (opt-in, no default) // - Sentry (`LOOPOVER_MINER_SENTRY_DSN`) -- only if set (opt-in, no default) +// - PostHog (`LOOPOVER_MINER_POSTHOG_API_KEY`, #8292) -- only if set (opt-in); host defaults to +// us.i.posthog.com when `LOOPOVER_MINER_POSTHOG_HOST` isn't also set, mirroring posthog.ts's own default // - Neon's API (console.neon.tech, #7858's per-attempt DB fork) -- only if all three // `LOOPOVER_MINER_NEON_*` vars are set, mirroring `resolveAttemptDbForkConfig`'s own all-or-nothing gate // This is deliberately NOT exhaustive against every possible operator configuration (a fully custom, self-run @@ -76,6 +78,10 @@ const ECOSYSTEM_REGISTRY_HOSTS: Record = /** Neon's REST API host (`attempt-db-fork.ts`'s own `DEFAULT_API_BASE_URL`) -- fixed, not per-project. */ const NEON_API_HOST = "console.neon.tech"; +/** PostHog's US-cloud ingestion host -- matches posthog.ts's own `DEFAULT_POSTHOG_HOST` default, used when + * an operator sets LOOPOVER_MINER_POSTHOG_API_KEY without also overriding LOOPOVER_MINER_POSTHOG_HOST. */ +const DEFAULT_POSTHOG_HOST = "us.i.posthog.com"; + function hostnameOf(url: string | undefined): string | undefined { if (!url) return undefined; try { @@ -115,6 +121,9 @@ export function resolveEgressAllowlist(networkAllowlist: AmsNetworkAllowlist, en if (discoveryIndexHost) add(discoveryIndexHost, "loopover-platform"); const sentryHost = hostnameOf(env.LOOPOVER_MINER_SENTRY_DSN); if (sentryHost) add(sentryHost, "loopover-platform"); + if (env.LOOPOVER_MINER_POSTHOG_API_KEY) { + add(hostnameOf(env.LOOPOVER_MINER_POSTHOG_HOST) ?? DEFAULT_POSTHOG_HOST, "loopover-platform"); + } if (env.LOOPOVER_MINER_NEON_API_KEY && env.LOOPOVER_MINER_NEON_PROJECT_ID && env.LOOPOVER_MINER_NEON_PARENT_BRANCH_ID) { add(NEON_API_HOST, "loopover-platform"); } diff --git a/packages/loopover-miner/lib/posthog.ts b/packages/loopover-miner/lib/posthog.ts new file mode 100644 index 0000000000..dab7a67bce --- /dev/null +++ b/packages/loopover-miner/lib/posthog.ts @@ -0,0 +1,150 @@ +/** Opt-in PostHog error tracking for the miner CLI (#8292, epic #8286 Phase 1). Complete no-op unless + * LOOPOVER_MINER_POSTHOG_API_KEY is set -- an operator points this at their OWN PostHog project, mirroring + * sentry.ts's identical no-phone-home posture (#6011): this is a published, independently-installed CLI + * (@loopover/miner), so nothing here is ever auto-enabled or phones home by default. `posthog-node` is + * lazy-imported only inside `initMinerPostHog()` so a miner invocation that never opts in pays zero + * module-load cost -- this CLI runs very frequently under an unattended loop (lib/loop-cli.js). + * + * Runs ALONGSIDE sentry.ts during the epic's parallel-run window (both capture when both configured) -- + * this is a new sink, not a replacement, until a gated Sentry-decommission sub-issue lands. + * + * Short-lived-process flush semantics (`flushAt: 1, flushInterval: 0`): the miner CLI runs frequently and + * exits quickly, so posthog-node's default batching would likely lose events to process exit before a + * batched flush ever fires -- the exact pattern packages/loopover-mcp/lib/telemetry.ts (#6238) already + * proves for opt-in CLI telemetry. + */ +import { randomUUID } from "node:crypto"; + +type PostHogNs = typeof import("posthog-node"); +type PostHogClient = InstanceType; + +let client: PostHogClient | undefined; +let active = false; + +/** PostHog US-cloud ingestion host -- the default when LOOPOVER_MINER_POSTHOG_HOST isn't set. */ +const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"; + +/** No per-user identity is tracked by this sink (operational error events, not user analytics) -- every + * event shares one anonymous, constant distinct id, mirroring src/mcp/telemetry.ts's/src/selfhost/posthog.ts's + * identical choice for the same reason. */ +const MINER_POSTHOG_DISTINCT_ID = "loopover-miner"; + +/** Property-key names that must never leave this process as-is -- a small, self-contained denylist (this + * package cannot import src/selfhost/redaction-scrub.ts: it's a separately published npm package with no + * dependency on the main app's src/, the same "no cross-package import" convention + * cli-subprocess-driver.ts's own header documents). Real `captureMinerError` call sites today only ever + * pass flat, non-secret context (kind/repoFullName/attemptId/branchId/scope) -- this is defense-in-depth + * for a future call site, not a response to an actual observed leak. */ +const SECRET_CONTEXT_KEY = /token|secret|password|passwd|dsn|credential|api[_-]?key|coldkey|hotkey|wallet/i; +const REDACTED = "[redacted]"; + +/** Drop any context value whose KEY looks secret-shaped; every other scalar passes through unchanged. Not + * recursive -- every real call site passes a flat object (see this file's header comment), and PostHog's + * own `properties` bag is itself flat by convention. */ +function scrubMinerContext(context: Record): Record { + const scrubbed: Record = {}; + for (const [key, value] of Object.entries(context)) { + scrubbed[key] = SECRET_CONTEXT_KEY.test(key) ? REDACTED : value; + } + return scrubbed; +} + +/** Initialize PostHog from `env` (default `process.env`). Returns whether it activated. Call once, as early + * as possible in a bin's startup -- after `loadMinerFileSecrets()` (so a `_FILE`-mounted key resolves + * first via that function's own generic `_FILE` scan, with no code change needed here) and before + * `installCliSignalHandlers()` (so a startup crash is still captured). */ +export async function initMinerPostHog(env: Record = process.env): Promise { + const apiKey = env.LOOPOVER_MINER_POSTHOG_API_KEY; + if (!apiKey) return false; + const { PostHog } = await import("posthog-node"); + const host = env.LOOPOVER_MINER_POSTHOG_HOST ?? DEFAULT_POSTHOG_HOST; + client = new PostHog(apiKey, { host, flushAt: 1, flushInterval: 0 }); + active = true; + return true; +} + +/** Capture an error with optional structured context. No-op when PostHog is off. Never throws -- + * `captureMinerError`'s own signature and never-throws contract, preserved so call sites don't churn. */ +export function captureMinerPostHogError(error: unknown, context?: Record): void { + if (!active || !client) return; + try { + client.captureException( + error instanceof Error ? error : new Error(String(error)), + MINER_POSTHOG_DISTINCT_ID, + context ? scrubMinerContext(context) : undefined, + ); + } catch { + /* PostHog capture must never crash the caller it's instrumenting. */ + } +} + +/** Flush buffered events before the process exits. No-op when off. Never throws. */ +export async function flushMinerPostHog(): Promise { + if (!active || !client) return; + try { + await client.flush(); + } catch { + /* Best-effort -- a flush failure must never block process exit. */ + } +} + +/** Capture AND flush before returning -- the crash-path convenience wrapper, mirroring + * `captureMinerErrorAndFlush`'s identical rationale: a bare `captureMinerPostHogError()` only QUEUES the + * event, and `process.exit()` tears the process down immediately afterward without waiting for any + * pending delivery. */ +export async function captureMinerPostHogErrorAndFlush(error: unknown, context?: Record): Promise { + captureMinerPostHogError(error, context); + await flushMinerPostHog(); +} + +/** One AMS coding-agent driver attempt (#8296 AMS follow-up, epic #8286 track 3). All three driver types + * (claude-cli/codex-cli/agent-sdk, packages/loopover-engine's CodingAgentDriver) report a single blended + * `costUsd`/`tokensUsed` -- unlike ORB's self-host `AiUsage`, there is no input/output split available at + * this layer, so this deliberately does NOT populate `$ai_input_tokens`/`$ai_output_tokens` with a + * fabricated split (they stay 0, honestly representing "no split known"); the real total rides in the + * plain `tokens_used` property instead. `$ai_total_cost_usd` IS one of PostHog's own recognized + * `$ai_generation` properties and needs no split, so it's populated directly when known. No field here + * ever carries prompt/diff/transcript content -- metadata only, same policy as the ORB side. */ +export type MinerAiGenerationEvent = { + provider: string; + model: string; + latencyMs: number; + isError: boolean; + totalTokens?: number | undefined; + totalCostUsd?: number | undefined; + error?: unknown; +}; + +/** Capture one AMS coding-agent attempt as PostHog's `$ai_generation` event. No-op when PostHog is off -- + * same contract as every other capture function in this file. */ +export function captureMinerPostHogAiGeneration(event: MinerAiGenerationEvent): void { + if (!active || !client) return; + const properties: Record = { + $ai_trace_id: randomUUID(), + $ai_model: event.model.trim() || "unknown", + $ai_provider: event.provider.trim() || "unknown", + // PostHog's own $ai_generation schema reports latency in SECONDS, not ms. + $ai_latency: event.latencyMs / 1000, + $ai_http_status: event.isError ? 500 : 200, + $ai_input_tokens: 0, + $ai_output_tokens: 0, + $ai_is_error: event.isError, + }; + if (Number.isFinite(event.totalTokens)) properties.tokens_used = event.totalTokens; + if (Number.isFinite(event.totalCostUsd)) properties.$ai_total_cost_usd = event.totalCostUsd; + if (event.isError) { + const error = event.error instanceof Error ? event.error : new Error(String(event.error)); + properties.$ai_error = error.message.slice(0, 500); + } + try { + client.capture({ distinctId: MINER_POSTHOG_DISTINCT_ID, event: "$ai_generation", properties }); + } catch { + /* PostHog capture must never crash the caller it's instrumenting. */ + } +} + +/** Test-only: reset module state so one test's activation can't leak into the next. */ +export function resetMinerPostHogForTesting(): void { + client = undefined; + active = false; +} diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 16ab184d75..427c2a82e6 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -11,6 +11,7 @@ import type { AiContentBlock, CombineStrategy, OnMerge } from "../services/ai-re import { isConfiguredSelfHostProvider, resolveConfiguredProviderNames } from "./ai-config"; export { assertNoLegacySharedAiEnv } from "./ai-config"; import { incr, observe } from "./metrics"; +import { capturePostHogAiGeneration, type PostHogAiGenerationRequestKind } from "./posthog"; import { withReviewSpan } from "./tracing"; import { delimiter } from "node:path"; @@ -1337,23 +1338,37 @@ async function runProviderWithOtel( request_kind: requestKindLabel, }); aiProviderCircuits.delete(provider.name); - if (result.usage) { - return { - ...result, - usage: { - ...result.usage, - provider: result.usage.provider ?? provider.name, - model: result.usage.model ?? (model || "default"), - }, - }; - } - return result; + const usage = result.usage + ? { ...result.usage, provider: result.usage.provider ?? provider.name, model: result.usage.model ?? (model || "default") } + : undefined; + capturePostHogAiGeneration({ + provider: usage?.provider ?? provider.name, + model: usage?.model ?? (model || "default"), + requestKind: requestKindLabel, + latencyMs: Date.now() - startedAtMs, + isError: false, + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + totalCostUsd: usage?.costUsd, + effort: usage?.effort, + context: { repo: options.repoFullName, pullNumber: options.pullNumber }, + }); + return usage ? { ...result, usage } : result; } catch (error) { observe("loopover_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, { provider: provider.name, request_kind: requestKindLabel, }); if (isExpectedEmbeddingRoutingError(options, error)) throw error; + capturePostHogAiGeneration({ + provider: provider.name, + model: model || "default", + requestKind: requestKindLabel, + latencyMs: Date.now() - startedAtMs, + isError: true, + error, + context: { repo: options.repoFullName, pullNumber: options.pullNumber }, + }); incr("loopover_ai_provider_failures_total", { provider: provider.name }); incr("loopover_ai_provider_request_errors_total", { provider: provider.name, request_kind: requestKindLabel }); // Re-read the map here rather than reusing the `circuit` captured above: that read happened BEFORE the @@ -1481,6 +1496,50 @@ export function createSelfHostAi(env: Record): SelfH return routeProviders(providers); } +/** Wrap a standalone `SelfHostAi` binding with the same `$ai_generation`/`$ai_embedding` PostHog capture + * {@link runProviderWithOtel} gives the main provider chain (#8296). AI_EMBED/AI_VISION/AI_ADVISORY + * (server.ts) are each a single, unrouted provider built directly by `createOpenAiCompatibleAi` -- they + * never pass through `buildProviders`/`routeProviders`/`runProviderWithOtel`, so before this wrapper they + * had zero per-call spend/latency visibility beyond a one-time boot log. No circuit breaker or chain + * metrics here (those are the main-chain's own concerns) -- this ONLY adds the capture call, matching this + * file's existing separation between chain-level concerns and the leaf capture itself. */ +export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): SelfHostAi { + return { + async run(model, options) { + const requestKindLabel: PostHogAiGenerationRequestKind = requestKind(options); + const startedAtMs = Date.now(); + try { + const result = await ai.run(model, options); + const usage = result.usage + ? { ...result.usage, provider: result.usage.provider ?? providerName, model: result.usage.model ?? (model || "default") } + : undefined; + capturePostHogAiGeneration({ + provider: usage?.provider ?? providerName, + model: usage?.model ?? (model || "default"), + requestKind: requestKindLabel, + latencyMs: Date.now() - startedAtMs, + isError: false, + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + totalCostUsd: usage?.costUsd, + effort: usage?.effort, + }); + return usage ? { ...result, usage } : result; + } catch (error) { + capturePostHogAiGeneration({ + provider: providerName, + model: model || "default", + requestKind: requestKindLabel, + latencyMs: Date.now() - startedAtMs, + isError: true, + error, + }); + throw error; + } + }, + }; +} + const COMBINE_STRATEGIES = new Set(["single", "consensus", "synthesis"]); const ON_MERGE_RULES = new Set(["either", "both"]); const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]); diff --git a/src/selfhost/posthog.ts b/src/selfhost/posthog.ts index 3683f60ae0..3741a0fffb 100644 --- a/src/selfhost/posthog.ts +++ b/src/selfhost/posthog.ts @@ -18,6 +18,7 @@ // POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT, POSTHOG_SERVER_NAME, POSTHOG_RELEASE) is self-host-only, // read off real process.env, never added to src/env.d.ts's typed Env, matching that file's precedent for // self-host-exclusive config. +import { randomUUID } from "node:crypto"; import { hostname } from "node:os"; import { currentOtelTraceIds, @@ -319,6 +320,72 @@ export async function withPostHogMonitor(name: PostHogMonitorName, context: R } } +/** `"review"` (chat/completion) vs `"embedding"` -- matches ai.ts's own `requestKind()` classification + * (`options.text` set ⇒ embedding), which picks the PostHog event name below. */ +export type PostHogAiGenerationRequestKind = "review" | "embedding"; + +/** One AI provider attempt (#8296, epic #8286 track 3): every field here is metadata -- model id, provider + * name, timing, token/cost accounting, and (on failure) an already-redacted error string ai.ts's own + * `redactSecrets`/`errorMessage` produced before ever throwing. There is deliberately no field for the + * prompt or the response text; `$ai_generation`'s own optional content properties are never populated. */ +export type PostHogAiGenerationEvent = { + provider: string; + model: string; + requestKind: PostHogAiGenerationRequestKind; + latencyMs: number; + isError: boolean; + inputTokens?: number | undefined; + outputTokens?: number | undefined; + /** A single blended figure -- ai.ts's own `AiUsage.costUsd` never splits input/output cost, so this + * never fabricates a split its source data doesn't have. Absent for the claude-code/codex subscription + * CLIs whenever their own stdout reports none (a flat subscription has no real per-call dollar cost). */ + totalCostUsd?: number | undefined; + /** Reasoning-effort dial ("low"/"medium"/"high"/"max") -- generation CONFIG, never prompt content. */ + effort?: string | undefined; + /** Correlation context (repo/PR), the same optional fields AiRunOptions already threads through for + * `logSelfHostAiProviderFailed` -- routed through {@link operationalProperties} so only the shared + * operational-tag allowlist survives, exactly like every other capture path in this file. */ + context?: Record | undefined; + /** Raw caught value on the error path (mirrors capturePostHogError's own `error` param) -- never a + * caller-preformatted string. Ignored when `isError` is false. */ + error?: unknown; +}; + +/** Capture one AI provider attempt as PostHog's `$ai_generation` (`$ai_embedding` for an embedding + * request) event (#8296). No-op when PostHog is off -- same contract as every other capture function in + * this file. Unlike {@link capturePostHogError}, this never gates on POSTHOG_MIN_SEVERITY: a successful + * generation is not an error-severity event at all, and a failed one is ALREADY captured as a real + * exception by the caller's own existing `selfhost_ai_provider_failed` log line (forwarded via + * {@link forwardStructuredLogToPostHog}) -- this event exists for spend/latency/failure ANALYTICS, a + * parallel concern to error tracking, not a substitute gate for it. */ +export function capturePostHogAiGeneration(event: PostHogAiGenerationEvent): void { + if (!active || !client) return; + const properties: Record = { + ...operationalProperties(event.context), + $ai_trace_id: randomUUID(), + $ai_model: nonBlank(event.model) ?? "unknown", + $ai_provider: nonBlank(event.provider) ?? "unknown", + // PostHog's own $ai_generation schema reports latency in SECONDS, not ms. + $ai_latency: event.latencyMs / 1000, + $ai_http_status: event.isError ? 500 : 200, + $ai_input_tokens: Number.isFinite(event.inputTokens) ? event.inputTokens : 0, + $ai_output_tokens: Number.isFinite(event.outputTokens) ? event.outputTokens : 0, + $ai_is_error: event.isError, + environment: posthogEnvironment, + }; + if (Number.isFinite(event.totalCostUsd)) properties.$ai_total_cost_usd = event.totalCostUsd; + if (event.effort) properties.$ai_model_parameters = { effort: event.effort }; + if (event.isError) { + const error = event.error instanceof Error ? event.error : new Error(String(event.error)); + properties.$ai_error = error.message.slice(0, 500); + } + client.capture({ + distinctId: POSTHOG_DISTINCT_ID, + event: event.requestKind === "embedding" ? "$ai_embedding" : "$ai_generation", + properties, + }); +} + /** Flush buffered events before exit. No-op when off. */ export async function flushPostHog(): Promise { if (!active || !client) return; diff --git a/src/server.ts b/src/server.ts index 326ece02d7..9711783acd 100644 --- a/src/server.ts +++ b/src/server.ts @@ -27,6 +27,7 @@ import { resolveSubscriptionCliPath, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, + withAiGenerationCapture, } from "./selfhost/ai"; import { cookieValue, @@ -572,12 +573,15 @@ async function main(): Promise { // the review chain — so a Claude/Codex outage never falls reviews back to a weak local model. Unset ⇒ absent ⇒ // createReviewAdapters falls back to the review `ai` for embeds (byte-identical to before). const embedAi = process.env.AI_EMBED_BASE_URL - ? createOpenAiCompatibleAi({ - baseUrl: process.env.AI_EMBED_BASE_URL, - apiKey: process.env.AI_EMBED_API_KEY ?? process.env.OPENAI_API_KEY, - embedModel: process.env.AI_EMBED_MODEL, - providerName: providerNameFromBaseUrl(process.env.AI_EMBED_BASE_URL), - }) + ? withAiGenerationCapture( + "ai_embed", + createOpenAiCompatibleAi({ + baseUrl: process.env.AI_EMBED_BASE_URL, + apiKey: process.env.AI_EMBED_API_KEY ?? process.env.OPENAI_API_KEY, + embedModel: process.env.AI_EMBED_MODEL, + providerName: providerNameFromBaseUrl(process.env.AI_EMBED_BASE_URL), + }), + ) : undefined; if (embedAi) console.log( @@ -593,12 +597,15 @@ async function main(): Promise { // different model, a different capability) the same way AI_EMBED is kept separate from the review chain. // Unset ⇒ absent ⇒ visual-vision falls back to BYOK-only (byte-identical to before this binding existed). const visionAi = process.env.AI_VISION_BASE_URL - ? createOpenAiCompatibleAi({ - baseUrl: process.env.AI_VISION_BASE_URL, - apiKey: process.env.AI_VISION_API_KEY ?? process.env.OPENAI_API_KEY, - model: process.env.AI_VISION_MODEL, - providerName: providerNameFromBaseUrl(process.env.AI_VISION_BASE_URL), - }) + ? withAiGenerationCapture( + "ai_vision", + createOpenAiCompatibleAi({ + baseUrl: process.env.AI_VISION_BASE_URL, + apiKey: process.env.AI_VISION_API_KEY ?? process.env.OPENAI_API_KEY, + model: process.env.AI_VISION_MODEL, + providerName: providerNameFromBaseUrl(process.env.AI_VISION_BASE_URL), + }), + ) : undefined; if (visionAi) console.log( @@ -616,12 +623,15 @@ async function main(): Promise { // config-driven, not hardcoded. Unset ⇒ absent ⇒ every advisory capability falls back to env.AI, byte- // identical to before this binding existed. const advisoryAi = process.env.AI_ADVISORY_BASE_URL - ? createOpenAiCompatibleAi({ - baseUrl: process.env.AI_ADVISORY_BASE_URL, - apiKey: process.env.AI_ADVISORY_API_KEY, - model: process.env.AI_ADVISORY_MODEL, - providerName: providerNameFromBaseUrl(process.env.AI_ADVISORY_BASE_URL), - }) + ? withAiGenerationCapture( + "ai_advisory", + createOpenAiCompatibleAi({ + baseUrl: process.env.AI_ADVISORY_BASE_URL, + apiKey: process.env.AI_ADVISORY_API_KEY, + model: process.env.AI_ADVISORY_MODEL, + providerName: providerNameFromBaseUrl(process.env.AI_ADVISORY_BASE_URL), + }), + ) : undefined; if (advisoryAi) console.log( diff --git a/test/unit/miner-coding-agent-construction.test.ts b/test/unit/miner-coding-agent-construction.test.ts index 0b17998ae5..68e578d956 100644 --- a/test/unit/miner-coding-agent-construction.test.ts +++ b/test/unit/miner-coding-agent-construction.test.ts @@ -1,11 +1,34 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@loopover/engine", async () => { return import("../../packages/loopover-engine/src/index"); }); -import { createRealCliSubprocessSpawn, constructProductionCodingAgentDriver } from "../../packages/loopover-miner/lib/coding-agent-construction.js"; -import type { AgentSdkQueryFn, CodingAgentDriverTask } from "../../packages/loopover-engine/src/index"; +const posthogMock = vi.hoisted(() => { + const capture = vi.fn(); + const PostHog = vi.fn(function (this: any) { + this.capture = capture; + this.flush = vi.fn().mockResolvedValue(undefined); + }); + return { capture, PostHog }; +}); +vi.mock("posthog-node", () => ({ PostHog: posthogMock.PostHog })); + +import { + createRealCliSubprocessSpawn, + constructProductionCodingAgentDriver, + withCodingAgentAiGenerationCapture, +} from "../../packages/loopover-miner/lib/coding-agent-construction.js"; +import { initMinerPostHog, resetMinerPostHogForTesting } from "../../packages/loopover-miner/lib/posthog.js"; +import type { AgentSdkQueryFn, CodingAgentDriver, CodingAgentDriverTask } from "../../packages/loopover-engine/src/index"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +afterEach(() => { + resetMinerPostHogForTesting(); +}); const task: CodingAgentDriverTask = { attemptId: "attempt-1", @@ -175,3 +198,91 @@ describe("constructProductionCodingAgentDriver (#5131)", () => { expect(append).toHaveBeenCalledWith(expect.objectContaining({ repoFullName: "acme/widgets" })); }); }); + +describe("withCodingAgentAiGenerationCapture (#8296 AMS follow-up)", () => { + function driverReturning(result: Awaited>): CodingAgentDriver { + return { run: async () => result }; + } + + it("stays a no-op end-to-end when PostHog is unconfigured", async () => { + const driver = withCodingAgentAiGenerationCapture("claude-cli", "claude-sonnet-5", driverReturning({ ok: true, changedFiles: [], summary: "done", transcript: "" })); + const result = await driver.run(task); + expect(result.ok).toBe(true); + expect(posthogMock.capture).not.toHaveBeenCalled(); + }); + + it("captures a successful attempt's cost/tokens as a combined figure (no fabricated input/output split)", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const driver = withCodingAgentAiGenerationCapture( + "claude-cli", + "claude-sonnet-5", + driverReturning({ ok: true, changedFiles: ["a.ts"], summary: "done", transcript: "", costUsd: 0.12, tokensUsed: 4000 }), + ); + await driver.run(task); + expect(posthogMock.capture).toHaveBeenCalledTimes(1); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("claude-cli"); + expect(properties.$ai_model).toBe("claude-sonnet-5"); + expect(properties.$ai_is_error).toBe(false); + expect(properties.tokens_used).toBe(4000); + expect(properties.$ai_total_cost_usd).toBe(0.12); + }); + + it("captures result.ok:false as a failure, using the driver's own error string -- no exception thrown", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const driver = withCodingAgentAiGenerationCapture( + "codex-cli", + "gpt-5-codex", + driverReturning({ ok: false, changedFiles: [], summary: "failed", transcript: "", error: "codex_timeout_120000ms" }), + ); + const result = await driver.run(task); + expect(result.ok).toBe(false); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect(properties.$ai_is_error).toBe(true); + expect(properties.$ai_http_status).toBe(500); + expect(properties.$ai_error).toBe("codex_timeout_120000ms"); + }); + + it("REGRESSION: still captures a failure AND rethrows when the wrapped driver itself throws unexpectedly", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const driver = withCodingAgentAiGenerationCapture("agent-sdk", "agent-sdk", { run: async () => { throw new Error("sdk crashed"); } }); + await expect(driver.run(task)).rejects.toThrow("sdk crashed"); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect(properties.$ai_is_error).toBe(true); + expect(properties.$ai_error).toBe("sdk crashed"); + }); +}); + +describe("constructProductionCodingAgentDriver -- $ai_generation capture wiring (#8296 AMS follow-up)", () => { + it("captures using the configured model env var for a CLI provider", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const driver = constructProductionCodingAgentDriver( + { MINER_CODING_AGENT_PROVIDER: "claude-cli", MINER_CODING_AGENT_CLAUDE_MODEL: "claude-opus-5" }, + { spawn: async () => ({ stdout: "done", code: 0 }) }, + ); + await driver.run(task); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("claude-cli"); + expect(properties.$ai_model).toBe("claude-opus-5"); + }); + + it("falls back to the provider name as the model when no model env var is configured (e.g. agent-sdk, which declares none)", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const driver = constructProductionCodingAgentDriver( + { MINER_CODING_AGENT_PROVIDER: "agent-sdk" }, + { query: queryCapturing({}), listChangedFiles: async () => [] }, + ); + await driver.run(task); + expect(posthogMock.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("agent-sdk"); + }); + + it("falls back to the provider name when a CLI provider's own model env var is configured but blank", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const driver = constructProductionCodingAgentDriver( + { MINER_CODING_AGENT_PROVIDER: "codex-cli", MINER_CODING_AGENT_CODEX_MODEL: "" }, + { spawn: async () => ({ stdout: "done", code: 0 }) }, + ); + await driver.run(task); + expect(posthogMock.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("codex-cli"); + }); +}); diff --git a/test/unit/miner-egress-allowlist.test.ts b/test/unit/miner-egress-allowlist.test.ts index a608f15219..ea0be2d036 100644 --- a/test/unit/miner-egress-allowlist.test.ts +++ b/test/unit/miner-egress-allowlist.test.ts @@ -91,6 +91,26 @@ describe("resolveEgressAllowlist (#7857)", () => { expect(entries).toContainEqual({ host: "o1.ingest.sentry.io", reason: "loopover-platform" }); }); + it("adds the PostHog default host only when LOOPOVER_MINER_POSTHOG_API_KEY is set, with no host override (#8292)", () => { + expect(resolveEgressAllowlist(EMPTY_ALLOWLIST, {}).some((e) => e.host === "us.i.posthog.com")).toBe(false); + const entries = resolveEgressAllowlist(EMPTY_ALLOWLIST, { LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test" }); + expect(entries).toContainEqual({ host: "us.i.posthog.com", reason: "loopover-platform" }); + }); + + it("adds the overridden PostHog host when LOOPOVER_MINER_POSTHOG_HOST is also set", () => { + const entries = resolveEgressAllowlist(EMPTY_ALLOWLIST, { + LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test", + LOOPOVER_MINER_POSTHOG_HOST: "https://eu.i.posthog.com", + }); + expect(entries).toContainEqual({ host: "eu.i.posthog.com", reason: "loopover-platform" }); + expect(entries.some((e) => e.host === "us.i.posthog.com")).toBe(false); + }); + + it("falls back to the PostHog default host when LOOPOVER_MINER_POSTHOG_HOST is malformed but the API key is set", () => { + const entries = resolveEgressAllowlist(EMPTY_ALLOWLIST, { LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test", LOOPOVER_MINER_POSTHOG_HOST: "not a url" }); + expect(entries).toContainEqual({ host: "us.i.posthog.com", reason: "loopover-platform" }); + }); + it("adds Neon's API host only when ALL THREE LOOPOVER_MINER_NEON_* vars are set (all-or-nothing, matching resolveAttemptDbForkConfig's own gate)", () => { expect( resolveEgressAllowlist(EMPTY_ALLOWLIST, { LOOPOVER_MINER_NEON_API_KEY: "k", LOOPOVER_MINER_NEON_PROJECT_ID: "p" }).some((e) => e.reason === "loopover-platform"), diff --git a/test/unit/miner-posthog.test.ts b/test/unit/miner-posthog.test.ts new file mode 100644 index 0000000000..b8aacf199d --- /dev/null +++ b/test/unit/miner-posthog.test.ts @@ -0,0 +1,248 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const posthogMock = vi.hoisted(() => { + const captureException = vi.fn(); + const capture = vi.fn(); + const flush = vi.fn().mockResolvedValue(undefined); + let lastArgs: any; + const PostHog = vi.fn(function (this: any, apiKey: string, options: any) { + lastArgs = { apiKey, options }; + this.captureException = captureException; + this.capture = capture; + this.flush = flush; + }); + return { captureException, capture, flush, PostHog, getLastArgs: () => lastArgs }; +}); + +vi.mock("posthog-node", () => ({ PostHog: posthogMock.PostHog })); + +import { + captureMinerPostHogAiGeneration, + captureMinerPostHogError, + captureMinerPostHogErrorAndFlush, + flushMinerPostHog, + initMinerPostHog, + resetMinerPostHogForTesting, +} from "../../packages/loopover-miner/lib/posthog.js"; + +beforeEach(() => { + vi.clearAllMocks(); + posthogMock.flush.mockResolvedValue(undefined); +}); + +afterEach(() => { + resetMinerPostHogForTesting(); +}); + +describe("loopover-miner opt-in PostHog (#8292, epic #8286)", () => { + describe("off state (no API key)", () => { + it("stays fully off when LOOPOVER_MINER_POSTHOG_API_KEY is unset", async () => { + expect(await initMinerPostHog({})).toBe(false); + expect(posthogMock.PostHog).not.toHaveBeenCalled(); + expect(() => captureMinerPostHogError(new Error("x"))).not.toThrow(); + expect(posthogMock.captureException).not.toHaveBeenCalled(); + await expect(flushMinerPostHog()).resolves.toBeUndefined(); + expect(posthogMock.flush).not.toHaveBeenCalled(); + }); + + it("REGRESSION: an empty-string API key is treated the same as unset (never activates)", async () => { + expect(await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "" })).toBe(false); + expect(posthogMock.PostHog).not.toHaveBeenCalled(); + }); + + it("captureMinerPostHogError never throws even when called before initMinerPostHog (default off state)", () => { + expect(() => captureMinerPostHogError("a plain string, not an Error")).not.toThrow(); + expect(() => captureMinerPostHogError(new Error("boom"), { kind: "test" })).not.toThrow(); + }); + + it("defaults to process.env when no env argument is passed", async () => { + const original = process.env.LOOPOVER_MINER_POSTHOG_API_KEY; + delete process.env.LOOPOVER_MINER_POSTHOG_API_KEY; + try { + expect(await initMinerPostHog()).toBe(false); + } finally { + if (original === undefined) delete process.env.LOOPOVER_MINER_POSTHOG_API_KEY; + else process.env.LOOPOVER_MINER_POSTHOG_API_KEY = original; + } + }); + }); + + describe("activated state (API key set)", () => { + it("initializes posthog-node with the key, default host, and short-lived-process flush semantics", async () => { + expect(await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" })).toBe(true); + expect(posthogMock.PostHog).toHaveBeenCalledWith("phc_test_key", { + host: "https://us.i.posthog.com", + flushAt: 1, + flushInterval: 0, + }); + }); + + it("uses LOOPOVER_MINER_POSTHOG_HOST when set", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key", LOOPOVER_MINER_POSTHOG_HOST: "https://eu.i.posthog.com" }); + expect(posthogMock.getLastArgs().options.host).toBe("https://eu.i.posthog.com"); + }); + + it("captureMinerPostHogError wraps a non-Error value and forwards it via captureException", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogError("plain string reason"); + expect(posthogMock.captureException).toHaveBeenCalledTimes(1); + const captured = posthogMock.captureException.mock.calls[0]?.[0] as Error; + expect(captured).toBeInstanceOf(Error); + expect(captured.message).toBe("plain string reason"); + }); + + it("captureMinerPostHogError forwards a real Error instance as-is (not re-wrapped)", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const original = new Error("real error"); + captureMinerPostHogError(original); + expect(posthogMock.captureException.mock.calls[0]?.[0]).toBe(original); + }); + + it("passes context as properties only when context is provided (both sides of the branch)", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogError(new Error("with context"), { kind: "test_kind", repoFullName: "acme/widgets" }); + expect(posthogMock.captureException).toHaveBeenCalledWith(expect.any(Error), "loopover-miner", { kind: "test_kind", repoFullName: "acme/widgets" }); + + captureMinerPostHogError(new Error("no context")); + expect(posthogMock.captureException).toHaveBeenLastCalledWith(expect.any(Error), "loopover-miner", undefined); + }); + + it("redacts a secret-shaped context key's VALUE, leaving its sibling keys untouched", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogError(new Error("boom"), { kind: "test", apiKey: "sk-should-not-leak", repoFullName: "acme/widgets" }); + const properties = posthogMock.captureException.mock.calls[0]?.[2] as Record; + expect(properties.apiKey).toBe("[redacted]"); + expect(properties.kind).toBe("test"); + expect(properties.repoFullName).toBe("acme/widgets"); + }); + + it("REGRESSION: captureMinerPostHogError never throws even when the client itself throws", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + posthogMock.captureException.mockImplementationOnce(() => { + throw new Error("posthog sdk internal failure"); + }); + expect(() => captureMinerPostHogError(new Error("boom"))).not.toThrow(); + }); + + it("flushMinerPostHog calls the client's flush", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + await flushMinerPostHog(); + expect(posthogMock.flush).toHaveBeenCalledTimes(1); + }); + + it("REGRESSION: flushMinerPostHog never throws or rejects even when flush itself rejects", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + posthogMock.flush.mockRejectedValueOnce(new Error("flush timed out")); + await expect(flushMinerPostHog()).resolves.toBeUndefined(); + }); + }); + + it("resetMinerPostHogForTesting returns an activated instance to the default-off no-op", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + resetMinerPostHogForTesting(); + captureMinerPostHogError(new Error("after reset")); + expect(posthogMock.captureException).not.toHaveBeenCalled(); + await flushMinerPostHog(); + expect(posthogMock.flush).not.toHaveBeenCalled(); + }); + + describe("captureMinerPostHogAiGeneration (#8296 AMS follow-up)", () => { + const BASE = { provider: "claude-cli", model: "claude-sonnet-5", latencyMs: 2500, isError: false }; + + it("is a no-op when PostHog is unconfigured", () => { + expect(() => captureMinerPostHogAiGeneration(BASE)).not.toThrow(); + expect(posthogMock.capture).not.toHaveBeenCalled(); + }); + + it("captures a well-formed $ai_generation event with a combined tokens_used property (no fabricated input/output split)", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogAiGeneration({ ...BASE, totalTokens: 1500, totalCostUsd: 0.05 }); + expect(posthogMock.capture).toHaveBeenCalledTimes(1); + const call = posthogMock.capture.mock.calls[0]?.[0]; + expect(call.event).toBe("$ai_generation"); + expect(call.distinctId).toBe("loopover-miner"); + expect(call.properties.$ai_model).toBe("claude-sonnet-5"); + expect(call.properties.$ai_provider).toBe("claude-cli"); + expect(call.properties.$ai_latency).toBe(2.5); + expect(call.properties.$ai_http_status).toBe(200); + expect(call.properties.$ai_input_tokens).toBe(0); + expect(call.properties.$ai_output_tokens).toBe(0); + expect(call.properties.tokens_used).toBe(1500); + expect(call.properties.$ai_total_cost_usd).toBe(0.05); + expect(call.properties.$ai_is_error).toBe(false); + expect("$ai_input" in call.properties).toBe(false); + expect("$ai_output_choices" in call.properties).toBe(false); + }); + + it("omits tokens_used/$ai_total_cost_usd when neither is supplied or finite", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogAiGeneration({ ...BASE, totalTokens: Number.NaN }); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect("tokens_used" in properties).toBe(false); + expect("$ai_total_cost_usd" in properties).toBe(false); + }); + + it("falls back to 'unknown' for a blank model/provider", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogAiGeneration({ ...BASE, model: "", provider: " " }); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect(properties.$ai_model).toBe("unknown"); + expect(properties.$ai_provider).toBe("unknown"); + }); + + it("marks a failed generation with $ai_is_error/$ai_http_status/$ai_error, redacted to 500 chars", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogAiGeneration({ ...BASE, isError: true, error: new Error("x".repeat(600)) }); + const { properties } = posthogMock.capture.mock.calls[0]?.[0]; + expect(properties.$ai_is_error).toBe(true); + expect(properties.$ai_http_status).toBe(500); + expect(properties.$ai_error).toHaveLength(500); + }); + + it("wraps a non-Error thrown value on the error path", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogAiGeneration({ ...BASE, isError: true, error: "just a string" }); + expect(posthogMock.capture.mock.calls[0]?.[0].properties.$ai_error).toBe("just a string"); + }); + + it("REGRESSION: never throws even when the client itself throws", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + posthogMock.capture.mockImplementationOnce(() => { + throw new Error("posthog sdk internal failure"); + }); + expect(() => captureMinerPostHogAiGeneration(BASE)).not.toThrow(); + }); + + it("mints a fresh, real UUID trace id on every call", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + captureMinerPostHogAiGeneration(BASE); + captureMinerPostHogAiGeneration(BASE); + const first = posthogMock.capture.mock.calls[0]?.[0].properties.$ai_trace_id; + const second = posthogMock.capture.mock.calls[1]?.[0].properties.$ai_trace_id; + expect(first).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); + expect(first).not.toBe(second); + }); + }); + + describe("captureMinerPostHogErrorAndFlush (the crash-path convenience wrapper)", () => { + it("REGRESSION: captures AND flushes, so a crash-path caller can await it before exiting instead of a bare capture that only queues the event", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + const error = new Error("crash"); + await captureMinerPostHogErrorAndFlush(error, { kind: "uncaughtException" }); + expect(posthogMock.captureException).toHaveBeenCalledWith(error, "loopover-miner", { kind: "uncaughtException" }); + expect(posthogMock.flush).toHaveBeenCalledTimes(1); + }); + + it("resolves cleanly (no throw) when PostHog is off", async () => { + await expect(captureMinerPostHogErrorAndFlush(new Error("x"))).resolves.toBeUndefined(); + expect(posthogMock.captureException).not.toHaveBeenCalled(); + expect(posthogMock.flush).not.toHaveBeenCalled(); + }); + + it("still resolves cleanly when the underlying flush rejects", async () => { + await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" }); + posthogMock.flush.mockRejectedValueOnce(new Error("flush timed out")); + await expect(captureMinerPostHogErrorAndFlush(new Error("crash"))).resolves.toBeUndefined(); + }); + }); +}); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 664952633a..38026ef5ea 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,9 +2,26 @@ import { chmodSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from " import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, codexErrorFromStdout, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, isAiProviderHealthy, markAiProviderUnhealthyAtBoot, providerNameFromBaseUrl, resetAiProviderCircuitBreakerForTest, resetAiProviderHealthForTest, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveClaudeFirstOutputTimeoutMs, resolveCodexAuthPath, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveCodexFirstOutputTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, withAdvisoryAiEnv, __selfHostAiInternals } from "../../src/selfhost/ai"; + +// Mock posthog-node so ai.ts's new $ai_generation capture (via src/selfhost/posthog.ts) resolves to a +// spy-backed client -- mirrors selfhost-posthog.test.ts's identical hoisted mocking pattern. +const posthogMocks = vi.hoisted(() => { + const capture = vi.fn(); + const captureException = vi.fn(); + const PostHog = vi.fn(function (this: any) { + this.capture = capture; + this.captureException = captureException; + this.flush = vi.fn().mockResolvedValue(undefined); + this.shutdown = vi.fn().mockResolvedValue(undefined); + }); + return { capture, captureException, PostHog }; +}); +vi.mock("posthog-node", () => ({ PostHog: posthogMocks.PostHog })); + +import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, codexErrorFromStdout, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, isAiProviderHealthy, markAiProviderUnhealthyAtBoot, providerNameFromBaseUrl, resetAiProviderCircuitBreakerForTest, resetAiProviderHealthForTest, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveClaudeFirstOutputTimeoutMs, resolveCodexAuthPath, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveCodexFirstOutputTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, withAdvisoryAiEnv, withAiGenerationCapture, __selfHostAiInternals } from "../../src/selfhost/ai"; import { labelSelfHostReviewerModel, labelSelfHostReviewerModels } from "../../src/selfhost/ai-config"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; +import { initPostHog, resetPostHogForTest } from "../../src/selfhost/posthog"; describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; @@ -127,6 +144,8 @@ afterEach(() => { resetMetrics(); resetAiProviderHealthForTest(); resetAiProviderCircuitBreakerForTest(); + resetPostHogForTest(); + vi.clearAllMocks(); }); type SpawnResult = { stdout: string; code: number | null; stderr?: string; timedOut?: boolean; stalledNoOutput?: boolean }; @@ -621,6 +640,142 @@ describe("AI provider request duration/error metrics (#4367)", () => { }); }); +describe("$ai_generation PostHog capture at the chain chokepoint (#8296)", () => { + it("captures a well-formed $ai_generation event on a successful chain call, using the provider's own usage", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { + name: "posthog-ok-provider", + ai: { run: async () => ({ response: "ok", usage: { inputTokens: 10, outputTokens: 5, costUsd: 0.01, effort: "low" } }) }, + }; + await createChainAi([provider]).run("m", { prompt: "review this", repoFullName: "owner/repo", pullNumber: 3 }); + expect(posthogMocks.capture).toHaveBeenCalledTimes(1); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_generation"); + expect(properties.$ai_provider).toBe("posthog-ok-provider"); + expect(properties.$ai_model).toBe("m"); + expect(properties.$ai_input_tokens).toBe(10); + expect(properties.$ai_output_tokens).toBe(5); + expect(properties.$ai_total_cost_usd).toBe(0.01); + expect(properties.$ai_model_parameters).toEqual({ effort: "low" }); + expect(properties.$ai_is_error).toBe(false); + expect(properties.repo).toBe("owner/repo"); + expect(properties.pullNumber).toBe(3); + }); + + it("captures $ai_embedding for an embedding request through the chain", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "posthog-embed-provider", ai: { run: async () => ({ data: [[0.1, 0.2]] }) } }; + await createChainAi([provider]).run("m", { text: ["chunk"] }); + expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_embedding"); + }); + + it("falls back to the provider's registry name when the result carries no usage", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "posthog-no-usage-provider", ai: { run: async () => ({ response: "ok" }) } }; + await createChainAi([provider]).run("m", { prompt: "x" }); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("posthog-no-usage-provider"); + expect(properties.$ai_input_tokens).toBe(0); + }); + + it("captures a failed generation with $ai_is_error/$ai_error on a real chain failure", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "posthog-fail-provider", ai: { run: async () => { throw new Error("boom"); } } }; + await expect(createChainAi([provider]).run("m", { prompt: "review this" })).rejects.toThrow(/boom/); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_is_error).toBe(true); + expect(properties.$ai_error).toBe("boom"); + expect(properties.$ai_provider).toBe("posthog-fail-provider"); + }); + + it("does NOT capture for an expected embedding-routing fallback (mirrors the metrics exemption above)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "posthog-routing-provider", ai: { run: async () => { throw new Error("claude_code_no_embed"); } } }; + await expect(createChainAi([provider]).run("m", { text: ["chunk"] })).rejects.toThrow(); + expect(posthogMocks.capture).not.toHaveBeenCalled(); + }); + + it("stays a no-op end-to-end when PostHog is unconfigured (no posthog-node client constructed)", async () => { + const provider = { name: "posthog-unconfigured-provider", ai: { run: async () => ({ response: "ok" }) } }; + await createChainAi([provider]).run("m", { prompt: "x" }); + expect(posthogMocks.capture).not.toHaveBeenCalled(); + }); +}); + +describe("withAiGenerationCapture (#8296 — AI_EMBED/AI_VISION/AI_ADVISORY, ungoverned single-provider bindings)", () => { + it("captures a successful call's usage, falling back to the given providerName when usage carries none", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_embed", { run: async () => ({ data: [[0.1]] }) }); + const result = await ai.run("bge-m3", { text: ["chunk"] }); + expect(result.data).toEqual([[0.1]]); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_embedding"); + expect(properties.$ai_provider).toBe("ai_embed"); + expect(properties.$ai_model).toBe("bge-m3"); + }); + + it("prefers the wrapped provider's own reported usage.provider/model over the given providerName", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_vision", { + run: async () => ({ response: "ok", usage: { provider: "ollama", model: "llava" } }), + }); + await ai.run("configured-model", { prompt: "describe this image" }); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("ollama"); + expect(properties.$ai_model).toBe("llava"); + }); + + it("captures a failure and still rethrows the original error", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_advisory", { run: async () => { throw new Error("advisory down"); } }); + await expect(ai.run("m", { prompt: "x" })).rejects.toThrow("advisory down"); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_is_error).toBe(true); + expect(properties.$ai_provider).toBe("ai_advisory"); + expect(properties.$ai_error).toBe("advisory down"); + }); + + it("labels a chat (non-embedding) request kind as $ai_generation, not $ai_embedding", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_advisory", { run: async () => ({ response: "ok" }) }); + await ai.run("m", { prompt: "x" }); + expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_generation"); + }); + + it("falls back to providerName/'default' when usage is present but omits its own provider/model, and the model arg is empty", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_vision", { run: async () => ({ response: "ok", usage: { inputTokens: 3 } }) }); + await ai.run("", { prompt: "x" }); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("ai_vision"); + expect(properties.$ai_model).toBe("default"); + expect(properties.$ai_input_tokens).toBe(3); + }); + + it("falls back to the model arg (not 'default') when usage omits its own model but the model arg is truthy", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_vision", { run: async () => ({ response: "ok", usage: { inputTokens: 3 } }) }); + await ai.run("configured-model", { prompt: "x" }); + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("configured-model"); + }); + + it("falls back to 'default' model on the failure path when the model arg is empty", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_embed", { run: async () => { throw new Error("down"); } }); + await expect(ai.run("", { text: ["chunk"] })).rejects.toThrow("down"); + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("default"); + }); + + it("falls back to 'default' model when the result carries no usage at all AND the model arg is empty", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_advisory", { run: async () => ({ response: "ok" }) }); + await ai.run("", { prompt: "x" }); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("ai_advisory"); + expect(properties.$ai_model).toBe("default"); + }); +}); + describe("per-provider circuit breaker (#2540 — skip fast during a sustained outage)", () => { afterEach(() => { vi.useRealTimers(); diff --git a/test/unit/selfhost-posthog.test.ts b/test/unit/selfhost-posthog.test.ts index 2b1b34161f..393b2093a6 100644 --- a/test/unit/selfhost-posthog.test.ts +++ b/test/unit/selfhost-posthog.test.ts @@ -24,6 +24,7 @@ vi.mock("../../src/selfhost/otel", () => ({ })); import { + capturePostHogAiGeneration, capturePostHogError, capturePostHogReviewFailure, flushPostHog, @@ -445,6 +446,125 @@ describe("withPostHogMonitor", () => { }); }); +describe("capturePostHogAiGeneration (#8296)", () => { + const BASE = { provider: "ollama", model: "llama3.1", requestKind: "review" as const, latencyMs: 1500, isError: false }; + + it("is a no-op when PostHog is unconfigured", () => { + capturePostHogAiGeneration(BASE); + expect(mocks.capture).not.toHaveBeenCalled(); + }); + + it("captures a well-formed $ai_generation event with token/latency metadata on success", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, inputTokens: 120, outputTokens: 40, totalCostUsd: 0.002, effort: "medium" }); + expect(mocks.capture).toHaveBeenCalledTimes(1); + const call = mocks.capture.mock.calls[0]?.[0]; + expect(call.event).toBe("$ai_generation"); + expect(call.properties.$ai_model).toBe("llama3.1"); + expect(call.properties.$ai_provider).toBe("ollama"); + expect(call.properties.$ai_latency).toBe(1.5); // ms -> seconds + expect(call.properties.$ai_http_status).toBe(200); + expect(call.properties.$ai_input_tokens).toBe(120); + expect(call.properties.$ai_output_tokens).toBe(40); + expect(call.properties.$ai_is_error).toBe(false); + expect(call.properties.$ai_total_cost_usd).toBe(0.002); + expect(call.properties.$ai_model_parameters).toEqual({ effort: "medium" }); + expect("$ai_error" in call.properties).toBe(false); + }); + + it("captures $ai_embedding for an embedding request kind", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, requestKind: "embedding" }); + expect(mocks.capture.mock.calls[0]?.[0].event).toBe("$ai_embedding"); + }); + + it("defaults input/output tokens to 0 when omitted or non-finite", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, inputTokens: undefined, outputTokens: Number.NaN }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_input_tokens).toBe(0); + expect(properties.$ai_output_tokens).toBe(0); + }); + + it("omits $ai_total_cost_usd when cost is not supplied or non-finite", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, totalCostUsd: Number.NaN }); + expect("$ai_total_cost_usd" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false); + }); + + it("omits $ai_model_parameters when effort is not supplied", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration(BASE); + expect("$ai_model_parameters" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false); + }); + + it("falls back to 'unknown' for a blank model/provider", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, model: "", provider: " " }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_model).toBe("unknown"); + expect(properties.$ai_provider).toBe("unknown"); + }); + + it("marks a failed generation with $ai_is_error/$ai_http_status/$ai_error, redacted to 500 chars", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, isError: true, error: new Error("x".repeat(600)) }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_is_error).toBe(true); + expect(properties.$ai_http_status).toBe(500); + expect(properties.$ai_error).toHaveLength(500); + }); + + it("wraps a non-Error thrown value on the error path", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, isError: true, error: "just a string" }); + expect(mocks.capture.mock.calls[0]?.[0].properties.$ai_error).toBe("just a string"); + }); + + it("never carries prompt/response content -- no field beyond model/provider ids", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration(BASE); + const keys = Object.keys(mocks.capture.mock.calls[0]?.[0].properties); + expect(keys).not.toContain("$ai_input"); + expect(keys).not.toContain("$ai_output_choices"); + }); + + it("mints a fresh, real UUID trace id on every call", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration(BASE); + capturePostHogAiGeneration(BASE); + const first = mocks.capture.mock.calls[0]?.[0].properties.$ai_trace_id; + const second = mocks.capture.mock.calls[1]?.[0].properties.$ai_trace_id; + expect(first).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); + expect(first).not.toBe(second); + }); + + it("merges correlation context (repo/pullNumber) through the shared operational-tag allowlist", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, context: { repo: "owner/repo", pullNumber: 7 } }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.repo).toBe("owner/repo"); + expect(properties.pullNumber).toBe(7); + }); + + it("drops an unlisted context key (not on OPERATIONAL_TAG_KEYS)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, context: { notAllowlisted: "should be dropped" } }); + expect("notAllowlisted" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false); + }); + + it("does not gate on POSTHOG_MIN_SEVERITY (unlike capturePostHogError)", async () => { + process.env.POSTHOG_MIN_SEVERITY = "critical"; + try { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, isError: true, error: new Error("boom") }); + expect(mocks.capture).toHaveBeenCalledTimes(1); + } finally { + delete process.env.POSTHOG_MIN_SEVERITY; + } + }); +}); + describe("flushPostHog / shutdownPostHog", () => { it("flushPostHog is a no-op when unconfigured", async () => { await flushPostHog();