From 84500430b015196a342132cafe3747d767d1a8d2 Mon Sep 17 00:00:00 2001
From: oktofeesh1 <287075021+oktofeesh1@users.noreply.github.com>
Date: Sat, 25 Jul 2026 17:01:01 +0200
Subject: [PATCH] feat(selfhost): PostHog AI observability across ORB and AMS
Captures $ai_generation/$ai_embedding events for every self-host AI
provider attempt (the AI_PROVIDER chain plus the AI_EMBED/AI_VISION/
AI_ADVISORY bindings) and for every AMS coding-agent driver attempt
(claude-cli/codex-cli/agent-sdk), metadata only -- model, provider,
latency, tokens, cost, never prompt/response content.
Adds the miner CLI's own opt-in PostHog client (packages/loopover-miner/
lib/posthog.ts) as the foundation the AMS side needs, running alongside
the existing Sentry integration during the parallel-run window. Includes
egress-allowlist entries, env-reference regeneration, and DEPLOYMENT.md
updates for the two new miner env vars.
---
.../content/docs/self-hosting-operations.mdx | 29 ++
apps/loopover-ui/src/lib/ams-env-reference.ts | 12 +
packages/loopover-miner/DEPLOYMENT.md | 1 +
.../loopover-miner/bin/loopover-miner-mcp.ts | 18 +-
packages/loopover-miner/bin/loopover-miner.ts | 26 +-
packages/loopover-miner/docs/env-reference.md | 2 +
.../lib/coding-agent-construction.ts | 51 +++-
.../loopover-miner/lib/egress-allowlist.ts | 9 +
packages/loopover-miner/lib/posthog.ts | 150 +++++++++++
src/selfhost/ai.ts | 81 +++++-
src/selfhost/posthog.ts | 67 +++++
src/server.ts | 46 ++--
.../miner-coding-agent-construction.test.ts | 117 ++++++++-
test/unit/miner-egress-allowlist.test.ts | 20 ++
test/unit/miner-posthog.test.ts | 248 ++++++++++++++++++
test/unit/selfhost-ai.test.ts | 157 ++++++++++-
test/unit/selfhost-posthog.test.ts | 120 +++++++++
17 files changed, 1106 insertions(+), 48 deletions(-)
create mode 100644 packages/loopover-miner/lib/posthog.ts
create mode 100644 test/unit/miner-posthog.test.ts
diff --git a/apps/loopover-ui/content/docs/self-hosting-operations.mdx b/apps/loopover-ui/content/docs/self-hosting-operations.mdx
index 786e9e14b3..0dd2c8ece3 100644
--- a/apps/loopover-ui/content/docs/self-hosting-operations.mdx
+++ b/apps/loopover-ui/content/docs/self-hosting-operations.mdx
@@ -627,6 +627,35 @@ captured event.
]}
/>
+## AI observability (`$ai_generation`)
+
+When PostHog is configured (the same `POSTHOG_API_KEY`/`POSTHOG_HOST` as error tracking above — no
+separate flag), every self-host AI provider attempt is also captured as a PostHog LLM-analytics
+event, so spend/latency/failure are visible per provider+model+effort without opening raw logs.
+This covers the full self-host AI surface: the configured review chain (`AI_PROVIDER` —
+ollama/openai/openai-compatible/anthropic, and the `claude-code`/`codex` subscription CLIs) and the
+three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`).
+
+
+
## PostHog alert classes and runbook
Tune PostHog insight alerts for **persistent failure classes**, not one-off
diff --git a/apps/loopover-ui/src/lib/ams-env-reference.ts b/apps/loopover-ui/src/lib/ams-env-reference.ts
index c940b04ca9..67f29e1f9a 100644
--- a/apps/loopover-ui/src/lib/ams-env-reference.ts
+++ b/apps/loopover-ui/src/lib/ams-env-reference.ts
@@ -161,6 +161,16 @@ export const AMS_ENV_REFERENCE_ROWS: MinerEnvReferenceRow[] = [
firstReference: "lib/portfolio-queue.ts",
defaultValue: null,
},
+ {
+ name: "LOOPOVER_MINER_POSTHOG_API_KEY",
+ firstReference: "lib/egress-allowlist.ts",
+ defaultValue: null,
+ },
+ {
+ name: "LOOPOVER_MINER_POSTHOG_HOST",
+ firstReference: "lib/egress-allowlist.ts",
+ defaultValue: null,
+ },
{
name: "LOOPOVER_MINER_PREDICTION_LEDGER_DB",
firstReference: "lib/prediction-ledger.ts",
@@ -271,6 +281,8 @@ export const AMS_ENV_REFERENCE_MARKDOWN = [
"| `LOOPOVER_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.ts` | (none) |",
"| `LOOPOVER_MINER_POLICY_VERDICT_CACHE_DB` | `lib/policy-verdict-cache.ts` | (none) |",
"| `LOOPOVER_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.ts` | (none) |",
+ "| `LOOPOVER_MINER_POSTHOG_API_KEY` | `lib/egress-allowlist.ts` | (none) |",
+ "| `LOOPOVER_MINER_POSTHOG_HOST` | `lib/egress-allowlist.ts` | (none) |",
"| `LOOPOVER_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.ts` | (none) |",
"| `LOOPOVER_MINER_RANKED_CANDIDATES_DB` | `lib/ranked-candidates.ts` | (none) |",
"| `LOOPOVER_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.ts` | (none) |",
diff --git a/packages/loopover-miner/DEPLOYMENT.md b/packages/loopover-miner/DEPLOYMENT.md
index dc914aa4ac..a8327f8c66 100644
--- a/packages/loopover-miner/DEPLOYMENT.md
+++ b/packages/loopover-miner/DEPLOYMENT.md
@@ -218,6 +218,7 @@ and `LOOPOVER_MINER_CONFIG_DIR` are covered above under the fleet/state notes; t
| `LOOPOVER_MINER_REPO_CLONE_DIR` | `dist/lib/repo-clone.js` | Base directory for cloned target repos. |
| `LOOPOVER_MINER_WORKTREE_DIR` | `dist/lib/worktree-allocator.js` | Base directory for per-attempt git worktrees. |
| `LOOPOVER_MINER_SENTRY_DSN`, `LOOPOVER_MINER_SENTRY_ENVIRONMENT` | `dist/lib/sentry.js` | Optional Sentry error reporting (DSN + environment tag). |
+| `LOOPOVER_MINER_POSTHOG_API_KEY`, `LOOPOVER_MINER_POSTHOG_HOST` | `dist/lib/posthog.js` | Optional PostHog error + AI-generation reporting ([#8292](https://github.com/JSONbored/loopover/issues/8292), epic [#8286](https://github.com/JSONbored/loopover/issues/8286)) — runs alongside Sentry during the parallel-run window; API key + optional ingestion host override (defaults to PostHog's US-cloud host). |
| `LOOPOVER_MINER_VERSION` | `dist/lib/version.js` | Overrides the reported release id (e.g. for a custom build). |
## Optional hosted discovery plane (opt-in)
diff --git a/packages/loopover-miner/bin/loopover-miner-mcp.ts b/packages/loopover-miner/bin/loopover-miner-mcp.ts
index 92048056d2..57bede90b4 100755
--- a/packages/loopover-miner/bin/loopover-miner-mcp.ts
+++ b/packages/loopover-miner/bin/loopover-miner-mcp.ts
@@ -24,6 +24,7 @@ import { initPredictionLedger, type PredictionLedgerEntry } from "../lib/predict
import { loadMinerFileSecrets } from "../lib/env-file-indirection.js";
import { installCliSignalHandlers } from "../lib/process-lifecycle.js";
import { captureMinerErrorAndFlush, initMinerSentry } from "../lib/sentry.js";
+import { captureMinerPostHogErrorAndFlush, initMinerPostHog } from "../lib/posthog.js";
// MCP stdio server for @loopover/miner (scaffold #5153). Mirrors the packages/loopover-mcp
// harness (MCP SDK server + stdio transport). Tools:
@@ -443,16 +444,23 @@ if (invokedPath && invokedPath === realpathSync(fileURLToPath(import.meta.url)))
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
}
- await initMinerSentry(process.env);
- installCliSignalHandlers({ captureError: captureMinerErrorAndFlush });
+ await Promise.all([initMinerSentry(process.env), initMinerPostHog(process.env)]);
+ installCliSignalHandlers({
+ captureError: (error, context) =>
+ Promise.all([captureMinerErrorAndFlush(error, context), captureMinerPostHogErrorAndFlush(error, context)]).then(() => undefined),
+ });
createMinerMcpServer()
.connect(new StdioServerTransport())
.catch(async (error) => {
console.error(error);
- // Awaited so the captured event has a chance to actually reach Sentry before exit() tears the process
- // down -- a bare synchronous capture only queues it (#6011 follow-up).
- await captureMinerErrorAndFlush(error, { kind: "mcp_startup_connect_failed" });
+ // Awaited so the captured event has a chance to actually reach each configured sink before exit() tears
+ // the process down -- a bare synchronous capture only queues it (#6011 follow-up, extended to PostHog
+ // per #8292's parallel-run posture).
+ await Promise.all([
+ captureMinerErrorAndFlush(error, { kind: "mcp_startup_connect_failed" }),
+ captureMinerPostHogErrorAndFlush(error, { kind: "mcp_startup_connect_failed" }),
+ ]);
process.exit(1);
});
}
diff --git a/packages/loopover-miner/bin/loopover-miner.ts b/packages/loopover-miner/bin/loopover-miner.ts
index 6694929fe9..fcaee91318 100755
--- a/packages/loopover-miner/bin/loopover-miner.ts
+++ b/packages/loopover-miner/bin/loopover-miner.ts
@@ -21,6 +21,7 @@ import { runOrbExportCli } from "../lib/orb-export.js";
import { runTenantCli } from "../lib/tenant-cli.js";
import { installCliSignalHandlers } from "../lib/process-lifecycle.js";
import { captureMinerErrorAndFlush, initMinerSentry } from "../lib/sentry.js";
+import { captureMinerPostHogErrorAndFlush, initMinerPostHog } from "../lib/posthog.js";
import { runStateCli } from "../lib/run-state-cli.js";
import { runInit } from "../lib/laptop-init.js";
import { createWizardIo, runInteractiveInit } from "../lib/init-wizard.js";
@@ -48,20 +49,27 @@ try {
process.exit(2);
}
-// Opt-in Sentry (#6011): a complete no-op unless the operator sets LOOPOVER_MINER_SENTRY_DSN themselves. Must
-// run AFTER loadMinerFileSecrets (so a `_FILE`-mounted DSN resolves first) and BEFORE installCliSignalHandlers
-// (so a startup crash is still captured).
+// Opt-in Sentry (#6011) + opt-in PostHog (#8292, epic #8286 -- parallel-run alongside Sentry, both capture
+// when both configured): each a complete no-op unless its own env var is set. Must run AFTER
+// loadMinerFileSecrets (so a `_FILE`-mounted DSN/key resolves first) and BEFORE installCliSignalHandlers
+// (so a startup crash is still captured by whichever sink(s) are configured).
/* v8 ignore start -- process entry point (this bin's top level runs unconditionally, every invocation);
* exercised by the real --help/--version subprocess spawn in miner-package-skeleton.test.ts (a different
* Node process, invisible to this test run's own coverage instrumentation), not unit-coverable here. The
- * functions themselves (initMinerSentry, installCliSignalHandlers) are fully unit-tested in isolation
- * (miner-sentry.test.ts, miner-process-lifecycle.test.ts) -- this is only the top-level wiring that calls
- * them, mirroring src/server.ts's identical, already-established exemption in codecov.yml. */
-await initMinerSentry(process.env);
+ * functions themselves (initMinerSentry, initMinerPostHog, installCliSignalHandlers) are fully unit-tested
+ * in isolation (miner-sentry.test.ts, miner-posthog.test.ts, miner-process-lifecycle.test.ts) -- this is
+ * only the top-level wiring that calls them, mirroring src/server.ts's identical, already-established
+ * exemption in codecov.yml. */
+await Promise.all([initMinerSentry(process.env), initMinerPostHog(process.env)]);
// Register signal + crash handlers once, before any command runs, so an interrupted run closes its open ledgers
-// cleanly instead of dying mid-write (#4826). Covers every subcommand below, including the local ones.
-installCliSignalHandlers({ captureError: captureMinerErrorAndFlush });
+// cleanly instead of dying mid-write (#4826). Covers every subcommand below, including the local ones. Both
+// opt-in sinks capture the crash independently -- neither call awaits the other, so a slow/hung flush on one
+// sink can never delay the other's delivery.
+installCliSignalHandlers({
+ captureError: (error, context) =>
+ Promise.all([captureMinerErrorAndFlush(error, context), captureMinerPostHogErrorAndFlush(error, context)]).then(() => undefined),
+});
/* v8 ignore stop */
// Peel the global logging flags (--quiet/--verbose/--log-level) off the front of argv and configure the
diff --git a/packages/loopover-miner/docs/env-reference.md b/packages/loopover-miner/docs/env-reference.md
index 794869abdf..8d25d3fda0 100644
--- a/packages/loopover-miner/docs/env-reference.md
+++ b/packages/loopover-miner/docs/env-reference.md
@@ -35,6 +35,8 @@ Generated by `npm run miner:env-reference`. Do not edit manually.
| `LOOPOVER_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.ts` | (none) |
| `LOOPOVER_MINER_POLICY_VERDICT_CACHE_DB` | `lib/policy-verdict-cache.ts` | (none) |
| `LOOPOVER_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.ts` | (none) |
+| `LOOPOVER_MINER_POSTHOG_API_KEY` | `lib/egress-allowlist.ts` | (none) |
+| `LOOPOVER_MINER_POSTHOG_HOST` | `lib/egress-allowlist.ts` | (none) |
| `LOOPOVER_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.ts` | (none) |
| `LOOPOVER_MINER_RANKED_CANDIDATES_DB` | `lib/ranked-candidates.ts` | (none) |
| `LOOPOVER_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.ts` | (none) |
diff --git a/packages/loopover-miner/lib/coding-agent-construction.ts b/packages/loopover-miner/lib/coding-agent-construction.ts
index 894389209d..1fe9d68bfa 100644
--- a/packages/loopover-miner/lib/coding-agent-construction.ts
+++ b/packages/loopover-miner/lib/coding-agent-construction.ts
@@ -9,6 +9,7 @@
import { spawn as nodeSpawn } from "node:child_process";
import {
+ CODING_AGENT_DRIVER_CONFIG_ENV,
createCodingAgentDriver,
resolveFirstConfiguredCodingAgentDriverName,
type AgentSdkHooks,
@@ -21,6 +22,7 @@ import {
type HouseRulesConfig,
type HouseRulesOptions,
} from "./coding-agent-house-rules.js";
+import { captureMinerPostHogAiGeneration } from "./posthog.js";
/**
* Real `child_process.spawn`-backed implementation of the engine's `CliSubprocessSpawnFn` contract. Captures
@@ -71,6 +73,46 @@ export function createRealCliSubprocessSpawn(): CliSubprocessSpawnFn {
});
}
+/** Wrap a real `CodingAgentDriver` with PostHog `$ai_generation` capture (#8296 AMS follow-up, epic #8286).
+ * Lives here, not in `@loopover/engine` (the pure driver package cannot depend on `posthog-node` or any
+ * vendor client -- the same "no cross-package import" boundary this file's own header already documents
+ * for its spawn implementation): this is the miner CLI's own host-bound construction site, exactly where
+ * `src/selfhost/`'s equivalent ORB-side wrapper (`withAiGenerationCapture`, ai.ts) lives relative to its
+ * own chain. `CodingAgentDriverResult` carries a single blended `costUsd`/`tokensUsed` (no input/output
+ * split, unlike ORB's `AiUsage`) -- captureMinerPostHogAiGeneration is deliberately built for that exact
+ * shape, never fabricating a split its source data doesn't have. A driver failure is reported via
+ * `result.ok === false` (the real, observed contract every shipped driver follows -- none of them throw
+ * for an ordinary task failure), with a genuine thrown exception handled defensively on top. */
+export function withCodingAgentAiGenerationCapture(providerName: string, model: string, driver: CodingAgentDriver): CodingAgentDriver {
+ return {
+ async run(task) {
+ const startedAtMs = Date.now();
+ try {
+ const result = await driver.run(task);
+ captureMinerPostHogAiGeneration({
+ provider: providerName,
+ model,
+ latencyMs: Date.now() - startedAtMs,
+ isError: !result.ok,
+ totalTokens: result.tokensUsed,
+ totalCostUsd: result.costUsd,
+ error: result.ok ? undefined : result.error,
+ });
+ return result;
+ } catch (error) {
+ captureMinerPostHogAiGeneration({
+ provider: providerName,
+ model,
+ latencyMs: Date.now() - startedAtMs,
+ isError: true,
+ error,
+ });
+ throw error;
+ }
+ },
+ };
+}
+
export type ConstructProductionCodingAgentDriverOptions = {
spawn?: CliSubprocessSpawnFn;
query?: AgentSdkQueryFn;
@@ -108,7 +150,7 @@ export function constructProductionCodingAgentDriver(
(providerName.trim().toLowerCase() === "agent-sdk"
? buildHouseRulesAgentSdkHooks(options.houseRulesConfig, options.houseRulesOptions)
: undefined);
- return createCodingAgentDriver({
+ const driver = createCodingAgentDriver({
providerName,
env,
spawn: options.spawn ?? createRealCliSubprocessSpawn(),
@@ -116,4 +158,11 @@ export function constructProductionCodingAgentDriver(
...(hooks !== undefined ? { hooks } : {}),
...(options.listChangedFiles !== undefined ? { listChangedFiles: options.listChangedFiles } : {}),
});
+ // #8296 AMS follow-up: the configured model env var (CODING_AGENT_DRIVER_CONFIG_ENV), when this
+ // provider declares one -- agent-sdk declares none (its session uses the account/CLI default), so it
+ // falls back to the provider name itself, mirroring ai.ts's own "unconfigured -> falls back to a
+ // sensible default" convention.
+ const modelEnvKey = CODING_AGENT_DRIVER_CONFIG_ENV[providerName as keyof typeof CODING_AGENT_DRIVER_CONFIG_ENV]?.model;
+ const model = (modelEnvKey ? env[modelEnvKey] : undefined) || providerName;
+ return withCodingAgentAiGenerationCapture(providerName, model, driver);
}
diff --git a/packages/loopover-miner/lib/egress-allowlist.ts b/packages/loopover-miner/lib/egress-allowlist.ts
index b3281375d3..af48d948f8 100644
--- a/packages/loopover-miner/lib/egress-allowlist.ts
+++ b/packages/loopover-miner/lib/egress-allowlist.ts
@@ -31,6 +31,8 @@
// (`ORB_ENROLLMENT_SECRET` or #8202/#8246's `LOOPOVER_TENANT_SECRET_TOKEN` is set)
// - the discovery-index plane (`LOOPOVER_MINER_DISCOVERY_INDEX_URL`) -- only if set (opt-in, no default)
// - Sentry (`LOOPOVER_MINER_SENTRY_DSN`) -- only if set (opt-in, no default)
+// - PostHog (`LOOPOVER_MINER_POSTHOG_API_KEY`, #8292) -- only if set (opt-in); host defaults to
+// us.i.posthog.com when `LOOPOVER_MINER_POSTHOG_HOST` isn't also set, mirroring posthog.ts's own default
// - Neon's API (console.neon.tech, #7858's per-attempt DB fork) -- only if all three
// `LOOPOVER_MINER_NEON_*` vars are set, mirroring `resolveAttemptDbForkConfig`'s own all-or-nothing gate
// This is deliberately NOT exhaustive against every possible operator configuration (a fully custom, self-run
@@ -76,6 +78,10 @@ const ECOSYSTEM_REGISTRY_HOSTS: Record =
/** Neon's REST API host (`attempt-db-fork.ts`'s own `DEFAULT_API_BASE_URL`) -- fixed, not per-project. */
const NEON_API_HOST = "console.neon.tech";
+/** PostHog's US-cloud ingestion host -- matches posthog.ts's own `DEFAULT_POSTHOG_HOST` default, used when
+ * an operator sets LOOPOVER_MINER_POSTHOG_API_KEY without also overriding LOOPOVER_MINER_POSTHOG_HOST. */
+const DEFAULT_POSTHOG_HOST = "us.i.posthog.com";
+
function hostnameOf(url: string | undefined): string | undefined {
if (!url) return undefined;
try {
@@ -115,6 +121,9 @@ export function resolveEgressAllowlist(networkAllowlist: AmsNetworkAllowlist, en
if (discoveryIndexHost) add(discoveryIndexHost, "loopover-platform");
const sentryHost = hostnameOf(env.LOOPOVER_MINER_SENTRY_DSN);
if (sentryHost) add(sentryHost, "loopover-platform");
+ if (env.LOOPOVER_MINER_POSTHOG_API_KEY) {
+ add(hostnameOf(env.LOOPOVER_MINER_POSTHOG_HOST) ?? DEFAULT_POSTHOG_HOST, "loopover-platform");
+ }
if (env.LOOPOVER_MINER_NEON_API_KEY && env.LOOPOVER_MINER_NEON_PROJECT_ID && env.LOOPOVER_MINER_NEON_PARENT_BRANCH_ID) {
add(NEON_API_HOST, "loopover-platform");
}
diff --git a/packages/loopover-miner/lib/posthog.ts b/packages/loopover-miner/lib/posthog.ts
new file mode 100644
index 0000000000..dab7a67bce
--- /dev/null
+++ b/packages/loopover-miner/lib/posthog.ts
@@ -0,0 +1,150 @@
+/** Opt-in PostHog error tracking for the miner CLI (#8292, epic #8286 Phase 1). Complete no-op unless
+ * LOOPOVER_MINER_POSTHOG_API_KEY is set -- an operator points this at their OWN PostHog project, mirroring
+ * sentry.ts's identical no-phone-home posture (#6011): this is a published, independently-installed CLI
+ * (@loopover/miner), so nothing here is ever auto-enabled or phones home by default. `posthog-node` is
+ * lazy-imported only inside `initMinerPostHog()` so a miner invocation that never opts in pays zero
+ * module-load cost -- this CLI runs very frequently under an unattended loop (lib/loop-cli.js).
+ *
+ * Runs ALONGSIDE sentry.ts during the epic's parallel-run window (both capture when both configured) --
+ * this is a new sink, not a replacement, until a gated Sentry-decommission sub-issue lands.
+ *
+ * Short-lived-process flush semantics (`flushAt: 1, flushInterval: 0`): the miner CLI runs frequently and
+ * exits quickly, so posthog-node's default batching would likely lose events to process exit before a
+ * batched flush ever fires -- the exact pattern packages/loopover-mcp/lib/telemetry.ts (#6238) already
+ * proves for opt-in CLI telemetry.
+ */
+import { randomUUID } from "node:crypto";
+
+type PostHogNs = typeof import("posthog-node");
+type PostHogClient = InstanceType;
+
+let client: PostHogClient | undefined;
+let active = false;
+
+/** PostHog US-cloud ingestion host -- the default when LOOPOVER_MINER_POSTHOG_HOST isn't set. */
+const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com";
+
+/** No per-user identity is tracked by this sink (operational error events, not user analytics) -- every
+ * event shares one anonymous, constant distinct id, mirroring src/mcp/telemetry.ts's/src/selfhost/posthog.ts's
+ * identical choice for the same reason. */
+const MINER_POSTHOG_DISTINCT_ID = "loopover-miner";
+
+/** Property-key names that must never leave this process as-is -- a small, self-contained denylist (this
+ * package cannot import src/selfhost/redaction-scrub.ts: it's a separately published npm package with no
+ * dependency on the main app's src/, the same "no cross-package import" convention
+ * cli-subprocess-driver.ts's own header documents). Real `captureMinerError` call sites today only ever
+ * pass flat, non-secret context (kind/repoFullName/attemptId/branchId/scope) -- this is defense-in-depth
+ * for a future call site, not a response to an actual observed leak. */
+const SECRET_CONTEXT_KEY = /token|secret|password|passwd|dsn|credential|api[_-]?key|coldkey|hotkey|wallet/i;
+const REDACTED = "[redacted]";
+
+/** Drop any context value whose KEY looks secret-shaped; every other scalar passes through unchanged. Not
+ * recursive -- every real call site passes a flat object (see this file's header comment), and PostHog's
+ * own `properties` bag is itself flat by convention. */
+function scrubMinerContext(context: Record): Record {
+ const scrubbed: Record = {};
+ for (const [key, value] of Object.entries(context)) {
+ scrubbed[key] = SECRET_CONTEXT_KEY.test(key) ? REDACTED : value;
+ }
+ return scrubbed;
+}
+
+/** Initialize PostHog from `env` (default `process.env`). Returns whether it activated. Call once, as early
+ * as possible in a bin's startup -- after `loadMinerFileSecrets()` (so a `_FILE`-mounted key resolves
+ * first via that function's own generic `_FILE` scan, with no code change needed here) and before
+ * `installCliSignalHandlers()` (so a startup crash is still captured). */
+export async function initMinerPostHog(env: Record = process.env): Promise {
+ const apiKey = env.LOOPOVER_MINER_POSTHOG_API_KEY;
+ if (!apiKey) return false;
+ const { PostHog } = await import("posthog-node");
+ const host = env.LOOPOVER_MINER_POSTHOG_HOST ?? DEFAULT_POSTHOG_HOST;
+ client = new PostHog(apiKey, { host, flushAt: 1, flushInterval: 0 });
+ active = true;
+ return true;
+}
+
+/** Capture an error with optional structured context. No-op when PostHog is off. Never throws --
+ * `captureMinerError`'s own signature and never-throws contract, preserved so call sites don't churn. */
+export function captureMinerPostHogError(error: unknown, context?: Record): void {
+ if (!active || !client) return;
+ try {
+ client.captureException(
+ error instanceof Error ? error : new Error(String(error)),
+ MINER_POSTHOG_DISTINCT_ID,
+ context ? scrubMinerContext(context) : undefined,
+ );
+ } catch {
+ /* PostHog capture must never crash the caller it's instrumenting. */
+ }
+}
+
+/** Flush buffered events before the process exits. No-op when off. Never throws. */
+export async function flushMinerPostHog(): Promise {
+ if (!active || !client) return;
+ try {
+ await client.flush();
+ } catch {
+ /* Best-effort -- a flush failure must never block process exit. */
+ }
+}
+
+/** Capture AND flush before returning -- the crash-path convenience wrapper, mirroring
+ * `captureMinerErrorAndFlush`'s identical rationale: a bare `captureMinerPostHogError()` only QUEUES the
+ * event, and `process.exit()` tears the process down immediately afterward without waiting for any
+ * pending delivery. */
+export async function captureMinerPostHogErrorAndFlush(error: unknown, context?: Record): Promise {
+ captureMinerPostHogError(error, context);
+ await flushMinerPostHog();
+}
+
+/** One AMS coding-agent driver attempt (#8296 AMS follow-up, epic #8286 track 3). All three driver types
+ * (claude-cli/codex-cli/agent-sdk, packages/loopover-engine's CodingAgentDriver) report a single blended
+ * `costUsd`/`tokensUsed` -- unlike ORB's self-host `AiUsage`, there is no input/output split available at
+ * this layer, so this deliberately does NOT populate `$ai_input_tokens`/`$ai_output_tokens` with a
+ * fabricated split (they stay 0, honestly representing "no split known"); the real total rides in the
+ * plain `tokens_used` property instead. `$ai_total_cost_usd` IS one of PostHog's own recognized
+ * `$ai_generation` properties and needs no split, so it's populated directly when known. No field here
+ * ever carries prompt/diff/transcript content -- metadata only, same policy as the ORB side. */
+export type MinerAiGenerationEvent = {
+ provider: string;
+ model: string;
+ latencyMs: number;
+ isError: boolean;
+ totalTokens?: number | undefined;
+ totalCostUsd?: number | undefined;
+ error?: unknown;
+};
+
+/** Capture one AMS coding-agent attempt as PostHog's `$ai_generation` event. No-op when PostHog is off --
+ * same contract as every other capture function in this file. */
+export function captureMinerPostHogAiGeneration(event: MinerAiGenerationEvent): void {
+ if (!active || !client) return;
+ const properties: Record = {
+ $ai_trace_id: randomUUID(),
+ $ai_model: event.model.trim() || "unknown",
+ $ai_provider: event.provider.trim() || "unknown",
+ // PostHog's own $ai_generation schema reports latency in SECONDS, not ms.
+ $ai_latency: event.latencyMs / 1000,
+ $ai_http_status: event.isError ? 500 : 200,
+ $ai_input_tokens: 0,
+ $ai_output_tokens: 0,
+ $ai_is_error: event.isError,
+ };
+ if (Number.isFinite(event.totalTokens)) properties.tokens_used = event.totalTokens;
+ if (Number.isFinite(event.totalCostUsd)) properties.$ai_total_cost_usd = event.totalCostUsd;
+ if (event.isError) {
+ const error = event.error instanceof Error ? event.error : new Error(String(event.error));
+ properties.$ai_error = error.message.slice(0, 500);
+ }
+ try {
+ client.capture({ distinctId: MINER_POSTHOG_DISTINCT_ID, event: "$ai_generation", properties });
+ } catch {
+ /* PostHog capture must never crash the caller it's instrumenting. */
+ }
+}
+
+/** Test-only: reset module state so one test's activation can't leak into the next. */
+export function resetMinerPostHogForTesting(): void {
+ client = undefined;
+ active = false;
+}
diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts
index 16ab184d75..427c2a82e6 100644
--- a/src/selfhost/ai.ts
+++ b/src/selfhost/ai.ts
@@ -11,6 +11,7 @@ import type { AiContentBlock, CombineStrategy, OnMerge } from "../services/ai-re
import { isConfiguredSelfHostProvider, resolveConfiguredProviderNames } from "./ai-config";
export { assertNoLegacySharedAiEnv } from "./ai-config";
import { incr, observe } from "./metrics";
+import { capturePostHogAiGeneration, type PostHogAiGenerationRequestKind } from "./posthog";
import { withReviewSpan } from "./tracing";
import { delimiter } from "node:path";
@@ -1337,23 +1338,37 @@ async function runProviderWithOtel(
request_kind: requestKindLabel,
});
aiProviderCircuits.delete(provider.name);
- if (result.usage) {
- return {
- ...result,
- usage: {
- ...result.usage,
- provider: result.usage.provider ?? provider.name,
- model: result.usage.model ?? (model || "default"),
- },
- };
- }
- return result;
+ const usage = result.usage
+ ? { ...result.usage, provider: result.usage.provider ?? provider.name, model: result.usage.model ?? (model || "default") }
+ : undefined;
+ capturePostHogAiGeneration({
+ provider: usage?.provider ?? provider.name,
+ model: usage?.model ?? (model || "default"),
+ requestKind: requestKindLabel,
+ latencyMs: Date.now() - startedAtMs,
+ isError: false,
+ inputTokens: usage?.inputTokens,
+ outputTokens: usage?.outputTokens,
+ totalCostUsd: usage?.costUsd,
+ effort: usage?.effort,
+ context: { repo: options.repoFullName, pullNumber: options.pullNumber },
+ });
+ return usage ? { ...result, usage } : result;
} catch (error) {
observe("loopover_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, {
provider: provider.name,
request_kind: requestKindLabel,
});
if (isExpectedEmbeddingRoutingError(options, error)) throw error;
+ capturePostHogAiGeneration({
+ provider: provider.name,
+ model: model || "default",
+ requestKind: requestKindLabel,
+ latencyMs: Date.now() - startedAtMs,
+ isError: true,
+ error,
+ context: { repo: options.repoFullName, pullNumber: options.pullNumber },
+ });
incr("loopover_ai_provider_failures_total", { provider: provider.name });
incr("loopover_ai_provider_request_errors_total", { provider: provider.name, request_kind: requestKindLabel });
// Re-read the map here rather than reusing the `circuit` captured above: that read happened BEFORE the
@@ -1481,6 +1496,50 @@ export function createSelfHostAi(env: Record): SelfH
return routeProviders(providers);
}
+/** Wrap a standalone `SelfHostAi` binding with the same `$ai_generation`/`$ai_embedding` PostHog capture
+ * {@link runProviderWithOtel} gives the main provider chain (#8296). AI_EMBED/AI_VISION/AI_ADVISORY
+ * (server.ts) are each a single, unrouted provider built directly by `createOpenAiCompatibleAi` -- they
+ * never pass through `buildProviders`/`routeProviders`/`runProviderWithOtel`, so before this wrapper they
+ * had zero per-call spend/latency visibility beyond a one-time boot log. No circuit breaker or chain
+ * metrics here (those are the main-chain's own concerns) -- this ONLY adds the capture call, matching this
+ * file's existing separation between chain-level concerns and the leaf capture itself. */
+export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): SelfHostAi {
+ return {
+ async run(model, options) {
+ const requestKindLabel: PostHogAiGenerationRequestKind = requestKind(options);
+ const startedAtMs = Date.now();
+ try {
+ const result = await ai.run(model, options);
+ const usage = result.usage
+ ? { ...result.usage, provider: result.usage.provider ?? providerName, model: result.usage.model ?? (model || "default") }
+ : undefined;
+ capturePostHogAiGeneration({
+ provider: usage?.provider ?? providerName,
+ model: usage?.model ?? (model || "default"),
+ requestKind: requestKindLabel,
+ latencyMs: Date.now() - startedAtMs,
+ isError: false,
+ inputTokens: usage?.inputTokens,
+ outputTokens: usage?.outputTokens,
+ totalCostUsd: usage?.costUsd,
+ effort: usage?.effort,
+ });
+ return usage ? { ...result, usage } : result;
+ } catch (error) {
+ capturePostHogAiGeneration({
+ provider: providerName,
+ model: model || "default",
+ requestKind: requestKindLabel,
+ latencyMs: Date.now() - startedAtMs,
+ isError: true,
+ error,
+ });
+ throw error;
+ }
+ },
+ };
+}
+
const COMBINE_STRATEGIES = new Set(["single", "consensus", "synthesis"]);
const ON_MERGE_RULES = new Set(["either", "both"]);
const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
diff --git a/src/selfhost/posthog.ts b/src/selfhost/posthog.ts
index 3683f60ae0..3741a0fffb 100644
--- a/src/selfhost/posthog.ts
+++ b/src/selfhost/posthog.ts
@@ -18,6 +18,7 @@
// POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT, POSTHOG_SERVER_NAME, POSTHOG_RELEASE) is self-host-only,
// read off real process.env, never added to src/env.d.ts's typed Env, matching that file's precedent for
// self-host-exclusive config.
+import { randomUUID } from "node:crypto";
import { hostname } from "node:os";
import {
currentOtelTraceIds,
@@ -319,6 +320,72 @@ export async function withPostHogMonitor(name: PostHogMonitorName, context: R
}
}
+/** `"review"` (chat/completion) vs `"embedding"` -- matches ai.ts's own `requestKind()` classification
+ * (`options.text` set ⇒ embedding), which picks the PostHog event name below. */
+export type PostHogAiGenerationRequestKind = "review" | "embedding";
+
+/** One AI provider attempt (#8296, epic #8286 track 3): every field here is metadata -- model id, provider
+ * name, timing, token/cost accounting, and (on failure) an already-redacted error string ai.ts's own
+ * `redactSecrets`/`errorMessage` produced before ever throwing. There is deliberately no field for the
+ * prompt or the response text; `$ai_generation`'s own optional content properties are never populated. */
+export type PostHogAiGenerationEvent = {
+ provider: string;
+ model: string;
+ requestKind: PostHogAiGenerationRequestKind;
+ latencyMs: number;
+ isError: boolean;
+ inputTokens?: number | undefined;
+ outputTokens?: number | undefined;
+ /** A single blended figure -- ai.ts's own `AiUsage.costUsd` never splits input/output cost, so this
+ * never fabricates a split its source data doesn't have. Absent for the claude-code/codex subscription
+ * CLIs whenever their own stdout reports none (a flat subscription has no real per-call dollar cost). */
+ totalCostUsd?: number | undefined;
+ /** Reasoning-effort dial ("low"/"medium"/"high"/"max") -- generation CONFIG, never prompt content. */
+ effort?: string | undefined;
+ /** Correlation context (repo/PR), the same optional fields AiRunOptions already threads through for
+ * `logSelfHostAiProviderFailed` -- routed through {@link operationalProperties} so only the shared
+ * operational-tag allowlist survives, exactly like every other capture path in this file. */
+ context?: Record | undefined;
+ /** Raw caught value on the error path (mirrors capturePostHogError's own `error` param) -- never a
+ * caller-preformatted string. Ignored when `isError` is false. */
+ error?: unknown;
+};
+
+/** Capture one AI provider attempt as PostHog's `$ai_generation` (`$ai_embedding` for an embedding
+ * request) event (#8296). No-op when PostHog is off -- same contract as every other capture function in
+ * this file. Unlike {@link capturePostHogError}, this never gates on POSTHOG_MIN_SEVERITY: a successful
+ * generation is not an error-severity event at all, and a failed one is ALREADY captured as a real
+ * exception by the caller's own existing `selfhost_ai_provider_failed` log line (forwarded via
+ * {@link forwardStructuredLogToPostHog}) -- this event exists for spend/latency/failure ANALYTICS, a
+ * parallel concern to error tracking, not a substitute gate for it. */
+export function capturePostHogAiGeneration(event: PostHogAiGenerationEvent): void {
+ if (!active || !client) return;
+ const properties: Record = {
+ ...operationalProperties(event.context),
+ $ai_trace_id: randomUUID(),
+ $ai_model: nonBlank(event.model) ?? "unknown",
+ $ai_provider: nonBlank(event.provider) ?? "unknown",
+ // PostHog's own $ai_generation schema reports latency in SECONDS, not ms.
+ $ai_latency: event.latencyMs / 1000,
+ $ai_http_status: event.isError ? 500 : 200,
+ $ai_input_tokens: Number.isFinite(event.inputTokens) ? event.inputTokens : 0,
+ $ai_output_tokens: Number.isFinite(event.outputTokens) ? event.outputTokens : 0,
+ $ai_is_error: event.isError,
+ environment: posthogEnvironment,
+ };
+ if (Number.isFinite(event.totalCostUsd)) properties.$ai_total_cost_usd = event.totalCostUsd;
+ if (event.effort) properties.$ai_model_parameters = { effort: event.effort };
+ if (event.isError) {
+ const error = event.error instanceof Error ? event.error : new Error(String(event.error));
+ properties.$ai_error = error.message.slice(0, 500);
+ }
+ client.capture({
+ distinctId: POSTHOG_DISTINCT_ID,
+ event: event.requestKind === "embedding" ? "$ai_embedding" : "$ai_generation",
+ properties,
+ });
+}
+
/** Flush buffered events before exit. No-op when off. */
export async function flushPostHog(): Promise {
if (!active || !client) return;
diff --git a/src/server.ts b/src/server.ts
index 326ece02d7..9711783acd 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -27,6 +27,7 @@ import {
resolveSubscriptionCliPath,
shouldMarkAiProviderUnhealthyAtBoot,
subscriptionCliEnv,
+ withAiGenerationCapture,
} from "./selfhost/ai";
import {
cookieValue,
@@ -572,12 +573,15 @@ async function main(): Promise {
// the review chain — so a Claude/Codex outage never falls reviews back to a weak local model. Unset ⇒ absent ⇒
// createReviewAdapters falls back to the review `ai` for embeds (byte-identical to before).
const embedAi = process.env.AI_EMBED_BASE_URL
- ? createOpenAiCompatibleAi({
- baseUrl: process.env.AI_EMBED_BASE_URL,
- apiKey: process.env.AI_EMBED_API_KEY ?? process.env.OPENAI_API_KEY,
- embedModel: process.env.AI_EMBED_MODEL,
- providerName: providerNameFromBaseUrl(process.env.AI_EMBED_BASE_URL),
- })
+ ? withAiGenerationCapture(
+ "ai_embed",
+ createOpenAiCompatibleAi({
+ baseUrl: process.env.AI_EMBED_BASE_URL,
+ apiKey: process.env.AI_EMBED_API_KEY ?? process.env.OPENAI_API_KEY,
+ embedModel: process.env.AI_EMBED_MODEL,
+ providerName: providerNameFromBaseUrl(process.env.AI_EMBED_BASE_URL),
+ }),
+ )
: undefined;
if (embedAi)
console.log(
@@ -593,12 +597,15 @@ async function main(): Promise {
// different model, a different capability) the same way AI_EMBED is kept separate from the review chain.
// Unset ⇒ absent ⇒ visual-vision falls back to BYOK-only (byte-identical to before this binding existed).
const visionAi = process.env.AI_VISION_BASE_URL
- ? createOpenAiCompatibleAi({
- baseUrl: process.env.AI_VISION_BASE_URL,
- apiKey: process.env.AI_VISION_API_KEY ?? process.env.OPENAI_API_KEY,
- model: process.env.AI_VISION_MODEL,
- providerName: providerNameFromBaseUrl(process.env.AI_VISION_BASE_URL),
- })
+ ? withAiGenerationCapture(
+ "ai_vision",
+ createOpenAiCompatibleAi({
+ baseUrl: process.env.AI_VISION_BASE_URL,
+ apiKey: process.env.AI_VISION_API_KEY ?? process.env.OPENAI_API_KEY,
+ model: process.env.AI_VISION_MODEL,
+ providerName: providerNameFromBaseUrl(process.env.AI_VISION_BASE_URL),
+ }),
+ )
: undefined;
if (visionAi)
console.log(
@@ -616,12 +623,15 @@ async function main(): Promise {
// config-driven, not hardcoded. Unset ⇒ absent ⇒ every advisory capability falls back to env.AI, byte-
// identical to before this binding existed.
const advisoryAi = process.env.AI_ADVISORY_BASE_URL
- ? createOpenAiCompatibleAi({
- baseUrl: process.env.AI_ADVISORY_BASE_URL,
- apiKey: process.env.AI_ADVISORY_API_KEY,
- model: process.env.AI_ADVISORY_MODEL,
- providerName: providerNameFromBaseUrl(process.env.AI_ADVISORY_BASE_URL),
- })
+ ? withAiGenerationCapture(
+ "ai_advisory",
+ createOpenAiCompatibleAi({
+ baseUrl: process.env.AI_ADVISORY_BASE_URL,
+ apiKey: process.env.AI_ADVISORY_API_KEY,
+ model: process.env.AI_ADVISORY_MODEL,
+ providerName: providerNameFromBaseUrl(process.env.AI_ADVISORY_BASE_URL),
+ }),
+ )
: undefined;
if (advisoryAi)
console.log(
diff --git a/test/unit/miner-coding-agent-construction.test.ts b/test/unit/miner-coding-agent-construction.test.ts
index 0b17998ae5..68e578d956 100644
--- a/test/unit/miner-coding-agent-construction.test.ts
+++ b/test/unit/miner-coding-agent-construction.test.ts
@@ -1,11 +1,34 @@
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@loopover/engine", async () => {
return import("../../packages/loopover-engine/src/index");
});
-import { createRealCliSubprocessSpawn, constructProductionCodingAgentDriver } from "../../packages/loopover-miner/lib/coding-agent-construction.js";
-import type { AgentSdkQueryFn, CodingAgentDriverTask } from "../../packages/loopover-engine/src/index";
+const posthogMock = vi.hoisted(() => {
+ const capture = vi.fn();
+ const PostHog = vi.fn(function (this: any) {
+ this.capture = capture;
+ this.flush = vi.fn().mockResolvedValue(undefined);
+ });
+ return { capture, PostHog };
+});
+vi.mock("posthog-node", () => ({ PostHog: posthogMock.PostHog }));
+
+import {
+ createRealCliSubprocessSpawn,
+ constructProductionCodingAgentDriver,
+ withCodingAgentAiGenerationCapture,
+} from "../../packages/loopover-miner/lib/coding-agent-construction.js";
+import { initMinerPostHog, resetMinerPostHogForTesting } from "../../packages/loopover-miner/lib/posthog.js";
+import type { AgentSdkQueryFn, CodingAgentDriver, CodingAgentDriverTask } from "../../packages/loopover-engine/src/index";
+
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+afterEach(() => {
+ resetMinerPostHogForTesting();
+});
const task: CodingAgentDriverTask = {
attemptId: "attempt-1",
@@ -175,3 +198,91 @@ describe("constructProductionCodingAgentDriver (#5131)", () => {
expect(append).toHaveBeenCalledWith(expect.objectContaining({ repoFullName: "acme/widgets" }));
});
});
+
+describe("withCodingAgentAiGenerationCapture (#8296 AMS follow-up)", () => {
+ function driverReturning(result: Awaited>): CodingAgentDriver {
+ return { run: async () => result };
+ }
+
+ it("stays a no-op end-to-end when PostHog is unconfigured", async () => {
+ const driver = withCodingAgentAiGenerationCapture("claude-cli", "claude-sonnet-5", driverReturning({ ok: true, changedFiles: [], summary: "done", transcript: "" }));
+ const result = await driver.run(task);
+ expect(result.ok).toBe(true);
+ expect(posthogMock.capture).not.toHaveBeenCalled();
+ });
+
+ it("captures a successful attempt's cost/tokens as a combined figure (no fabricated input/output split)", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const driver = withCodingAgentAiGenerationCapture(
+ "claude-cli",
+ "claude-sonnet-5",
+ driverReturning({ ok: true, changedFiles: ["a.ts"], summary: "done", transcript: "", costUsd: 0.12, tokensUsed: 4000 }),
+ );
+ await driver.run(task);
+ expect(posthogMock.capture).toHaveBeenCalledTimes(1);
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_provider).toBe("claude-cli");
+ expect(properties.$ai_model).toBe("claude-sonnet-5");
+ expect(properties.$ai_is_error).toBe(false);
+ expect(properties.tokens_used).toBe(4000);
+ expect(properties.$ai_total_cost_usd).toBe(0.12);
+ });
+
+ it("captures result.ok:false as a failure, using the driver's own error string -- no exception thrown", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const driver = withCodingAgentAiGenerationCapture(
+ "codex-cli",
+ "gpt-5-codex",
+ driverReturning({ ok: false, changedFiles: [], summary: "failed", transcript: "", error: "codex_timeout_120000ms" }),
+ );
+ const result = await driver.run(task);
+ expect(result.ok).toBe(false);
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_is_error).toBe(true);
+ expect(properties.$ai_http_status).toBe(500);
+ expect(properties.$ai_error).toBe("codex_timeout_120000ms");
+ });
+
+ it("REGRESSION: still captures a failure AND rethrows when the wrapped driver itself throws unexpectedly", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const driver = withCodingAgentAiGenerationCapture("agent-sdk", "agent-sdk", { run: async () => { throw new Error("sdk crashed"); } });
+ await expect(driver.run(task)).rejects.toThrow("sdk crashed");
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_is_error).toBe(true);
+ expect(properties.$ai_error).toBe("sdk crashed");
+ });
+});
+
+describe("constructProductionCodingAgentDriver -- $ai_generation capture wiring (#8296 AMS follow-up)", () => {
+ it("captures using the configured model env var for a CLI provider", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const driver = constructProductionCodingAgentDriver(
+ { MINER_CODING_AGENT_PROVIDER: "claude-cli", MINER_CODING_AGENT_CLAUDE_MODEL: "claude-opus-5" },
+ { spawn: async () => ({ stdout: "done", code: 0 }) },
+ );
+ await driver.run(task);
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_provider).toBe("claude-cli");
+ expect(properties.$ai_model).toBe("claude-opus-5");
+ });
+
+ it("falls back to the provider name as the model when no model env var is configured (e.g. agent-sdk, which declares none)", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const driver = constructProductionCodingAgentDriver(
+ { MINER_CODING_AGENT_PROVIDER: "agent-sdk" },
+ { query: queryCapturing({}), listChangedFiles: async () => [] },
+ );
+ await driver.run(task);
+ expect(posthogMock.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("agent-sdk");
+ });
+
+ it("falls back to the provider name when a CLI provider's own model env var is configured but blank", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const driver = constructProductionCodingAgentDriver(
+ { MINER_CODING_AGENT_PROVIDER: "codex-cli", MINER_CODING_AGENT_CODEX_MODEL: "" },
+ { spawn: async () => ({ stdout: "done", code: 0 }) },
+ );
+ await driver.run(task);
+ expect(posthogMock.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("codex-cli");
+ });
+});
diff --git a/test/unit/miner-egress-allowlist.test.ts b/test/unit/miner-egress-allowlist.test.ts
index a608f15219..ea0be2d036 100644
--- a/test/unit/miner-egress-allowlist.test.ts
+++ b/test/unit/miner-egress-allowlist.test.ts
@@ -91,6 +91,26 @@ describe("resolveEgressAllowlist (#7857)", () => {
expect(entries).toContainEqual({ host: "o1.ingest.sentry.io", reason: "loopover-platform" });
});
+ it("adds the PostHog default host only when LOOPOVER_MINER_POSTHOG_API_KEY is set, with no host override (#8292)", () => {
+ expect(resolveEgressAllowlist(EMPTY_ALLOWLIST, {}).some((e) => e.host === "us.i.posthog.com")).toBe(false);
+ const entries = resolveEgressAllowlist(EMPTY_ALLOWLIST, { LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test" });
+ expect(entries).toContainEqual({ host: "us.i.posthog.com", reason: "loopover-platform" });
+ });
+
+ it("adds the overridden PostHog host when LOOPOVER_MINER_POSTHOG_HOST is also set", () => {
+ const entries = resolveEgressAllowlist(EMPTY_ALLOWLIST, {
+ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test",
+ LOOPOVER_MINER_POSTHOG_HOST: "https://eu.i.posthog.com",
+ });
+ expect(entries).toContainEqual({ host: "eu.i.posthog.com", reason: "loopover-platform" });
+ expect(entries.some((e) => e.host === "us.i.posthog.com")).toBe(false);
+ });
+
+ it("falls back to the PostHog default host when LOOPOVER_MINER_POSTHOG_HOST is malformed but the API key is set", () => {
+ const entries = resolveEgressAllowlist(EMPTY_ALLOWLIST, { LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test", LOOPOVER_MINER_POSTHOG_HOST: "not a url" });
+ expect(entries).toContainEqual({ host: "us.i.posthog.com", reason: "loopover-platform" });
+ });
+
it("adds Neon's API host only when ALL THREE LOOPOVER_MINER_NEON_* vars are set (all-or-nothing, matching resolveAttemptDbForkConfig's own gate)", () => {
expect(
resolveEgressAllowlist(EMPTY_ALLOWLIST, { LOOPOVER_MINER_NEON_API_KEY: "k", LOOPOVER_MINER_NEON_PROJECT_ID: "p" }).some((e) => e.reason === "loopover-platform"),
diff --git a/test/unit/miner-posthog.test.ts b/test/unit/miner-posthog.test.ts
new file mode 100644
index 0000000000..b8aacf199d
--- /dev/null
+++ b/test/unit/miner-posthog.test.ts
@@ -0,0 +1,248 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const posthogMock = vi.hoisted(() => {
+ const captureException = vi.fn();
+ const capture = vi.fn();
+ const flush = vi.fn().mockResolvedValue(undefined);
+ let lastArgs: any;
+ const PostHog = vi.fn(function (this: any, apiKey: string, options: any) {
+ lastArgs = { apiKey, options };
+ this.captureException = captureException;
+ this.capture = capture;
+ this.flush = flush;
+ });
+ return { captureException, capture, flush, PostHog, getLastArgs: () => lastArgs };
+});
+
+vi.mock("posthog-node", () => ({ PostHog: posthogMock.PostHog }));
+
+import {
+ captureMinerPostHogAiGeneration,
+ captureMinerPostHogError,
+ captureMinerPostHogErrorAndFlush,
+ flushMinerPostHog,
+ initMinerPostHog,
+ resetMinerPostHogForTesting,
+} from "../../packages/loopover-miner/lib/posthog.js";
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ posthogMock.flush.mockResolvedValue(undefined);
+});
+
+afterEach(() => {
+ resetMinerPostHogForTesting();
+});
+
+describe("loopover-miner opt-in PostHog (#8292, epic #8286)", () => {
+ describe("off state (no API key)", () => {
+ it("stays fully off when LOOPOVER_MINER_POSTHOG_API_KEY is unset", async () => {
+ expect(await initMinerPostHog({})).toBe(false);
+ expect(posthogMock.PostHog).not.toHaveBeenCalled();
+ expect(() => captureMinerPostHogError(new Error("x"))).not.toThrow();
+ expect(posthogMock.captureException).not.toHaveBeenCalled();
+ await expect(flushMinerPostHog()).resolves.toBeUndefined();
+ expect(posthogMock.flush).not.toHaveBeenCalled();
+ });
+
+ it("REGRESSION: an empty-string API key is treated the same as unset (never activates)", async () => {
+ expect(await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "" })).toBe(false);
+ expect(posthogMock.PostHog).not.toHaveBeenCalled();
+ });
+
+ it("captureMinerPostHogError never throws even when called before initMinerPostHog (default off state)", () => {
+ expect(() => captureMinerPostHogError("a plain string, not an Error")).not.toThrow();
+ expect(() => captureMinerPostHogError(new Error("boom"), { kind: "test" })).not.toThrow();
+ });
+
+ it("defaults to process.env when no env argument is passed", async () => {
+ const original = process.env.LOOPOVER_MINER_POSTHOG_API_KEY;
+ delete process.env.LOOPOVER_MINER_POSTHOG_API_KEY;
+ try {
+ expect(await initMinerPostHog()).toBe(false);
+ } finally {
+ if (original === undefined) delete process.env.LOOPOVER_MINER_POSTHOG_API_KEY;
+ else process.env.LOOPOVER_MINER_POSTHOG_API_KEY = original;
+ }
+ });
+ });
+
+ describe("activated state (API key set)", () => {
+ it("initializes posthog-node with the key, default host, and short-lived-process flush semantics", async () => {
+ expect(await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" })).toBe(true);
+ expect(posthogMock.PostHog).toHaveBeenCalledWith("phc_test_key", {
+ host: "https://us.i.posthog.com",
+ flushAt: 1,
+ flushInterval: 0,
+ });
+ });
+
+ it("uses LOOPOVER_MINER_POSTHOG_HOST when set", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key", LOOPOVER_MINER_POSTHOG_HOST: "https://eu.i.posthog.com" });
+ expect(posthogMock.getLastArgs().options.host).toBe("https://eu.i.posthog.com");
+ });
+
+ it("captureMinerPostHogError wraps a non-Error value and forwards it via captureException", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogError("plain string reason");
+ expect(posthogMock.captureException).toHaveBeenCalledTimes(1);
+ const captured = posthogMock.captureException.mock.calls[0]?.[0] as Error;
+ expect(captured).toBeInstanceOf(Error);
+ expect(captured.message).toBe("plain string reason");
+ });
+
+ it("captureMinerPostHogError forwards a real Error instance as-is (not re-wrapped)", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const original = new Error("real error");
+ captureMinerPostHogError(original);
+ expect(posthogMock.captureException.mock.calls[0]?.[0]).toBe(original);
+ });
+
+ it("passes context as properties only when context is provided (both sides of the branch)", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogError(new Error("with context"), { kind: "test_kind", repoFullName: "acme/widgets" });
+ expect(posthogMock.captureException).toHaveBeenCalledWith(expect.any(Error), "loopover-miner", { kind: "test_kind", repoFullName: "acme/widgets" });
+
+ captureMinerPostHogError(new Error("no context"));
+ expect(posthogMock.captureException).toHaveBeenLastCalledWith(expect.any(Error), "loopover-miner", undefined);
+ });
+
+ it("redacts a secret-shaped context key's VALUE, leaving its sibling keys untouched", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogError(new Error("boom"), { kind: "test", apiKey: "sk-should-not-leak", repoFullName: "acme/widgets" });
+ const properties = posthogMock.captureException.mock.calls[0]?.[2] as Record;
+ expect(properties.apiKey).toBe("[redacted]");
+ expect(properties.kind).toBe("test");
+ expect(properties.repoFullName).toBe("acme/widgets");
+ });
+
+ it("REGRESSION: captureMinerPostHogError never throws even when the client itself throws", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ posthogMock.captureException.mockImplementationOnce(() => {
+ throw new Error("posthog sdk internal failure");
+ });
+ expect(() => captureMinerPostHogError(new Error("boom"))).not.toThrow();
+ });
+
+ it("flushMinerPostHog calls the client's flush", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ await flushMinerPostHog();
+ expect(posthogMock.flush).toHaveBeenCalledTimes(1);
+ });
+
+ it("REGRESSION: flushMinerPostHog never throws or rejects even when flush itself rejects", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ posthogMock.flush.mockRejectedValueOnce(new Error("flush timed out"));
+ await expect(flushMinerPostHog()).resolves.toBeUndefined();
+ });
+ });
+
+ it("resetMinerPostHogForTesting returns an activated instance to the default-off no-op", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ resetMinerPostHogForTesting();
+ captureMinerPostHogError(new Error("after reset"));
+ expect(posthogMock.captureException).not.toHaveBeenCalled();
+ await flushMinerPostHog();
+ expect(posthogMock.flush).not.toHaveBeenCalled();
+ });
+
+ describe("captureMinerPostHogAiGeneration (#8296 AMS follow-up)", () => {
+ const BASE = { provider: "claude-cli", model: "claude-sonnet-5", latencyMs: 2500, isError: false };
+
+ it("is a no-op when PostHog is unconfigured", () => {
+ expect(() => captureMinerPostHogAiGeneration(BASE)).not.toThrow();
+ expect(posthogMock.capture).not.toHaveBeenCalled();
+ });
+
+ it("captures a well-formed $ai_generation event with a combined tokens_used property (no fabricated input/output split)", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogAiGeneration({ ...BASE, totalTokens: 1500, totalCostUsd: 0.05 });
+ expect(posthogMock.capture).toHaveBeenCalledTimes(1);
+ const call = posthogMock.capture.mock.calls[0]?.[0];
+ expect(call.event).toBe("$ai_generation");
+ expect(call.distinctId).toBe("loopover-miner");
+ expect(call.properties.$ai_model).toBe("claude-sonnet-5");
+ expect(call.properties.$ai_provider).toBe("claude-cli");
+ expect(call.properties.$ai_latency).toBe(2.5);
+ expect(call.properties.$ai_http_status).toBe(200);
+ expect(call.properties.$ai_input_tokens).toBe(0);
+ expect(call.properties.$ai_output_tokens).toBe(0);
+ expect(call.properties.tokens_used).toBe(1500);
+ expect(call.properties.$ai_total_cost_usd).toBe(0.05);
+ expect(call.properties.$ai_is_error).toBe(false);
+ expect("$ai_input" in call.properties).toBe(false);
+ expect("$ai_output_choices" in call.properties).toBe(false);
+ });
+
+ it("omits tokens_used/$ai_total_cost_usd when neither is supplied or finite", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogAiGeneration({ ...BASE, totalTokens: Number.NaN });
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect("tokens_used" in properties).toBe(false);
+ expect("$ai_total_cost_usd" in properties).toBe(false);
+ });
+
+ it("falls back to 'unknown' for a blank model/provider", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogAiGeneration({ ...BASE, model: "", provider: " " });
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_model).toBe("unknown");
+ expect(properties.$ai_provider).toBe("unknown");
+ });
+
+ it("marks a failed generation with $ai_is_error/$ai_http_status/$ai_error, redacted to 500 chars", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogAiGeneration({ ...BASE, isError: true, error: new Error("x".repeat(600)) });
+ const { properties } = posthogMock.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_is_error).toBe(true);
+ expect(properties.$ai_http_status).toBe(500);
+ expect(properties.$ai_error).toHaveLength(500);
+ });
+
+ it("wraps a non-Error thrown value on the error path", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogAiGeneration({ ...BASE, isError: true, error: "just a string" });
+ expect(posthogMock.capture.mock.calls[0]?.[0].properties.$ai_error).toBe("just a string");
+ });
+
+ it("REGRESSION: never throws even when the client itself throws", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ posthogMock.capture.mockImplementationOnce(() => {
+ throw new Error("posthog sdk internal failure");
+ });
+ expect(() => captureMinerPostHogAiGeneration(BASE)).not.toThrow();
+ });
+
+ it("mints a fresh, real UUID trace id on every call", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ captureMinerPostHogAiGeneration(BASE);
+ captureMinerPostHogAiGeneration(BASE);
+ const first = posthogMock.capture.mock.calls[0]?.[0].properties.$ai_trace_id;
+ const second = posthogMock.capture.mock.calls[1]?.[0].properties.$ai_trace_id;
+ expect(first).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
+ expect(first).not.toBe(second);
+ });
+ });
+
+ describe("captureMinerPostHogErrorAndFlush (the crash-path convenience wrapper)", () => {
+ it("REGRESSION: captures AND flushes, so a crash-path caller can await it before exiting instead of a bare capture that only queues the event", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ const error = new Error("crash");
+ await captureMinerPostHogErrorAndFlush(error, { kind: "uncaughtException" });
+ expect(posthogMock.captureException).toHaveBeenCalledWith(error, "loopover-miner", { kind: "uncaughtException" });
+ expect(posthogMock.flush).toHaveBeenCalledTimes(1);
+ });
+
+ it("resolves cleanly (no throw) when PostHog is off", async () => {
+ await expect(captureMinerPostHogErrorAndFlush(new Error("x"))).resolves.toBeUndefined();
+ expect(posthogMock.captureException).not.toHaveBeenCalled();
+ expect(posthogMock.flush).not.toHaveBeenCalled();
+ });
+
+ it("still resolves cleanly when the underlying flush rejects", async () => {
+ await initMinerPostHog({ LOOPOVER_MINER_POSTHOG_API_KEY: "phc_test_key" });
+ posthogMock.flush.mockRejectedValueOnce(new Error("flush timed out"));
+ await expect(captureMinerPostHogErrorAndFlush(new Error("crash"))).resolves.toBeUndefined();
+ });
+ });
+});
diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts
index 664952633a..38026ef5ea 100644
--- a/test/unit/selfhost-ai.test.ts
+++ b/test/unit/selfhost-ai.test.ts
@@ -2,9 +2,26 @@ import { chmodSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
-import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, codexErrorFromStdout, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, isAiProviderHealthy, markAiProviderUnhealthyAtBoot, providerNameFromBaseUrl, resetAiProviderCircuitBreakerForTest, resetAiProviderHealthForTest, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveClaudeFirstOutputTimeoutMs, resolveCodexAuthPath, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveCodexFirstOutputTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, withAdvisoryAiEnv, __selfHostAiInternals } from "../../src/selfhost/ai";
+
+// Mock posthog-node so ai.ts's new $ai_generation capture (via src/selfhost/posthog.ts) resolves to a
+// spy-backed client -- mirrors selfhost-posthog.test.ts's identical hoisted mocking pattern.
+const posthogMocks = vi.hoisted(() => {
+ const capture = vi.fn();
+ const captureException = vi.fn();
+ const PostHog = vi.fn(function (this: any) {
+ this.capture = capture;
+ this.captureException = captureException;
+ this.flush = vi.fn().mockResolvedValue(undefined);
+ this.shutdown = vi.fn().mockResolvedValue(undefined);
+ });
+ return { capture, captureException, PostHog };
+});
+vi.mock("posthog-node", () => ({ PostHog: posthogMocks.PostHog }));
+
+import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, codexErrorFromStdout, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, isAiProviderHealthy, markAiProviderUnhealthyAtBoot, providerNameFromBaseUrl, resetAiProviderCircuitBreakerForTest, resetAiProviderHealthForTest, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveClaudeFirstOutputTimeoutMs, resolveCodexAuthPath, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveCodexFirstOutputTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, withAdvisoryAiEnv, withAiGenerationCapture, __selfHostAiInternals } from "../../src/selfhost/ai";
import { labelSelfHostReviewerModel, labelSelfHostReviewerModels } from "../../src/selfhost/ai-config";
import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics";
+import { initPostHog, resetPostHogForTest } from "../../src/selfhost/posthog";
describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => {
const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast";
@@ -127,6 +144,8 @@ afterEach(() => {
resetMetrics();
resetAiProviderHealthForTest();
resetAiProviderCircuitBreakerForTest();
+ resetPostHogForTest();
+ vi.clearAllMocks();
});
type SpawnResult = { stdout: string; code: number | null; stderr?: string; timedOut?: boolean; stalledNoOutput?: boolean };
@@ -621,6 +640,142 @@ describe("AI provider request duration/error metrics (#4367)", () => {
});
});
+describe("$ai_generation PostHog capture at the chain chokepoint (#8296)", () => {
+ it("captures a well-formed $ai_generation event on a successful chain call, using the provider's own usage", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const provider = {
+ name: "posthog-ok-provider",
+ ai: { run: async () => ({ response: "ok", usage: { inputTokens: 10, outputTokens: 5, costUsd: 0.01, effort: "low" } }) },
+ };
+ await createChainAi([provider]).run("m", { prompt: "review this", repoFullName: "owner/repo", pullNumber: 3 });
+ expect(posthogMocks.capture).toHaveBeenCalledTimes(1);
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_generation");
+ expect(properties.$ai_provider).toBe("posthog-ok-provider");
+ expect(properties.$ai_model).toBe("m");
+ expect(properties.$ai_input_tokens).toBe(10);
+ expect(properties.$ai_output_tokens).toBe(5);
+ expect(properties.$ai_total_cost_usd).toBe(0.01);
+ expect(properties.$ai_model_parameters).toEqual({ effort: "low" });
+ expect(properties.$ai_is_error).toBe(false);
+ expect(properties.repo).toBe("owner/repo");
+ expect(properties.pullNumber).toBe(3);
+ });
+
+ it("captures $ai_embedding for an embedding request through the chain", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const provider = { name: "posthog-embed-provider", ai: { run: async () => ({ data: [[0.1, 0.2]] }) } };
+ await createChainAi([provider]).run("m", { text: ["chunk"] });
+ expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_embedding");
+ });
+
+ it("falls back to the provider's registry name when the result carries no usage", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const provider = { name: "posthog-no-usage-provider", ai: { run: async () => ({ response: "ok" }) } };
+ await createChainAi([provider]).run("m", { prompt: "x" });
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_provider).toBe("posthog-no-usage-provider");
+ expect(properties.$ai_input_tokens).toBe(0);
+ });
+
+ it("captures a failed generation with $ai_is_error/$ai_error on a real chain failure", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const provider = { name: "posthog-fail-provider", ai: { run: async () => { throw new Error("boom"); } } };
+ await expect(createChainAi([provider]).run("m", { prompt: "review this" })).rejects.toThrow(/boom/);
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_is_error).toBe(true);
+ expect(properties.$ai_error).toBe("boom");
+ expect(properties.$ai_provider).toBe("posthog-fail-provider");
+ });
+
+ it("does NOT capture for an expected embedding-routing fallback (mirrors the metrics exemption above)", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const provider = { name: "posthog-routing-provider", ai: { run: async () => { throw new Error("claude_code_no_embed"); } } };
+ await expect(createChainAi([provider]).run("m", { text: ["chunk"] })).rejects.toThrow();
+ expect(posthogMocks.capture).not.toHaveBeenCalled();
+ });
+
+ it("stays a no-op end-to-end when PostHog is unconfigured (no posthog-node client constructed)", async () => {
+ const provider = { name: "posthog-unconfigured-provider", ai: { run: async () => ({ response: "ok" }) } };
+ await createChainAi([provider]).run("m", { prompt: "x" });
+ expect(posthogMocks.capture).not.toHaveBeenCalled();
+ });
+});
+
+describe("withAiGenerationCapture (#8296 — AI_EMBED/AI_VISION/AI_ADVISORY, ungoverned single-provider bindings)", () => {
+ it("captures a successful call's usage, falling back to the given providerName when usage carries none", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_embed", { run: async () => ({ data: [[0.1]] }) });
+ const result = await ai.run("bge-m3", { text: ["chunk"] });
+ expect(result.data).toEqual([[0.1]]);
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_embedding");
+ expect(properties.$ai_provider).toBe("ai_embed");
+ expect(properties.$ai_model).toBe("bge-m3");
+ });
+
+ it("prefers the wrapped provider's own reported usage.provider/model over the given providerName", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_vision", {
+ run: async () => ({ response: "ok", usage: { provider: "ollama", model: "llava" } }),
+ });
+ await ai.run("configured-model", { prompt: "describe this image" });
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_provider).toBe("ollama");
+ expect(properties.$ai_model).toBe("llava");
+ });
+
+ it("captures a failure and still rethrows the original error", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_advisory", { run: async () => { throw new Error("advisory down"); } });
+ await expect(ai.run("m", { prompt: "x" })).rejects.toThrow("advisory down");
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_is_error).toBe(true);
+ expect(properties.$ai_provider).toBe("ai_advisory");
+ expect(properties.$ai_error).toBe("advisory down");
+ });
+
+ it("labels a chat (non-embedding) request kind as $ai_generation, not $ai_embedding", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_advisory", { run: async () => ({ response: "ok" }) });
+ await ai.run("m", { prompt: "x" });
+ expect(posthogMocks.capture.mock.calls[0]?.[0].event).toBe("$ai_generation");
+ });
+
+ it("falls back to providerName/'default' when usage is present but omits its own provider/model, and the model arg is empty", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_vision", { run: async () => ({ response: "ok", usage: { inputTokens: 3 } }) });
+ await ai.run("", { prompt: "x" });
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_provider).toBe("ai_vision");
+ expect(properties.$ai_model).toBe("default");
+ expect(properties.$ai_input_tokens).toBe(3);
+ });
+
+ it("falls back to the model arg (not 'default') when usage omits its own model but the model arg is truthy", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_vision", { run: async () => ({ response: "ok", usage: { inputTokens: 3 } }) });
+ await ai.run("configured-model", { prompt: "x" });
+ expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("configured-model");
+ });
+
+ it("falls back to 'default' model on the failure path when the model arg is empty", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_embed", { run: async () => { throw new Error("down"); } });
+ await expect(ai.run("", { text: ["chunk"] })).rejects.toThrow("down");
+ expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("default");
+ });
+
+ it("falls back to 'default' model when the result carries no usage at all AND the model arg is empty", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ const ai = withAiGenerationCapture("ai_advisory", { run: async () => ({ response: "ok" }) });
+ await ai.run("", { prompt: "x" });
+ const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_provider).toBe("ai_advisory");
+ expect(properties.$ai_model).toBe("default");
+ });
+});
+
describe("per-provider circuit breaker (#2540 — skip fast during a sustained outage)", () => {
afterEach(() => {
vi.useRealTimers();
diff --git a/test/unit/selfhost-posthog.test.ts b/test/unit/selfhost-posthog.test.ts
index 2b1b34161f..393b2093a6 100644
--- a/test/unit/selfhost-posthog.test.ts
+++ b/test/unit/selfhost-posthog.test.ts
@@ -24,6 +24,7 @@ vi.mock("../../src/selfhost/otel", () => ({
}));
import {
+ capturePostHogAiGeneration,
capturePostHogError,
capturePostHogReviewFailure,
flushPostHog,
@@ -445,6 +446,125 @@ describe("withPostHogMonitor", () => {
});
});
+describe("capturePostHogAiGeneration (#8296)", () => {
+ const BASE = { provider: "ollama", model: "llama3.1", requestKind: "review" as const, latencyMs: 1500, isError: false };
+
+ it("is a no-op when PostHog is unconfigured", () => {
+ capturePostHogAiGeneration(BASE);
+ expect(mocks.capture).not.toHaveBeenCalled();
+ });
+
+ it("captures a well-formed $ai_generation event with token/latency metadata on success", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, inputTokens: 120, outputTokens: 40, totalCostUsd: 0.002, effort: "medium" });
+ expect(mocks.capture).toHaveBeenCalledTimes(1);
+ const call = mocks.capture.mock.calls[0]?.[0];
+ expect(call.event).toBe("$ai_generation");
+ expect(call.properties.$ai_model).toBe("llama3.1");
+ expect(call.properties.$ai_provider).toBe("ollama");
+ expect(call.properties.$ai_latency).toBe(1.5); // ms -> seconds
+ expect(call.properties.$ai_http_status).toBe(200);
+ expect(call.properties.$ai_input_tokens).toBe(120);
+ expect(call.properties.$ai_output_tokens).toBe(40);
+ expect(call.properties.$ai_is_error).toBe(false);
+ expect(call.properties.$ai_total_cost_usd).toBe(0.002);
+ expect(call.properties.$ai_model_parameters).toEqual({ effort: "medium" });
+ expect("$ai_error" in call.properties).toBe(false);
+ });
+
+ it("captures $ai_embedding for an embedding request kind", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, requestKind: "embedding" });
+ expect(mocks.capture.mock.calls[0]?.[0].event).toBe("$ai_embedding");
+ });
+
+ it("defaults input/output tokens to 0 when omitted or non-finite", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, inputTokens: undefined, outputTokens: Number.NaN });
+ const { properties } = mocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_input_tokens).toBe(0);
+ expect(properties.$ai_output_tokens).toBe(0);
+ });
+
+ it("omits $ai_total_cost_usd when cost is not supplied or non-finite", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, totalCostUsd: Number.NaN });
+ expect("$ai_total_cost_usd" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false);
+ });
+
+ it("omits $ai_model_parameters when effort is not supplied", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration(BASE);
+ expect("$ai_model_parameters" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false);
+ });
+
+ it("falls back to 'unknown' for a blank model/provider", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, model: "", provider: " " });
+ const { properties } = mocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_model).toBe("unknown");
+ expect(properties.$ai_provider).toBe("unknown");
+ });
+
+ it("marks a failed generation with $ai_is_error/$ai_http_status/$ai_error, redacted to 500 chars", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, isError: true, error: new Error("x".repeat(600)) });
+ const { properties } = mocks.capture.mock.calls[0]?.[0];
+ expect(properties.$ai_is_error).toBe(true);
+ expect(properties.$ai_http_status).toBe(500);
+ expect(properties.$ai_error).toHaveLength(500);
+ });
+
+ it("wraps a non-Error thrown value on the error path", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, isError: true, error: "just a string" });
+ expect(mocks.capture.mock.calls[0]?.[0].properties.$ai_error).toBe("just a string");
+ });
+
+ it("never carries prompt/response content -- no field beyond model/provider ids", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration(BASE);
+ const keys = Object.keys(mocks.capture.mock.calls[0]?.[0].properties);
+ expect(keys).not.toContain("$ai_input");
+ expect(keys).not.toContain("$ai_output_choices");
+ });
+
+ it("mints a fresh, real UUID trace id on every call", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration(BASE);
+ capturePostHogAiGeneration(BASE);
+ const first = mocks.capture.mock.calls[0]?.[0].properties.$ai_trace_id;
+ const second = mocks.capture.mock.calls[1]?.[0].properties.$ai_trace_id;
+ expect(first).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
+ expect(first).not.toBe(second);
+ });
+
+ it("merges correlation context (repo/pullNumber) through the shared operational-tag allowlist", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, context: { repo: "owner/repo", pullNumber: 7 } });
+ const { properties } = mocks.capture.mock.calls[0]?.[0];
+ expect(properties.repo).toBe("owner/repo");
+ expect(properties.pullNumber).toBe(7);
+ });
+
+ it("drops an unlisted context key (not on OPERATIONAL_TAG_KEYS)", async () => {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, context: { notAllowlisted: "should be dropped" } });
+ expect("notAllowlisted" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false);
+ });
+
+ it("does not gate on POSTHOG_MIN_SEVERITY (unlike capturePostHogError)", async () => {
+ process.env.POSTHOG_MIN_SEVERITY = "critical";
+ try {
+ await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+ capturePostHogAiGeneration({ ...BASE, isError: true, error: new Error("boom") });
+ expect(mocks.capture).toHaveBeenCalledTimes(1);
+ } finally {
+ delete process.env.POSTHOG_MIN_SEVERITY;
+ }
+ });
+});
+
describe("flushPostHog / shutdownPostHog", () => {
it("flushPostHog is a no-op when unconfigured", async () => {
await flushPostHog();