From a7b49b20ed1159ae2f87ac1f3aa25b658a44a85e Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 25 Jul 2026 00:04:38 -0700 Subject: [PATCH] feat(selfhost): add PostHog error tracking as a parallel sink alongside Sentry Adds src/selfhost/posthog.ts, mirroring sentry.ts's capture surface (capturePostHogError, capturePostHogReviewFailure, structured-log forwarding, cron-monitor heartbeats) and wires it into every existing Sentry call site across the self-host server, queues, and monitors. Extracts the shared redaction primitives both sinks now depend on into src/selfhost/redaction-scrub.ts so the security-critical scrubbing logic isn't duplicated. Runs opt-in and fully parallel with Sentry per the epic's gated decommission plan (#8298) -- Sentry is untouched here. Closes #8287 --- .../content/docs/self-hosting-operations.mdx | 44 ++ .../src/lib/selfhost-env-reference.ts | 10 + src/env.d.ts | 22 +- src/queue/ai-review-orchestration.ts | 43 ++ src/queue/processors.ts | 20 + src/selfhost/monitored-work.ts | 22 +- src/selfhost/pg-queue.ts | 31 +- src/selfhost/posthog.ts | 342 ++++++++++++ src/selfhost/redaction-scrub.ts | 196 +++++++ src/selfhost/sentry.ts | 188 +------ src/selfhost/sqlite-queue.ts | 31 +- src/server.ts | 47 +- src/services/agent-action-executor.ts | 4 + ...ocs-selfhost-posthog-observability.test.ts | 35 ++ test/unit/selfhost-posthog.test.ts | 491 ++++++++++++++++++ 15 files changed, 1335 insertions(+), 191 deletions(-) create mode 100644 src/selfhost/posthog.ts create mode 100644 src/selfhost/redaction-scrub.ts create mode 100644 test/unit/docs-selfhost-posthog-observability.test.ts create mode 100644 test/unit/selfhost-posthog.test.ts diff --git a/apps/loopover-ui/content/docs/self-hosting-operations.mdx b/apps/loopover-ui/content/docs/self-hosting-operations.mdx index 9dc8812807..8df82316fd 100644 --- a/apps/loopover-ui/content/docs/self-hosting-operations.mdx +++ b/apps/loopover-ui/content/docs/self-hosting-operations.mdx @@ -549,6 +549,50 @@ changing Sentry env, restart the `loopover` service — there is no hot reload. home to a maintainer-owned project unless you configure one. +## Enabling PostHog error tracking (parallel run alongside Sentry) + +PostHog error tracking is **opt-in and off by default**, and runs **in parallel with Sentry +above** — both sinks are active simultaneously when both are configured; enabling one does not +disable the other. Leave `POSTHOG_API_KEY` unset for a complete no-op with negligible overhead. +The same key also activates MCP tool-call telemetry (`src/mcp/telemetry.ts`) — one PostHog +project covers both surfaces. + + + +Every captured event is scrubbed by the same redaction primitives Sentry's `beforeSend` uses +(`src/selfhost/redaction-scrub.ts`, shared by both sinks so there is one security-critical +implementation, not two independently-drifting copies): secret-shaped keys/values, JWTs, local +filesystem paths, and installation ids (hashed, never raw) are all redacted before an event +leaves the box. + +PostHog's own [exception autocapture](https://posthog.com/docs/error-tracking/installation/node) +is enabled, so genuinely uncaught exceptions and unhandled rejections are captured automatically +— mirroring Sentry's own default global-handler posture — in addition to the explicit capture +call sites mirrored throughout the codebase. + +There is no PostHog equivalent to Sentry Cron Monitors' missed-check-in alerting, so the five +recurring jobs (`scheduled-loop`, `orb-export`, `orb-relay-drain`, `orb-relay-register`, +`queue-dead-letter-revive`) instead emit a plain heartbeat event, `orb_monitor_heartbeat` +(properties: `monitor`, `status: "ok" | "error"`, `duration_ms`), on every run. Configure a +PostHog insight alert on a no-data condition over that event (filtered to a given monitor name) +to reproduce the same "silent death is an alert" property Sentry Crons provides. + +After changing PostHog env, restart the `loopover` service — there is no hot reload. + ## Browser Sentry (operator UI) The operator UI (`apps/loopover-ui`) has its own, separate client-side Sentry diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index bf580440a2..f4701460cf 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -429,6 +429,14 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "PORT", firstReference: "src/server.ts", }, + { + name: "POSTHOG_ENVIRONMENT", + firstReference: "src/server.ts", + }, + { + name: "POSTHOG_RELEASE", + firstReference: "src/selfhost/posthog.ts", + }, { name: "PUBLIC_API_ORIGIN", firstReference: "src/selfhost/preflight.ts", @@ -656,6 +664,8 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts` |", "| `PGVECTOR_ENABLED` | `src/server.ts` |", "| `PORT` | `src/server.ts` |", + "| `POSTHOG_ENVIRONMENT` | `src/server.ts` |", + "| `POSTHOG_RELEASE` | `src/selfhost/posthog.ts` |", "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts` |", "| `PUBLIC_ORIGIN_ACKNOWLEDGED` | `src/server.ts` |", "| `PUBLIC_SITE_ORIGIN` | `src/server.ts` |", diff --git a/src/env.d.ts b/src/env.d.ts index 7ab1919feb..9d401cc4a6 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -629,15 +629,21 @@ declare global { * token is sufficient — this probe never writes). A secret — never commit a real value. See * CLOUDFLARE_D1_MONITOR_ACCOUNT_ID. */ CLOUDFLARE_D1_MONITOR_API_TOKEN?: string; - /** Opt-in MCP telemetry (#6228/#6235): the PostHog project API key the typed `src/mcp/telemetry.ts` - * wrapper sends anonymized tool-call counters to. Unset (default — every self-hoster who doesn't opt in) - * ⇒ recordMcpToolCall is a safe no-op that records nothing, byte-identical to before this module existed. - * Only the #6228 allowlist is ever sent — tool name, caller type, ok, coarse duration — never arguments, - * source, or any wallet/hotkey/trust-score data. A secret — inject via `wrangler secret`, never commit. */ + /** Opt-in PostHog project API key, shared by two independent surfaces (#8287's env-var decision): (1) MCP + * telemetry (#6228/#6235) — the typed `src/mcp/telemetry.ts` wrapper sends anonymized tool-call counters + * (tool name, caller type, ok, coarse duration — never arguments, source, or wallet/hotkey/trust-score + * data); (2) self-host ORB error tracking (`src/selfhost/posthog.ts`) — the parallel-run PostHog sink + * alongside SENTRY_DSN, both active simultaneously when both are configured. Unset (default — every + * self-hoster who doesn't opt in) ⇒ both surfaces are safe no-ops, byte-identical to before either + * module existed. A secret — inject via `wrangler secret` (Worker) or a mounted secret file (self-host), + * never commit. */ POSTHOG_API_KEY?: string; - /** Opt-in MCP telemetry host override (#6235): the PostHog ingestion host recordMcpToolCall points at - * (e.g. https://eu.i.posthog.com for EU-cloud). Unset ⇒ the US-cloud default (https://us.i.posthog.com). - * Only meaningful alongside POSTHOG_API_KEY; ignored when telemetry is unconfigured. */ + /** PostHog ingestion host override (e.g. https://eu.i.posthog.com for EU-cloud), shared by both surfaces + * POSTHOG_API_KEY activates. Unset ⇒ the US-cloud default (https://us.i.posthog.com). Only meaningful + * alongside POSTHOG_API_KEY; ignored when unconfigured. Self-host error tracking also reads its own + * process.env-only vars (POSTHOG_MIN_SEVERITY, POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT, + * POSTHOG_SERVER_NAME, POSTHOG_RELEASE) — not declared here, mirroring SENTRY_MIN_SEVERITY/ + * SENTRY_REPO_MIN_SEVERITY's identical self-host-only precedent (src/selfhost/sentry.ts). */ POSTHOG_HOST?: string; } } diff --git a/src/queue/ai-review-orchestration.ts b/src/queue/ai-review-orchestration.ts index b0da79fc8d..d9cf92248a 100644 --- a/src/queue/ai-review-orchestration.ts +++ b/src/queue/ai-review-orchestration.ts @@ -59,6 +59,7 @@ import { resolveEnrichmentLinkedIssueNumbers, } from "../review/enrichment-wire"; import { captureReviewFailure } from "../selfhost/sentry"; +import { capturePostHogReviewFailure } from "../selfhost/posthog"; import { isReputationEnabled, shouldSkipAiForReputation } from "../review/reputation-wire"; import { isConvergenceRepoAllowed } from "../review/cutover-gate"; import { resolveConvergedFeature } from "../review/feature-activation"; @@ -822,6 +823,20 @@ export async function runAiReviewForAdvisory( /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []), }, "ai_review_inconclusive"); + capturePostHogReviewFailure(new Error("AI review inconclusive — no usable verdict for the PR head"), { + kind: "review", + reason: "ai_review_inconclusive", + installationId: args.installationId, + owner: args.repoFullName.split("/")[0], + repo: args.repoFullName, + pr: args.pr.number, + head_sha: args.advisory.headSha, + ai_review_mode: args.settings.aiReviewMode, + reviewer_count: result.reviewerCount, + public_notes: hasPublicReviewAssessment(result.advisoryNotes), + /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ + review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []), + }, "ai_review_inconclusive"); } args.advisory.findings.push(...findings); const metadataFor = ( @@ -900,6 +915,27 @@ export async function runAiReviewForAdvisory( }, "ai_review_public_summary_missing", ); + capturePostHogReviewFailure( + new Error("AI review did not produce public notes for the PR head"), + { + kind: "review", + reason: "ai_review_public_summary_missing", + installationId: args.installationId, + owner: args.repoFullName.split("/")[0], + repo: args.repoFullName, + pr: args.pr.number, + head_sha: args.advisory.headSha, + ai_review_mode: args.settings.aiReviewMode, + reviewer_count: result.reviewerCount, + /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ + review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []), + configured_reviewers: + env.AI_REVIEW_PLAN?.reviewers?.map((reviewer) => reviewer.model) ?? + null, + combine: env.AI_REVIEW_PLAN?.combine ?? null, + }, + "ai_review_public_summary_missing", + ); return { notes: "AI review is unavailable for this PR head. LoopOver is holding this PR for manual review until the configured AI provider returns a usable public review summary.", @@ -929,6 +965,13 @@ export async function runAiReviewForAdvisory( pr: args.pr.number, head_sha: args.advisory.headSha, }, "ai_review_failed"); + capturePostHogReviewFailure(error, { + kind: "review", + installationId: args.installationId, + repo: args.repoFullName, + pr: args.pr.number, + head_sha: args.advisory.headSha, + }, "ai_review_failed"); return undefined; } finally { // #regate-dup-prep: only release a lock THIS call actually claimed. A caller-supplied diff --git a/src/queue/processors.ts b/src/queue/processors.ts index a666b9304b..e4e3a3ec73 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -585,6 +585,7 @@ import { shouldApplyRepoCultureProfile } from "../review/repo-culture-profile-wi import { applyReviewMemorySuppression, getCachedReviewSuppressions, invalidateReviewSuppressionCache, shouldApplyReviewMemory } from "../review/review-memory-wire"; import { isEnrichmentEnabled } from "../review/enrichment-wire"; import { captureReviewFailure } from "../selfhost/sentry"; +import { capturePostHogReviewFailure } from "../selfhost/posthog"; import { setReviewPipelineSpanOutcome, withReviewPipelineSpan, @@ -9168,6 +9169,15 @@ async function maybePublishPrPublicSurface( head_sha: advisory.headSha, failedOutputs: failedOutputs.map((failure) => failure.output), }, "pr_public_surface_publish_failed"); + capturePostHogReviewFailure(new Error("PR public-surface publish failed — review produced output but nothing was posted to the PR"), { + kind: "publish", + installationId, + owner: repoFullName.split("/")[0], + repo: repoFullName, + pr: pr.number, + head_sha: advisory.headSha, + failedOutputs: failedOutputs.map((failure) => failure.output), + }, "pr_public_surface_publish_failed"); // At least one output failed for a reason that can plausibly clear on its own (rate limit / 5xx / momentary // token issue) — retry the whole job instead of leaving the review permanently unposted. A mix of transient // and permanent failures still retries: the permanent one re-fails identically next pass and re-audits, but @@ -10367,6 +10377,16 @@ async function maybePublishPrPublicSurface( reviewer_count: aiReview?.reviewerCount ?? 0, public_notes: hasPublicReviewAssessment(aiReview?.notes), }, "ai_review_public_summary_missing"); + capturePostHogReviewFailure(new Error(message), { + kind: "review", + reason: "ai_review_public_summary_missing", + installationId, + repo: repoFullName, + pr: pr.number, + head_sha: advisory.headSha, + reviewer_count: aiReview?.reviewerCount ?? 0, + public_notes: hasPublicReviewAssessment(aiReview?.notes), + }, "ai_review_public_summary_missing"); } // Secrets-scan (#audit-3.4): always scans the REAL resolved diff and, on a CONCRETE credential hit, appends a diff --git a/src/selfhost/monitored-work.ts b/src/selfhost/monitored-work.ts index 149771ecbf..856f9c756f 100644 --- a/src/selfhost/monitored-work.ts +++ b/src/selfhost/monitored-work.ts @@ -2,6 +2,20 @@ import type { EnqueueWebhookResult } from "../github/webhook"; import { ORB_RELAY_REGISTER_UNHEALTHY_FAILURE_STREAK, type OrbRelayRegistrationState } from "../orb/broker-client"; import { incr } from "./metrics"; import { withSentryMonitor } from "./sentry"; +import { withPostHogMonitor } from "./posthog"; + +/** Run both monitor wrappers around one callback: Sentry's structured check-in + PostHog's heartbeat event, + * in parallel (#8287) -- neither observes the other's outcome, each independently no-ops when its own sink + * is unconfigured. The callback itself only runs ONCE; withPostHogMonitor wraps a thunk that replays whatever + * withSentryMonitor already produced/threw, so a failure is captured by both sinks without executing the + * underlying work twice. */ +async function withBothMonitors( + name: Parameters[0] & Parameters[0], + context: Record | undefined, + callback: () => Promise, +): Promise { + return withPostHogMonitor(name, context, () => withSentryMonitor(name, context, callback)); +} export type OrbRelayEvent = { deliveryId: string; @@ -59,7 +73,7 @@ export async function runScheduledLoopWithMonitor( cron: string, scheduled: () => T | Promise, ): Promise { - return withSentryMonitor( + return withBothMonitors( "scheduled-loop", { jobType: "scheduled-loop", cron }, () => Promise.resolve(scheduled()), @@ -70,7 +84,7 @@ export async function runOrbExportWithMonitor( exportBatch: () => Promise, log: (line: string) => void = console.log, ): Promise { - await withSentryMonitor("orb-export", { jobType: "orb-export" }, async () => { + await withBothMonitors("orb-export", { jobType: "orb-export" }, async () => { const exported = await exportBatch(); if (exported > 0) log(JSON.stringify({ event: "selfhost_orb_export", exported })); @@ -91,7 +105,7 @@ export async function drainOrbRelayWithMonitor(args: { log?: (line: string) => void; nowMs?: number; }): Promise { - await withSentryMonitor( + await withBothMonitors( "orb-relay-drain", { jobType: "orb-relay-drain", pendingAckCount: args.state.pendingAck.length }, async () => { @@ -247,7 +261,7 @@ export async function registerOrbRelayWithMonitor(args: { log?: (line: string) => void; nowMs?: number; }): Promise { - await withSentryMonitor("orb-relay-register", { jobType: "orb-relay-register" }, async () => { + await withBothMonitors("orb-relay-register", { jobType: "orb-relay-register" }, async () => { const result = await args.register(args.env, args.state); if (result.status === "skipped" || result.status === "already_registered" || result.status === "backoff") return; const mode = args.env.ORB_RELAY_MODE === "pull" ? "pull" : "push"; diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index e74512c067..f925ef2a6f 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -9,6 +9,7 @@ import { incr } from "./metrics"; import { withReviewSpan } from "./tracing"; import { withOtelSpan } from "./otel"; import { captureError, withSentryMonitor } from "./sentry"; +import { capturePostHogError, withPostHogMonitor } from "./posthog"; import { consumingRetryDelayMs, deterministicJitterMs, @@ -632,10 +633,15 @@ export function createPgQueue( * fires; the outer try/catch (this function's actual job) still guards the setInterval callback. */ async function reviveDeadLetterJobsSafely(): Promise { try { - await withSentryMonitor( + await withPostHogMonitor( "queue-dead-letter-revive", { jobType: "queue-dead-letter-revive" }, - reviveDeadLetterJobs, + () => + withSentryMonitor( + "queue-dead-letter-revive", + { jobType: "queue-dead-letter-revive" }, + reviveDeadLetterJobs, + ), ); } catch (error) { console.error( @@ -646,6 +652,7 @@ export function createPgQueue( }), ); captureError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); + capturePostHogError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); } } @@ -772,6 +779,7 @@ export function createPgQueue( }), ); captureError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); + capturePostHogError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); } } @@ -1102,6 +1110,12 @@ export function createPgQueue( recovered, timeoutMs: processingTimeoutMs, }, "processing_timeout"); + capturePostHogError(new Error("self-host queue processing lease expired"), { + kind: "job_recovered", + reason: "processing_timeout", + recovered, + timeoutMs: processingTimeoutMs, + }, "processing_timeout"); } const job = await claimNext(); if (!job) return false; @@ -1130,6 +1144,11 @@ export function createPgQueue( reason: "unparseable_payload", jobId: job.id, }, "unparseable_payload"); + capturePostHogError(new Error("unparseable queue payload"), { + kind: "job_dead", + reason: "unparseable_payload", + jobId: job.id, + }, "unparseable_payload"); return true; } const jobTraceParent = message.type === "github-webhook" ? message.traceParent : undefined; @@ -1441,6 +1460,13 @@ export function createPgQueue( jobId: job.id, attempts, }, "job_dead"); + capturePostHogError(error, { + kind: "job_dead", + reason: "max_retries_exhausted", + jobType: extractPayloadType(job.payload), + jobId: job.id, + attempts, + }, "job_dead"); } else { const retryDelayMs = consumingRetryDelayMs(error, backoff(attempts)); await pool.query( @@ -1488,6 +1514,7 @@ export function createPgQueue( }), ); captureError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); + capturePostHogError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); } finally { active--; } diff --git a/src/selfhost/posthog.ts b/src/selfhost/posthog.ts new file mode 100644 index 0000000000..5edad7b25f --- /dev/null +++ b/src/selfhost/posthog.ts @@ -0,0 +1,342 @@ +// Self-host PostHog error tracking (#8287, epic #8286). Opt-in: a complete NO-OP unless POSTHOG_API_KEY is set, +// mirroring sentry.ts's env-gated posture exactly. posthog-node is dynamically imported inside initPostHog() +// so it never enters the Worker bundle when unconfigured, same discipline as @sentry/node in sentry.ts. +// +// PARALLEL-RUN (epic principle, confirmed 2026-07-25): this sink runs alongside sentry.ts, not instead of it. +// Both are active simultaneously when both are configured; Sentry is only removed once the gated decommission +// issue (#8298) says so, per-issue across all six Phase-1 surfaces at once -- never removed early just because +// this surface's PostHog sink works. +// +// Reuses the SAME pure redaction primitives sentry.ts uses (./redaction-scrub) -- one security-critical scrub +// implementation, two provider-specific event-shape orchestrators. PostHog's event shape (event name + a flat +// `properties` bag) is materially simpler than Sentry's (request/contexts/extra/tags/breadcrumbs/exception), +// so this file's own orchestrator (scrubPostHogEvent) is correspondingly smaller than sentry.ts's scrubEvent. +// +// Env-var decision (#8287's own deliverable): POSTHOG_API_KEY/POSTHOG_HOST are the SAME vars #6235's MCP +// telemetry (src/mcp/telemetry.ts) already reads off the typed Cloudflare Env -- one project key activates +// both surfaces, the MCP tool-call allowlist (#6228) is untouched. Everything else here (POSTHOG_MIN_SEVERITY, +// POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT, POSTHOG_SERVER_NAME, POSTHOG_RELEASE) is self-host-only, +// read off real process.env exactly like sentry.ts's own SENTRY_MIN_SEVERITY/SENTRY_REPO_MIN_SEVERITY/etc -- +// never added to src/env.d.ts's typed Env, matching that file's precedent for self-host-exclusive config. +import { hostname } from "node:os"; +import { + currentOtelTraceIds, +} from "./otel"; +import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "../services/severity-threshold"; +import { SENTRY_OPERATIONAL_TAG_KEYS } from "./sentry"; +import { + hashedInstallationContext, + loadNodeHasher, + nonBlank, + REDACTED, + resetRedactionScrubForTest, + scrubRecord, + scrubString, + SECRET_KEY, +} from "./redaction-scrub"; + +type PostHogNs = typeof import("posthog-node"); +type PostHogClient = InstanceType; +type PostHogEventMessage = import("posthog-node").EventMessage; + +let client: PostHogClient | undefined; +let active = false; +let posthogEnvironment = "production"; + +/** No per-user identity is tracked by this sink (operational error events, not user analytics) -- every event + * shares one anonymous, constant distinct id, mirroring src/mcp/telemetry.ts's identical MCP_TELEMETRY_DISTINCT_ID + * choice for the same reason (#6228). */ +const POSTHOG_DISTINCT_ID = "loopover-selfhost"; + +/** PostHog US-cloud ingestion host, matching src/mcp/telemetry.ts's default. */ +const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"; + +/** Resolve a self-host-only config var: process.env only (unlike POSTHOG_API_KEY/POSTHOG_HOST, these have no + * Worker/Env-typed counterpart -- mirrors sentry.ts's own SENTRY_* self-host var reads verbatim). */ +function processEnvString(env: NodeJS.ProcessEnv, name: string): string | undefined { + return nonBlank(env[name]); +} + +/** Resolve the PostHog release id: explicit override first, then the image-baked self-host version, matching + * {@link resolveSentryRelease}'s identical precedence in sentry.ts. */ +export function resolvePostHogRelease(env: NodeJS.ProcessEnv): string | undefined { + return nonBlank(env.POSTHOG_RELEASE) ?? nonBlank(env.LOOPOVER_VERSION); +} + +/** The repo a capture's properties belong to, for per-repo severity-threshold lookup -- mirrors sentry.ts's + * contextRepoFullName's identical repo-over-repository normalization. */ +function contextRepoFullName(properties: Record | undefined): string { + if (!properties) return ""; + const repo = typeof properties.repo === "string" ? properties.repo : typeof properties.repository === "string" ? properties.repository : undefined; + return repo ?? ""; +} + +/** Resolve the minimum severity for `repoFullName`: POSTHOG_REPO_MIN_SEVERITY (a JSON `{repoFullName: severity}` + * map) wins, else the global POSTHOG_MIN_SEVERITY, else `"error"`. Separate knobs from Sentry's + * SENTRY_MIN_SEVERITY/SENTRY_REPO_MIN_SEVERITY on purpose: an operator running both sinks in parallel may + * legitimately want PostHog quieter/noisier than Sentry while comparing the two during the parallel-run window. */ +function resolvePostHogMinSeverity(repoFullName: string): LoopoverSeverity { + const processEnv = (globalThis as unknown as { process?: { env?: Record } }).process?.env ?? {}; + return resolveSeverityThreshold(processEnv as unknown as Env, repoFullName, "POSTHOG_MIN_SEVERITY", "POSTHOG_REPO_MIN_SEVERITY"); +} + +/** Map a structured log's own level onto the shared severity taxonomy, matching sentry.ts's + * normalizeLoopoverSeverity exactly (debug folds into info; an unrecognized word is treated as info, never + * promoted). */ +function normalizePostHogSeverity(level: string): LoopoverSeverity { + const lower = level.toLowerCase(); + if (lower === "critical" || lower === "fatal") return "critical"; + if (lower === "error") return "error"; + if (lower === "warning" || lower === "warn") return "warning"; + return "info"; +} + +/** before_send scrubber -- redact anything token/secret-like before an event leaves the box (privacy + * boundary), applied to every outgoing event including the ones captureException() builds internally. + * PostHog's event shape is just {event, properties, ...}, so unlike sentry.ts's scrubEvent this only needs to + * walk one bag, not five separately-shaped sub-objects. */ +export function scrubPostHogEvent(event: PostHogEventMessage | null): PostHogEventMessage | null { + if (!event) return event; + try { + if (event.properties) scrubRecord(event.properties, 0); + if (typeof event.event === "string") event.event = scrubString(event.event); + } catch { + return null; + } + return event; +} + +/** Build the properties bag for a captured error/log: hashes any installation id, then tags the shared + * operational key allowlist (reused from sentry.ts -- the same fields are exactly as safe to surface as a + * PostHog property as they are as a Sentry tag) alongside whatever else the caller passed. */ +function operationalProperties(context: Record | undefined): Record { + const safeContext = context ? hashedInstallationContext(context) : {}; + const normalized: Record = + typeof safeContext.repository === "string" && safeContext.repo === undefined + ? { ...safeContext, repo: safeContext.repository } + : { ...safeContext }; + const properties: Record = {}; + for (const key of SENTRY_OPERATIONAL_TAG_KEYS) { + const value = normalized[key]; + if (typeof value === "string" || typeof value === "number") properties[key] = value; + } + const trace = currentOtelTraceIds(); + if (trace) { + properties.trace_id = trace.trace_id; + properties.span_id = trace.span_id; + } + return properties; +} + +/** Initialize PostHog from the environment. Returns false (and stays a no-op) when POSTHOG_API_KEY is unset -- + * the SAME var #6235's MCP telemetry reads (env-var decision, #8287). `env` is real process.env, matching + * initSentry's identical NodeJS.ProcessEnv shape. */ +export async function initPostHog(env: NodeJS.ProcessEnv): Promise { + const apiKey = processEnvString(env, "POSTHOG_API_KEY"); + if (!apiKey) return false; + await loadNodeHasher(); + const { PostHog } = await import("posthog-node"); + posthogEnvironment = processEnvString(env, "POSTHOG_ENVIRONMENT") ?? "production"; + const host = processEnvString(env, "POSTHOG_HOST") ?? DEFAULT_POSTHOG_HOST; + client = new PostHog(apiKey, { + host, + // Long-running server, not a per-request/edge context (unlike src/mcp/telemetry.ts's ephemeral + // per-call client) -- default batching/flush interval is the right posture here; flushPostHog() below + // still exists for explicit drain-before-exit, and PostHog's own recommended client.shutdown() covers + // graceful process shutdown. + before_send: scrubPostHogEvent, + // Matches Sentry's own Node SDK default posture (Sentry.init() installs global uncaughtException/ + // unhandledRejection handlers unless explicitly disabled) -- a safety net for genuinely-unhandled cases + // beyond the explicit capturePostHogError/capturePostHogReviewFailure call sites below, per PostHog's + // own documented recommendation for Node.js error tracking. + enableExceptionAutocapture: true, + }); + active = true; + return true; +} + +/** Name a captured Error before capture so its PostHog issue title reads "eventName: message" instead of the + * generic "Error: message", mirroring sentry.ts's namedCaptureError exactly (including its never-mutate-the- + * caught-value care for read-only `name` on some runtime errors like DOMException). */ +function namedCaptureError(error: unknown, eventName?: string): Error { + const err = error instanceof Error ? error : new Error(String(error)); + if (!eventName) return err; + const namedError = new Error(err.message, { cause: err }); + namedError.name = eventName; + Object.defineProperty(namedError, "stack", { value: err.stack, configurable: true, writable: true }); + return namedError; +} + +/** Capture an error with optional structured context. No-op when PostHog is off OR the repo's resolved + * severity threshold is above `error`. Mirrors sentry.ts's captureError exactly, using PostHog's own + * documented captureException(error, distinctId, properties) instead of Sentry.withScope/captureException. */ +export function capturePostHogError(error: unknown, context?: Record, eventName?: string): void { + if (!active || !client) return; + if (!meetsSeverityThreshold("error", resolvePostHogMinSeverity(contextRepoFullName(context)))) return; + const properties = operationalProperties(context); + properties.server_name = nonBlank((globalThis as unknown as { process?: { env?: Record } }).process?.env?.POSTHOG_SERVER_NAME) ?? hostname(); + properties.environment = posthogEnvironment; + client.captureException(namedCaptureError(error, eventName), POSTHOG_DISTINCT_ID, properties); +} + +/** Capture a failed review at ERROR level, tagged by repo/PR/SHA for triage. Mirrors sentry.ts's + * captureReviewFailure exactly -- a review that cannot be produced is a real failure, always captured at + * error grade when the threshold allows it through at all. */ +export function capturePostHogReviewFailure(error: unknown, context?: Record, eventName?: string): void { + if (!active || !client) return; + if (!meetsSeverityThreshold("error", resolvePostHogMinSeverity(contextRepoFullName(context)))) return; + const properties = operationalProperties(context); + properties.kind = "review_failure"; + client.captureException(namedCaptureError(error, eventName), POSTHOG_DISTINCT_ID, properties); +} + +/** A SHORT location suffix for a no-message log's summary, matching sentry.ts's logLocation exactly. */ +function logLocation(obj: Record): string { + const repo = typeof obj.repository === "string" ? obj.repository : typeof obj.repo === "string" ? obj.repo : undefined; + if (!repo) return ""; + const pr = obj.pullNumber; + return typeof pr === "number" ? ` (${repo}#${pr})` : ` (${repo})`; +} + +const SUMMARY_SKIP_KEYS = new Set([ + "level", "event", "ts", "time", "timestamp", "msg", "ev", "message", "error", "repo", "repository", + "installationId", "installation_id", "installation_id_hash", "pullNumber", "deliveryId", "trace_id", "span_id", +]); + +function redactSummaryValue(value: unknown, depth = 0): unknown { + if (!value || typeof value !== "object") return value; + if (depth >= 6) return REDACTED; + if (Array.isArray(value)) return value.map((item) => redactSummaryValue(item, depth + 1)); + return Object.fromEntries( + Object.entries(value as Record).map(([key, nested]) => [key, SECRET_KEY.test(key) ? REDACTED : redactSummaryValue(nested, depth + 1)]), + ); +} + +/** Summarize a field-only log's salient scalars into the captured value, matching sentry.ts's + * summarizeLogFields exactly. */ +function summarizeLogFields(obj: Record): string { + return Object.entries(obj) + .filter(([k, v]) => !SUMMARY_SKIP_KEYS.has(k) && !SECRET_KEY.test(k) && v !== null) + .map(([k, v]) => `${k}=${typeof v === "object" ? JSON.stringify(redactSummaryValue(v)) : String(v)}`) + .filter((part) => part.length <= 90) + .slice(0, 5) + .join(", "); +} + +/** Forward a structured console line to PostHog when its level meets the repo's resolved severity threshold, + * matching sentry.ts's forwardStructuredLogToSentry exactly (same error-sink-defaults-to-error-level + * behavior, same synthetic-exception construction so the captured event always has a real type+value). */ +export function forwardStructuredLogToPostHog(line: unknown, fromErrorSink = false): void { + if (!active || !client) return; + if (typeof line !== "string" || line.charCodeAt(0) !== 123 /* "{" */) return; + let obj: Record; + try { + obj = JSON.parse(line) as Record; + } catch { + return; + } + const safeObj = hashedInstallationContext(obj); + const explicitLevel = typeof obj.level === "string" ? obj.level : undefined; + const level = explicitLevel ?? (fromErrorSink ? "error" : undefined); + if (!level) return; + const loopoverSeverity = normalizePostHogSeverity(level); + if (!meetsSeverityThreshold(loopoverSeverity, resolvePostHogMinSeverity(contextRepoFullName(safeObj)))) return; + const event = typeof obj.event === "string" ? obj.event : undefined; + const subEvent = typeof obj.ev === "string" ? obj.ev : undefined; + const detail = typeof obj.message === "string" ? obj.message : typeof obj.error === "string" ? obj.error : undefined; + const value = detail ?? ([logLocation(safeObj).trim(), summarizeLogFields(safeObj)].filter(Boolean).join(" ") || "(no message — see the log context)"); + const errorEvent = new Error(value); + errorEvent.name = event ? (subEvent ? `${event}/${subEvent}` : event) : "LoopOverLog"; + errorEvent.stack = `${errorEvent.name}: ${value}`; + const properties = operationalProperties(safeObj); + properties.severity = loopoverSeverity; + if (event) properties.event_slug = event; + if (subEvent) properties.event_sub_slug = subEvent; + client.captureException(errorEvent, POSTHOG_DISTINCT_ID, properties); +} + +interface StructuredLogConsole { + log: (...args: unknown[]) => void; + error: (...args: unknown[]) => void; +} + +/** Install PostHog structured-log forwarding for both stdout and stderr sinks. Safe to call alongside + * sentry.ts's installStructuredLogForwarding (parallel run) -- each wraps whatever target.log/target.error + * currently is, so calling both chains correctly regardless of call order. */ +export function installPostHogStructuredLogForwarding(target: StructuredLogConsole = console): void { + const baseConsoleLog = target.log.bind(target); + const baseConsoleError = target.error.bind(target); + let forwarding = false; + const forward = (line: unknown, fromErrorSink: boolean): void => { + if (forwarding) return; + forwarding = true; + try { + forwardStructuredLogToPostHog(line, fromErrorSink); + } finally { + forwarding = false; + } + }; + target.log = (...args: unknown[]): void => { + baseConsoleLog(...args); + forward(args[0], false); + }; + target.error = (...args: unknown[]): void => { + baseConsoleError(...args); + forward(args[0], true); + }; +} + +/** Cron-monitor replacement (#8287 deliverable 4): PostHog has no native cron-monitor/check-in product, so + * this emits a plain heartbeat event per run instead -- start, then ok/error with duration -- rather than + * Sentry's structured monitor-slug + check-in-id + schedule-config concept, which has no PostHog equivalent + * to map onto. The "silent death is an alert" property is preserved at the ALERTING layer, not here: a + * PostHog insight alert on a no-data condition over `orb_monitor_heartbeat` (filtered to status:"ok" and a + * given monitor name) fires exactly when check-ins stop arriving, the same failure mode Sentry Crons' + * missed-check-in alerting catches. Configuring that alert is explicitly out of scope here -- it belongs to + * #8294 (the epic's insights/dashboards/alert-routing issue), which this event stream is built to feed. */ +export type PostHogMonitorName = "scheduled-loop" | "orb-export" | "orb-relay-drain" | "orb-relay-register" | "queue-dead-letter-revive"; +export const POSTHOG_MONITOR_HEARTBEAT_EVENT = "orb_monitor_heartbeat"; + +export async function withPostHogMonitor(name: PostHogMonitorName, context: Record | undefined, callback: () => Promise): Promise { + if (!active || !client) return callback(); + const startedAt = Date.now(); + try { + const result = await callback(); + client.capture({ + distinctId: POSTHOG_DISTINCT_ID, + event: POSTHOG_MONITOR_HEARTBEAT_EVENT, + properties: { monitor: name, status: "ok", duration_ms: Date.now() - startedAt, environment: posthogEnvironment }, + }); + return result; + } catch (error) { + client.capture({ + distinctId: POSTHOG_DISTINCT_ID, + event: POSTHOG_MONITOR_HEARTBEAT_EVENT, + properties: { monitor: name, status: "error", duration_ms: Date.now() - startedAt, environment: posthogEnvironment }, + }); + const properties = operationalProperties({ ...context, monitor: name, kind: `posthog_monitor_${name}`, subsystem: "scheduled" }); + client.captureException(error instanceof Error ? error : new Error(String(error)), POSTHOG_DISTINCT_ID, properties); + throw error; + } +} + +/** Flush buffered events before exit. No-op when off. */ +export async function flushPostHog(): Promise { + if (!active || !client) return; + await client.flush().catch(() => undefined); +} + +/** Gracefully shut the PostHog client down (flushes, then stops its internal timers) -- PostHog's own + * documented cleanup call for a long-running Node process, distinct from flushPostHog's mid-life drain. */ +export async function shutdownPostHog(): Promise { + if (!active || !client) return; + await client.shutdown().catch(() => undefined); +} + +/** Test-only: reset module state between cases. */ +export function resetPostHogForTest(): void { + client = undefined; + active = false; + posthogEnvironment = "production"; + resetRedactionScrubForTest(); +} diff --git a/src/selfhost/redaction-scrub.ts b/src/selfhost/redaction-scrub.ts new file mode 100644 index 0000000000..101c4f4677 --- /dev/null +++ b/src/selfhost/redaction-scrub.ts @@ -0,0 +1,196 @@ +// Shared, provider-agnostic redaction primitives (#8287) -- extracted out of src/selfhost/sentry.ts's +// beforeSend scrubber so the PostHog sink (src/selfhost/posthog.ts) can reuse the exact same secret/private- +// text detection without hand-duplicating a security-critical regex set into a second file. This module owns +// only the pure "is this key/value secret-shaped, and how do I redact it" logic; each provider's own +// event-SHAPE orchestration (Sentry's request/contexts/extra/tags/breadcrumbs vs PostHog's flat properties +// bag) stays in that provider's own file, since the two shapes are genuinely different and forcing them +// through one shared walker would make both providers depend on the same brittle shape-guessing. +import { + PUBLIC_LOCAL_PATH_SCRUB_PATTERN, + PUBLIC_UNSAFE_TERMS, +} from "../signals/redaction"; + +export const SECRET_KEY = + /(token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)/i; +export const PAYLOAD_KEY = + /(^|[_-])(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content|comment[-_]?text|comment[-_]?body)([_-]|$)|^(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content|comment[-_]?text|comment[-_]?body)$/i; +export const SECRET_VALUE = new RegExp( + [ + `${"github" + "_pat_"}[A-Za-z0-9_]+`, + String.raw`gh[opsru]_[A-Za-z0-9_]{20,}`, + String.raw`sk-[A-Za-z0-9_-]{20,}`, + String.raw`xox[baprs]-[A-Za-z0-9-]+`, + // LoopOver's own opaque tokens (createOpaqueToken, src/auth/security.ts): gts_ is the default session-token + // prefix, orbenr_/orbsec_ are the Orb broker's enrollment id/secret (#1825) -- a broker error message can quote + // these bare (no "secret"/"token"-named field for the key-based redaction below to catch), so the VALUE itself + // must be recognized here too. + String.raw`(?:gts|orbenr|orbsec)_[A-Za-z0-9_]{20,}`, + String.raw`Bearer\s+[A-Za-z0-9._~+/=-]{12,}`, + String.raw`-----BEGIN [^-]+ PRIVATE KEY-----[\s\S]*?-----END [^-]+ PRIVATE KEY-----`, + ].join("|"), + "gi", +); +export const JWT_VALUE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; +export const QUERY_SECRET_VALUE = + /([?&;][^=\s&#;]*(?:token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)[^=\s&#;]*=)[^&#\s;]+/gi; +export const PRIVATE_TEXT = + /\b(raw[-_\s]?score|scoring context|private rubric|gate prompt|review prompt|guardrail paths?|pull request body|pr body|pr title|raw diff)\b/gi; +export const PUBLIC_UNSAFE_SCRUB = new RegExp(String.raw`\b(${PUBLIC_UNSAFE_TERMS})\b`, "gi"); +export const REDACTED = "[redacted]"; + +export function nonBlank(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +type DigestHex = (input: string) => string; +let digestHexSync: DigestHex | undefined; + +/** Lazily load a SYNCHRONOUS sha256 hasher (node:crypto's createHash) -- kept out of the module's static + * import graph so this module stays safe to import from anywhere (including a Cloudflare Worker bundle) + * when the caller never actually invokes {@link installationIdHash}. Mirrors sentry.ts's identical + * loadNodeHasher discipline pre-extraction. A sync hasher is required because {@link scrubRecord} walks + * its object tree synchronously; the async Web-Crypto-based sha256Hex (src/utils/crypto.ts) can't be + * awaited mid-walk without restructuring every caller into an async tree-walk. */ +export async function loadNodeHasher(): Promise { + const { createHash } = await import("node:crypto"); + digestHexSync = (input: string): string => createHash("sha256").update(input).digest("hex"); +} + +/** Test-only: reset the lazily-loaded hasher between cases. */ +export function resetRedactionScrubForTest(): void { + digestHexSync = undefined; +} + +const INSTALLATION_HASH_SEED = "github-installation:"; + +function normalizeInstallationId(value: unknown): string | undefined { + if (typeof value === "number" && Number.isFinite(value)) return String(Math.trunc(value)); + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return /^[0-9]+$/.test(trimmed) ? trimmed : undefined; +} + +/** Hash a raw installation id into a short, non-reversible tag -- undefined when the hasher hasn't been + * loaded yet ({@link loadNodeHasher}) or the value isn't a real installation id. */ +export function installationIdHash(value: unknown): string | undefined { + if (!digestHexSync) return undefined; + const normalized = normalizeInstallationId(value); + if (!normalized) return undefined; + return digestHexSync(`${INSTALLATION_HASH_SEED}${normalized}`).slice(0, 16); +} + +export function isInstallationIdKey(key: string): boolean { + return key.replace(/[^A-Za-z0-9]/g, "").toLowerCase() === "installationid"; +} + +/** Replace a context's raw installation_id/installationId with its hash, matching sentry.ts's + * hashedInstallationContext. Returns the input unchanged when there's nothing to hash. */ +export function hashedInstallationContext(context: Record): Record { + const hasInstallationId = "installation_id" in context || "installationId" in context; + const hash = installationIdHash(context.installation_id ?? context.installationId); + if (!hash && !hasInstallationId) return context; + const safe: Record = { ...context }; + if (hash) safe.installation_id_hash = hash; + delete safe.installation_id; + delete safe.installationId; + return safe; +} + +export function shouldRedactKey(key: string): boolean { + const compact = key.replace(/[^A-Za-z0-9]/g, "").toLowerCase(); + return ( + SECRET_KEY.test(key) || + PAYLOAD_KEY.test(key) || + /(body|payload|patch|diff|prompt|rubric|guardrail|header|cookie|title|config|reviewtext|reviewcontent|prcontent|pullrequest)/.test(compact) + ); +} + +export function scrubString(value: string): string { + return value + .replace(QUERY_SECRET_VALUE, `$1${REDACTED}`) + .replace(SECRET_VALUE, REDACTED) + .replace(JWT_VALUE, REDACTED) + .replace(PUBLIC_LOCAL_PATH_SCRUB_PATTERN, "") + .replace(PUBLIC_UNSAFE_SCRUB, "private context") + .replace(PRIVATE_TEXT, "private context"); +} + +export function isUrlKey(key: string): boolean { + return key.replace(/[^A-Za-z0-9]/g, "").toLowerCase().endsWith("url"); +} + +export function isQueryKey(key: string): boolean { + const compact = key.replace(/[^A-Za-z0-9]/g, "").toLowerCase(); + return compact === "query" || compact === "querystring"; +} + +export function scrubQueryString(value: string): string { + const hasQuestionMark = value.startsWith("?"); + const source = hasQuestionMark ? value.slice(1) : value; + const params = new URLSearchParams(source); + for (const key of Array.from(new Set(params.keys()))) { + const values = params.getAll(key); + params.delete(key); + for (const entry of values) { + params.append(key, shouldRedactKey(key) ? REDACTED : scrubString(entry)); + } + } + const scrubbed = params.toString(); + return hasQuestionMark ? `?${scrubbed}` : scrubbed; +} + +export function scrubUrl(value: string): string { + const scrubbed = scrubString(value); + const queryStart = scrubbed.indexOf("?"); + if (queryStart === -1) return scrubbed; + try { + const parsed = new URL(scrubbed); + parsed.search = scrubQueryString(parsed.search); + return parsed.toString(); + } catch { + return `${scrubbed.slice(0, queryStart + 1)}${scrubQueryString(scrubbed.slice(queryStart + 1))}`; + } +} + +export function scrubStringField(key: string, value: string): string { + if (isUrlKey(key)) return scrubUrl(value); + if (isQueryKey(key)) return scrubQueryString(value); + return scrubString(value); +} + +/** Recursively redact secret-shaped keys/values in place, depth-bounded to avoid runaway recursion on a + * pathological/cyclic-looking structure. Shared verbatim by both providers' event orchestrators. */ +export function scrubRecord(obj: unknown, depth: number): void { + if (!obj || typeof obj !== "object") return; + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + const value = obj[i]; + if (typeof value === "string") obj[i] = scrubString(value); + else if (value && typeof value === "object") { + if (depth >= 6) obj[i] = REDACTED; + else scrubRecord(value, depth + 1); + } + } + return; + } + const rec = obj as Record; + for (const key of Object.keys(rec)) { + if (isInstallationIdKey(key)) { + const hash = installationIdHash(rec[key]); + if (hash) rec.installation_id_hash = hash; + delete rec[key]; + continue; + } + if (shouldRedactKey(key)) { + rec[key] = REDACTED; + continue; + } + const value = rec[key]; + if (typeof value === "string") rec[key] = scrubStringField(key, value); + else if (value && typeof value === "object") { + if (depth >= 6) rec[key] = REDACTED; + else scrubRecord(value, depth + 1); + } + } +} diff --git a/src/selfhost/sentry.ts b/src/selfhost/sentry.ts index b4e96cd64d..ee6f0b7154 100644 --- a/src/selfhost/sentry.ts +++ b/src/selfhost/sentry.ts @@ -2,19 +2,35 @@ // env-gated, dynamically-imported selfhost-integration pattern (Redis/Qdrant/embed-provider in server.ts). // @sentry/node is NEVER imported at module top level — it loads lazily inside initSentry(), so it never enters // the Worker bundle (src/index.ts) and cloudflare:* stubbing stays clean. All helpers are safe to call when off. -import { - PUBLIC_LOCAL_PATH_SCRUB_PATTERN, - PUBLIC_UNSAFE_TERMS, -} from "../signals/redaction"; +// +// The pure secret/private-text redaction primitives (SECRET_KEY, scrubString, scrubRecord, installation-id +// hashing, ...) live in ./redaction-scrub (#8287) so the PostHog sink (./posthog.ts) reuses the identical +// security-critical detection instead of a hand-duplicated second copy. Only this file's own Sentry-event- +// SHAPE orchestration (scrubEvent walking request/contexts/extra/tags/breadcrumbs) stays here. import { hostname } from "node:os"; import { currentOtelTraceIds, openTelemetryTraceExportEnabled, type OpenTelemetryBridge, } from "./otel"; -import { hashedInstallationIdWith } from "./review-tracing"; import { queueDeadLetterReviveIntervalMs } from "./queue-common"; import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "../services/severity-threshold"; +import { + hashedInstallationContext, + installationIdHash, + isInstallationIdKey, + loadNodeHasher, + nonBlank, + REDACTED, + resetRedactionScrubForTest, + scrubQueryString, + scrubRecord, + scrubString, + scrubStringField, + scrubUrl, + SECRET_KEY, + shouldRedactKey, +} from "./redaction-scrub"; type SentryNs = typeof import("@sentry/node"); type SentryClient = NonNullable>; @@ -27,40 +43,12 @@ type SentryScope = { setContext(name: string, context: Record): void; setTag(key: string, value: string): void; }; -type DigestHex = (input: string) => string; let Sentry: SentryNs | undefined; let sentryClient: SentryClient | undefined; let sentryTraceSampleRate: number | undefined; let active = false; let sentryEnvironment = "production"; -let digestHexSync: DigestHex | undefined; -const SECRET_KEY = - /(token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)/i; -const PAYLOAD_KEY = - /(^|[_-])(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content|comment[-_]?text|comment[-_]?body)([_-]|$)|^(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content|comment[-_]?text|comment[-_]?body)$/i; -const SECRET_VALUE = new RegExp( - [ - `${"github" + "_pat_"}[A-Za-z0-9_]+`, - String.raw`gh[opsru]_[A-Za-z0-9_]{20,}`, - String.raw`sk-[A-Za-z0-9_-]{20,}`, - String.raw`xox[baprs]-[A-Za-z0-9-]+`, - // LoopOver's own opaque tokens (createOpaqueToken, src/auth/security.ts): gts_ is the default session-token - // prefix, orbenr_/orbsec_ are the Orb broker's enrollment id/secret (#1825) — a broker error message can quote - // these bare (no "secret"/"token"-named field for the key-based redaction above to catch), so the VALUE itself - // must be recognized here too. - String.raw`(?:gts|orbenr|orbsec)_[A-Za-z0-9_]{20,}`, - String.raw`Bearer\s+[A-Za-z0-9._~+/=-]{12,}`, - String.raw`-----BEGIN [^-]+ PRIVATE KEY-----[\s\S]*?-----END [^-]+ PRIVATE KEY-----`, - ].join("|"), - "gi", -); -const JWT_VALUE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; -const QUERY_SECRET_VALUE = - /([?&;][^=\s&#;]*(?:token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)[^=\s&#;]*=)[^&#\s;]+/gi; -const PRIVATE_TEXT = - /\b(raw[-_\s]?score|scoring context|private rubric|gate prompt|review prompt|guardrail paths?|pull request body|pr body|pr title|raw diff)\b/gi; -const PUBLIC_UNSAFE_SCRUB = new RegExp(String.raw`\b(${PUBLIC_UNSAFE_TERMS})\b`, "gi"); const ALLOWED_CONTEXTS = new Set([ "loopover", "review", @@ -71,18 +59,6 @@ const ALLOWED_CONTEXTS = new Set([ "runtime", "os", ]); -const REDACTED = "[redacted]"; - -function nonBlank(value: string | undefined): string | undefined { - const trimmed = value?.trim(); - return trimmed ? trimmed : undefined; -} - -async function loadNodeHasher(): Promise { - const { createHash } = await import("node:crypto"); - digestHexSync = (input: string): string => - createHash("sha256").update(input).digest("hex"); -} const SENTRY_MONITORS: Record SentryMonitorConfig) }> = { "scheduled-loop": { @@ -253,132 +229,12 @@ export function scrubEvent(event: T): T | null { return event; } -function shouldRedactKey(key: string): boolean { - const compact = key.replace(/[^A-Za-z0-9]/g, "").toLowerCase(); - return ( - SECRET_KEY.test(key) || - PAYLOAD_KEY.test(key) || - /(body|payload|patch|diff|prompt|rubric|guardrail|header|cookie|title|config|reviewtext|reviewcontent|prcontent|pullrequest)/.test(compact) - ); -} - -function isInstallationIdKey(key: string): boolean { - return key.replace(/[^A-Za-z0-9]/g, "").toLowerCase() === "installationid"; -} - -function installationIdHash(value: unknown): string | undefined { - if (!digestHexSync) return undefined; - return hashedInstallationIdWith(value, digestHexSync); -} - -function hashedInstallationContext( - context: Record, -): Record { - const hasInstallationId = - "installation_id" in context || "installationId" in context; - const hash = installationIdHash(context.installation_id ?? context.installationId); - if (!hash && !hasInstallationId) return context; - const safe: Record = { ...context }; - if (hash) safe.installation_id_hash = hash; - delete safe.installation_id; - delete safe.installationId; - return safe; -} - function tagHashedInstallation(scope: SentryScope, context: Record): void { const hash = installationIdHash(context.installation_id ?? context.installationId); if (hash) scope.setTag("installation_id_hash", hash); } function applyOperationalTags(scope: SentryScope, context: Record): void { const normalized: Record = typeof context.repository === "string" && context.repo === undefined ? { ...context, repo: context.repository } : { ...context }; tagHashedInstallation(scope, normalized); for (const key of SENTRY_OPERATIONAL_TAG_KEYS) { const tagValue = normalized[key]; if (typeof tagValue === "string" || typeof tagValue === "number") scope.setTag(key, String(tagValue)); } } -function scrubString(value: string): string { - return value - .replace(QUERY_SECRET_VALUE, `$1${REDACTED}`) - .replace(SECRET_VALUE, REDACTED) - .replace(JWT_VALUE, REDACTED) - .replace(PUBLIC_LOCAL_PATH_SCRUB_PATTERN, "") - .replace(PUBLIC_UNSAFE_SCRUB, "private context") - .replace(PRIVATE_TEXT, "private context"); -} - -function scrubRecord(obj: unknown, depth: number): void { - if (!obj || typeof obj !== "object") return; - if (Array.isArray(obj)) { - for (let i = 0; i < obj.length; i++) { - const value = obj[i]; - if (typeof value === "string") obj[i] = scrubString(value); - else if (value && typeof value === "object") { - if (depth >= 6) obj[i] = REDACTED; - else scrubRecord(value, depth + 1); - } - } - return; - } - const rec = obj as Record; - for (const key of Object.keys(rec)) { - if (isInstallationIdKey(key)) { - const hash = installationIdHash(rec[key]); - if (hash) rec.installation_id_hash = hash; - delete rec[key]; - continue; - } - if (shouldRedactKey(key)) { - rec[key] = REDACTED; - continue; - } - const value = rec[key]; - if (typeof value === "string") rec[key] = scrubStringField(key, value); - else if (value && typeof value === "object") { - if (depth >= 6) rec[key] = REDACTED; - else scrubRecord(value, depth + 1); - } - } -} - -function scrubStringField(key: string, value: string): string { - if (isUrlKey(key)) return scrubUrl(value); - if (isQueryKey(key)) return scrubQueryString(value); - return scrubString(value); -} - -function isUrlKey(key: string): boolean { - return key.replace(/[^A-Za-z0-9]/g, "").toLowerCase().endsWith("url"); -} - -function isQueryKey(key: string): boolean { - const compact = key.replace(/[^A-Za-z0-9]/g, "").toLowerCase(); - return compact === "query" || compact === "querystring"; -} - -function scrubUrl(value: string): string { - const scrubbed = scrubString(value); - const queryStart = scrubbed.indexOf("?"); - if (queryStart === -1) return scrubbed; - try { - const parsed = new URL(scrubbed); - parsed.search = scrubQueryString(parsed.search); - return parsed.toString(); - } catch { - return `${scrubbed.slice(0, queryStart + 1)}${scrubQueryString( - scrubbed.slice(queryStart + 1), - )}`; - } -} - -function scrubQueryString(value: string): string { - const hasQuestionMark = value.startsWith("?"); - const source = hasQuestionMark ? value.slice(1) : value; - const params = new URLSearchParams(source); - for (const key of Array.from(new Set(params.keys()))) { - const values = params.getAll(key); - params.delete(key); - for (const entry of values) { - params.append(key, shouldRedactKey(key) ? REDACTED : scrubString(entry)); - } - } - const scrubbed = params.toString(); - return hasQuestionMark ? `?${scrubbed}` : scrubbed; -} function scrubRequest(request: Record | undefined): void { if (!request) return; @@ -750,7 +606,7 @@ export function resetSentryForTest(): void { sentryTraceSampleRate = undefined; active = false; sentryEnvironment = "production"; - digestHexSync = undefined; + resetRedactionScrubForTest(); } interface StructuredLogConsole { diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 6f7e13be58..1ca737647c 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -10,6 +10,7 @@ import { incr } from "./metrics"; import { withReviewSpan } from "./tracing"; import { withOtelSpan } from "./otel"; import { captureError, withSentryMonitor } from "./sentry"; +import { capturePostHogError, withPostHogMonitor } from "./posthog"; import { consumingRetryDelayMs, deterministicJitterMs, @@ -311,10 +312,15 @@ export function createSqliteQueue( * synchronous one it replaces -- see the call site). */ async function reviveDeadLetterJobsSafely(): Promise { try { - await withSentryMonitor( + await withPostHogMonitor( "queue-dead-letter-revive", { jobType: "queue-dead-letter-revive" }, - () => Promise.resolve(reviveDeadLetterJobs()), + () => + withSentryMonitor( + "queue-dead-letter-revive", + { jobType: "queue-dead-letter-revive" }, + () => Promise.resolve(reviveDeadLetterJobs()), + ), ); } catch (error) { console.error( @@ -325,6 +331,7 @@ export function createSqliteQueue( }), ); captureError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); + capturePostHogError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); } } @@ -446,6 +453,7 @@ export function createSqliteQueue( }), ); captureError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); + capturePostHogError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); } } @@ -831,6 +839,12 @@ export function createSqliteQueue( recovered, timeoutMs: processingTimeoutMs, }, "processing_timeout"); + capturePostHogError(new Error("self-host queue processing lease expired"), { + kind: "job_recovered", + reason: "processing_timeout", + recovered, + timeoutMs: processingTimeoutMs, + }, "processing_timeout"); } const job = claimNext(); if (!job) return false; @@ -859,6 +873,11 @@ export function createSqliteQueue( reason: "unparseable_payload", jobId: job.id, }, "unparseable_payload"); + capturePostHogError(new Error("unparseable queue payload"), { + kind: "job_dead", + reason: "unparseable_payload", + jobId: job.id, + }, "unparseable_payload"); return true; } const jobTraceParent = message.type === "github-webhook" ? message.traceParent : undefined; @@ -1118,6 +1137,13 @@ export function createSqliteQueue( jobId: job.id, attempts, }, "job_dead"); + capturePostHogError(error, { + kind: "job_dead", + reason: "max_retries_exhausted", + jobType: extractPayloadType(job.payload), + jobId: job.id, + attempts, + }, "job_dead"); } else { const retryDelayMs = consumingRetryDelayMs(error, backoff(attempts)); driver.query( @@ -1168,6 +1194,7 @@ export function createSqliteQueue( }), ); captureError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); + capturePostHogError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); } finally { active--; } diff --git a/src/server.ts b/src/server.ts index 4991d96214..38a77e48a9 100644 --- a/src/server.ts +++ b/src/server.ts @@ -91,6 +91,13 @@ import { initSentry, installStructuredLogForwarding, } from "./selfhost/sentry"; +import { + capturePostHogError, + flushPostHog, + initPostHog, + installPostHogStructuredLogForwarding, + shutdownPostHog, +} from "./selfhost/posthog"; import { drainOrbRelayWithMonitor, registerOrbRelayWithMonitor, @@ -354,6 +361,17 @@ async function main(): Promise { // stderr. Wrap both sinks so every level:"error"/"fatal" line surfaces as a Sentry issue WITHOUT per-site wiring. installStructuredLogForwarding(); } + // PostHog error tracking (#8287, epic #8286): opt-in via POSTHOG_API_KEY -- the same var #6235's MCP telemetry + // already reads. PARALLEL-RUN: active alongside Sentry above, not instead of it, until the gated decommission + // issue (#8298) says otherwise. enableExceptionAutocapture (set inside initPostHog) already installs its own + // uncaughtException/unhandledRejection handlers per PostHog's own documented Node.js setup, so -- unlike + // Sentry's explicit process.on wiring above, added before that capability existed -- no manual mirror is + // needed here for the crash case; only structured-log forwarding needs an explicit install, same as Sentry. + const posthogEnabled = await initPostHog(process.env); + if (posthogEnabled) { + console.log(JSON.stringify({ event: "selfhost_posthog", environment: process.env.POSTHOG_ENVIRONMENT ?? "production" })); + installPostHogStructuredLogForwarding(); + } if (await initOpenTelemetry(process.env, sentryEnabled ? await buildSentryOpenTelemetryBridge() : undefined)) console.log(JSON.stringify({ event: "selfhost_otel", traces: openTelemetryTraceExportEnabled(process.env) ? "otlp" : "sentry" })); /* v8 ignore stop */ @@ -1221,7 +1239,10 @@ async function main(): Promise { state: orbRelayRegistrationState, register: registerOrbRelayTargetWithRetry, ...(relayDrainState ? { drainState: relayDrainState } : {}), - }).catch((error) => captureError(error, { kind: "orb_relay_register" }, "orb_relay_register")); + }).catch((error) => { + captureError(error, { kind: "orb_relay_register" }, "orb_relay_register"); + capturePostHogError(error, { kind: "orb_relay_register" }, "orb_relay_register"); + }); void attemptOrbRelayRegistration(); setInterval(() => void attemptOrbRelayRegistration(), 60_000); // Dashboard-visible counterparts to the streak/no-progress alert gate in isOrbRelayRegistrationAlerting: @@ -1245,7 +1266,11 @@ async function main(): Promise { }; if (isD1SizeProbeEnabled(d1ProbeEnv)) { /* v8 ignore start -- self-host entrypoint timer; probe logic itself is unit-tested in d1-size-probe.test.ts. */ - const runD1Probe = () => runD1SizeProbe(d1ProbeEnv).catch((error) => captureError(error, { kind: "d1_size_probe" }, "d1_size_probe")); + const runD1Probe = () => + runD1SizeProbe(d1ProbeEnv).catch((error) => { + captureError(error, { kind: "d1_size_probe" }, "d1_size_probe"); + capturePostHogError(error, { kind: "d1_size_probe" }, "d1_size_probe"); + }); void runD1Probe(); setInterval(runD1Probe, 900_000); /* v8 ignore stop */ @@ -1271,16 +1296,18 @@ async function main(): Promise { enqueue: enqueueWebhookByEnv, }), ); - void drainRelay().catch((error) => - captureError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"), - ); + void drainRelay().catch((error) => { + captureError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); + capturePostHogError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); + }); // 30s matches broker-client's request timeout so a slow/degraded broker's in-flight drain has fully // timed out (or completed) before the next tick would otherwise pile another request on top of it. setInterval( () => - void drainRelay().catch((error) => - captureError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"), - ), + void drainRelay().catch((error) => { + captureError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); + capturePostHogError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); + }), 30_000, ); /* v8 ignore stop */ @@ -1298,6 +1325,7 @@ async function main(): Promise { /* v8 ignore next -- graceful process signal path is not imported in unit tests; shutdown helper is covered. */ await shutdownOpenTelemetry(); await flushSentry(); + await shutdownPostHog(); process.exit(0); }; process.on("SIGTERM", () => void shutdown("SIGTERM")); @@ -1306,7 +1334,8 @@ async function main(): Promise { main().catch((error) => { captureError(error, { kind: "boot" }, "boot"); + capturePostHogError(error, { kind: "boot" }, "boot"); console.error(error); /* v8 ignore next -- boot failure exits the process; shutdown helper is covered independently. */ - void Promise.all([shutdownOpenTelemetry(), flushSentry()]).finally(() => process.exit(1)); + void Promise.all([shutdownOpenTelemetry(), flushSentry(), flushPostHog()]).finally(() => process.exit(1)); }); diff --git a/src/services/agent-action-executor.ts b/src/services/agent-action-executor.ts index d43636d927..5ea34b6d3d 100644 --- a/src/services/agent-action-executor.ts +++ b/src/services/agent-action-executor.ts @@ -41,6 +41,7 @@ import { import { incr } from "../selfhost/metrics"; import { shouldWaitForOlderSiblings } from "../review/merge-train"; import { captureError } from "../selfhost/sentry"; +import { capturePostHogError } from "../selfhost/posthog"; import { claimContributorCapLock, releaseContributorCapLock } from "../queue/transient-locks"; // The agent actor name on every audit record — the App acts on the maintainer's behalf per their configured @@ -639,6 +640,7 @@ export async function executeAgentMaintenanceActions(env: Env, ctx: AgentActionE // for review-pass failures (selfhost/sentry.ts's captureReviewFailure, queue/processors.ts). Previously // this class of failure was audit-log-only, invisible without a manual audit_events query. captureError(error, { kind: "agent_action_execution_failed", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_action_execution_failed"); + capturePostHogError(error, { kind: "agent_action_execution_failed", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_action_execution_failed"); } // #2265: a permission-looking 403 on a PR-write mutation can mean the LOCAL installations.permissions // snapshot is stale after a maintainer-initiated downgrade (GitHub sends no downgrade webhook). Rate-limit @@ -941,6 +943,7 @@ export async function executeIssueMaintenanceActions(env: Env, ctx: IssueActionE // Mirrors executeAgentMaintenanceActions's non-merge capture below -- issue-side label/close has no retry // loop either, so a single failure here is already this pass's terminal outcome. captureError(error, { kind: "agent_issue_action_execution_failed", repo: ctx.repoFullName, issue: ctx.issueNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_issue_action_execution_failed"); + capturePostHogError(error, { kind: "agent_issue_action_execution_failed", repo: ctx.repoFullName, issue: ctx.issueNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_issue_action_execution_failed"); } } @@ -977,6 +980,7 @@ async function handleMergeFailure(env: Env, ctx: AgentActionExecutionContext, er // merge hold groups under one readable title regardless of which HTTP status caused it -- the specific // status/reason stays in the message and the "review" context object either way. captureError(error, { kind: "agent_merge_blocked", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, reason: reason.slice(0, 280) }, "agent_merge_blocked"); + capturePostHogError(error, { kind: "agent_merge_blocked", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, reason: reason.slice(0, 280) }, "agent_merge_blocked"); await recordAuditEvent(env, { eventType: "agent.action.merge_blocked", actor: AGENT_ACTOR, diff --git a/test/unit/docs-selfhost-posthog-observability.test.ts b/test/unit/docs-selfhost-posthog-observability.test.ts new file mode 100644 index 0000000000..d002c35143 --- /dev/null +++ b/test/unit/docs-selfhost-posthog-observability.test.ts @@ -0,0 +1,35 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +import { POSTHOG_MONITOR_HEARTBEAT_EVENT } from "../../src/selfhost/posthog"; + +// Drift guard (#8287): self-host PostHog docs must stay aligned with the exported monitor-heartbeat event +// name, mirroring docs-selfhost-sentry-observability.test.ts's identical discipline for Sentry. + +const OPERATIONS = "apps/loopover-ui/content/docs/self-hosting-operations.mdx"; +const operations = readFileSync(OPERATIONS, "utf8"); + +describe("self-host PostHog observability docs (#8287)", () => { + it("documents enabling PostHog as opt-in and parallel-run alongside Sentry", () => { + expect(operations).toContain("Enabling PostHog error tracking"); + expect(operations).toContain("POSTHOG_API_KEY"); + expect(operations).toContain("opt-in and off by default"); + expect(operations).toContain("parallel with Sentry"); + }); + + it("documents the shared redaction module both sinks use", () => { + expect(operations).toContain("redaction-scrub.ts"); + }); + + it("documents exception autocapture", () => { + expect(operations).toContain("exception autocapture"); + }); + + it("documents the cron-monitor heartbeat replacement with the real exported event name", () => { + expect(operations).toContain("Cron Monitors"); + expect(operations).toContain(POSTHOG_MONITOR_HEARTBEAT_EVENT); + for (const monitor of ["scheduled-loop", "orb-export", "orb-relay-drain", "orb-relay-register", "queue-dead-letter-revive"]) { + expect(operations).toContain(monitor); + } + }); +}); diff --git a/test/unit/selfhost-posthog.test.ts b/test/unit/selfhost-posthog.test.ts new file mode 100644 index 0000000000..2b1b34161f --- /dev/null +++ b/test/unit/selfhost-posthog.test.ts @@ -0,0 +1,491 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Mock posthog-node so the dynamic import inside initPostHog() resolves to a spy-backed client. Hoisted so +// vi.mock can see it, mirroring selfhost-sentry.test.ts's identical @sentry/node mocking pattern. +const mocks = vi.hoisted(() => { + const captureException = vi.fn(); + const capture = vi.fn(); + const flush = vi.fn().mockResolvedValue(undefined); + const shutdown = vi.fn().mockResolvedValue(undefined); + let lastOptions: any; + const PostHog = vi.fn(function (this: any, _apiKey: string, options: any) { + lastOptions = options; + this.captureException = captureException; + this.capture = capture; + this.flush = flush; + this.shutdown = shutdown; + }); + return { captureException, capture, flush, shutdown, PostHog, getLastOptions: () => lastOptions }; +}); +const otelMocks = vi.hoisted(() => ({ currentOtelTraceIds: vi.fn() })); +vi.mock("posthog-node", () => ({ PostHog: mocks.PostHog })); +vi.mock("../../src/selfhost/otel", () => ({ + currentOtelTraceIds: otelMocks.currentOtelTraceIds, +})); + +import { + capturePostHogError, + capturePostHogReviewFailure, + flushPostHog, + forwardStructuredLogToPostHog, + initPostHog, + installPostHogStructuredLogForwarding, + POSTHOG_MONITOR_HEARTBEAT_EVENT, + resetPostHogForTest, + resolvePostHogRelease, + scrubPostHogEvent, + shutdownPostHog, + withPostHogMonitor, +} from "../../src/selfhost/posthog"; + +beforeEach(() => { + resetPostHogForTest(); + vi.clearAllMocks(); + otelMocks.currentOtelTraceIds.mockReturnValue(undefined); +}); + +const lastCapturedException = (): Error => mocks.captureException.mock.calls.at(-1)?.[0] as Error; +const lastCapturedProperties = (): Record => mocks.captureException.mock.calls.at(-1)?.[2] as Record; +const fakeClassicAccessToken = (): string => `${"github" + "_pat_"}${"a".repeat(24)}`; + +describe("initPostHog", () => { + it("stays a no-op when POSTHOG_API_KEY is unset", async () => { + const enabled = await initPostHog({} as unknown as NodeJS.ProcessEnv); + expect(enabled).toBe(false); + expect(mocks.PostHog).not.toHaveBeenCalled(); + }); + + it("stays a no-op when POSTHOG_API_KEY is blank/whitespace", async () => { + expect(await initPostHog({ POSTHOG_API_KEY: " " } as unknown as NodeJS.ProcessEnv)).toBe(false); + }); + + it("activates with the configured key and default host when POSTHOG_HOST is unset", async () => { + const enabled = await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + expect(enabled).toBe(true); + expect(mocks.PostHog).toHaveBeenCalledWith("phc_test_key", expect.objectContaining({ host: "https://us.i.posthog.com" })); + }); + + it("uses POSTHOG_HOST when set", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", POSTHOG_HOST: "https://eu.i.posthog.com" } as unknown as NodeJS.ProcessEnv); + expect(mocks.PostHog).toHaveBeenCalledWith("phc_test_key", expect.objectContaining({ host: "https://eu.i.posthog.com" })); + }); + + it("enables exception autocapture and installs the before_send scrubber", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const options = mocks.getLastOptions(); + expect(options.enableExceptionAutocapture).toBe(true); + expect(options.before_send).toBe(scrubPostHogEvent); + }); +}); + +describe("resolvePostHogRelease", () => { + it("prefers an explicit POSTHOG_RELEASE override", () => { + expect(resolvePostHogRelease({ POSTHOG_RELEASE: "1.2.3", LOOPOVER_VERSION: "9.9.9" } as unknown as NodeJS.ProcessEnv)).toBe("1.2.3"); + }); + it("falls back to LOOPOVER_VERSION when POSTHOG_RELEASE is unset", () => { + expect(resolvePostHogRelease({ LOOPOVER_VERSION: "9.9.9" } as unknown as NodeJS.ProcessEnv)).toBe("9.9.9"); + }); + it("is undefined when neither is set", () => { + expect(resolvePostHogRelease({} as unknown as NodeJS.ProcessEnv)).toBeUndefined(); + }); +}); + +describe("scrubPostHogEvent — redact secrets before an event leaves the box", () => { + it("passes null through unchanged", () => { + expect(scrubPostHogEvent(null)).toBeNull(); + }); + + it("redacts secret-keyed properties, recurses, and leaves safe fields", () => { + const event = scrubPostHogEvent({ + event: "$exception", + properties: { + jobId: "j1", + apiKey: "shh", + nested: { secretToken: "deep" }, + }, + } as any)!; + expect((event.properties as any).apiKey).toBe("[redacted]"); + expect((event.properties as any).jobId).toBe("j1"); + expect((event.properties as any).nested.secretToken).toBe("[redacted]"); + }); + + it("is safe when properties is absent", () => { + expect(() => scrubPostHogEvent({ event: "x" } as any)).not.toThrow(); + expect(scrubPostHogEvent({ event: "x" } as any)).toEqual({ event: "x" }); + }); + + it("redacts a real credential-shaped value embedded in a property string", () => { + const fakeToken = fakeClassicAccessToken(); + const event = scrubPostHogEvent({ + event: "$exception", + properties: { note: `token leaked: ${fakeToken} at /home/alice/project` }, + } as any)!; + expect((event.properties as any).note).not.toContain(fakeToken); + expect((event.properties as any).note).toContain(""); + }); + + it("scrubs the event name string too", () => { + const fakeToken = fakeClassicAccessToken(); + const event = scrubPostHogEvent({ event: `leaked ${fakeToken}`, properties: {} } as any)!; + expect(event.event).not.toContain(fakeToken); + }); + + it("leaves a non-string event name untouched", () => { + const event = scrubPostHogEvent({ event: 123 as unknown as string, properties: {} } as any)!; + expect(event.event).toBe(123); + }); + + it("returns null (fail-closed) when scrubbing itself throws", () => { + const properties: Record = {}; + Object.defineProperty(properties, "poison", { + enumerable: true, + get(): never { + throw new Error("boom during scrub"); + }, + }); + expect(scrubPostHogEvent({ event: "x", properties } as any)).toBeNull(); + }); +}); + +describe("capturePostHogError", () => { + it("is a no-op when PostHog is unconfigured", () => { + capturePostHogError(new Error("boom")); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("captures with a named error and operational properties when configured", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom"), { repo: "owner/repo", pull: 7 }, "my_event"); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + const captured = lastCapturedException(); + expect(captured.name).toBe("my_event"); + expect(captured.message).toBe("boom"); + const properties = lastCapturedProperties(); + expect(properties.repo).toBe("owner/repo"); + expect(properties.pull).toBe(7); + }); + + it("captures without a custom name when eventName is omitted", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom")); + expect(lastCapturedException().message).toBe("boom"); + }); + + it("wraps a non-Error thrown value", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError("a string error", undefined, "my_event"); + expect(lastCapturedException()).toBeInstanceOf(Error); + expect(lastCapturedException().message).toBe("a string error"); + }); + + it("respects POSTHOG_MIN_SEVERITY -- suppressed above error", async () => { + process.env.POSTHOG_MIN_SEVERITY = "critical"; + try { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom")); + expect(mocks.captureException).not.toHaveBeenCalled(); + } finally { + delete process.env.POSTHOG_MIN_SEVERITY; + } + }); + + it("hashes an installation id in the context instead of leaking the raw value", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom"), { installation_id: 12345 }); + const properties = lastCapturedProperties(); + expect(properties.installation_id).toBeUndefined(); + expect(properties.installationId).toBeUndefined(); + }); + + it("attaches otel trace ids to properties when present", async () => { + otelMocks.currentOtelTraceIds.mockReturnValue({ trace_id: "t1", span_id: "s1" }); + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom")); + const properties = lastCapturedProperties(); + expect(properties.trace_id).toBe("t1"); + expect(properties.span_id).toBe("s1"); + }); + + it("does not overwrite an already-present repo when only repository is normalized", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom"), { repo: "explicit/repo", repository: "should-be-ignored/repo" }); + expect(lastCapturedProperties().repo).toBe("explicit/repo"); + }); + + it("resolves the per-repo severity threshold from a repository-only context (no repo field)", async () => { + process.env.POSTHOG_REPO_MIN_SEVERITY = JSON.stringify({ "owner/repo": "critical" }); + try { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogError(new Error("boom"), { repository: "owner/repo" }); + expect(mocks.captureException).not.toHaveBeenCalled(); + } finally { + delete process.env.POSTHOG_REPO_MIN_SEVERITY; + } + }); +}); + +describe("capturePostHogReviewFailure", () => { + it("is a no-op when PostHog is unconfigured", () => { + capturePostHogReviewFailure(new Error("review failed")); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("captures at error grade with kind: review_failure tagged", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogReviewFailure(new Error("review failed"), { repo: "owner/repo" }, "review_event"); + expect(lastCapturedException().name).toBe("review_event"); + expect(lastCapturedProperties().kind).toBe("review_failure"); + }); + + it("respects POSTHOG_MIN_SEVERITY -- suppressed above error", async () => { + process.env.POSTHOG_MIN_SEVERITY = "critical"; + try { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogReviewFailure(new Error("review failed")); + expect(mocks.captureException).not.toHaveBeenCalled(); + } finally { + delete process.env.POSTHOG_MIN_SEVERITY; + } + }); +}); + +describe("forwardStructuredLogToPostHog", () => { + it("is a no-op when PostHog is unconfigured", () => { + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "x" })); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("ignores non-string and non-JSON-object lines", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(42); + forwardStructuredLogToPostHog("not json"); + forwardStructuredLogToPostHog("[1,2,3]"); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("ignores a line that starts with { but fails to parse", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog("{not valid json"); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("includes the repo#pr location suffix in a field-only log's summary", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", repo: "owner/repo", pullNumber: 5 })); + expect(lastCapturedException().message).toContain("(owner/repo#5)"); + }); + + it("includes just the repo when a field-only log has no pullNumber", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", repo: "owner/repo" })); + expect(lastCapturedException().message).toContain("(owner/repo)"); + }); + + it("redacts a secret-keyed field inside a nested object value when summarizing", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "close_breaker_engaged", details: { apiKey: "shh", floor: 0.8 } })); + expect(lastCapturedException().message).toContain("[redacted]"); + expect(lastCapturedException().message).not.toContain("shh"); + }); + + it("redacts inside an array-valued summarized field", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "x", affectedTargets: ["a", "b"] })); + expect(lastCapturedException().message).toContain("affectedTargets="); + }); + + it("caps deeply-nested summarized values at the redaction depth guard", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + let deep: unknown = { apiKey: "shh" }; + for (let i = 0; i < 8; i++) deep = { a: deep }; + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "x", deep })); + expect(lastCapturedException().message).toContain("[redacted]"); + }); + + it("falls back to obj.error when message is absent", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "job_dead", error: "the error string" })); + expect(lastCapturedException().message).toBe("the error string"); + }); + + it("forwards an explicit level:error JSON line", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "job_dead", message: "bad thing" })); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + expect(lastCapturedException().name).toBe("job_dead"); + expect(lastCapturedException().message).toBe("bad thing"); + }); + + it("ignores a level:warn line when not from the error sink", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "warn", event: "x" }), false); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("defaults to error level when fromErrorSink is true and no explicit level is present", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ event: "x" }), true); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + }); + + it("is skipped entirely when there is no severity signal at all (no level, not from error sink)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ event: "x" }), false); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("folds a sub-event (ev) into the synthetic error name", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "rag_failure", ev: "upsert_error" })); + expect(lastCapturedException().name).toBe("rag_failure/upsert_error"); + }); + + it("summarizes salient fields when no message/error is present", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", event: "close_breaker_engaged", project: "x", closePrecision: 0.6 })); + expect(lastCapturedException().message).toContain("project=x"); + }); + + it("falls back to a context pointer when there is truly nothing to summarize", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error" })); + expect(lastCapturedException().message).toBe("(no message — see the log context)"); + }); + + it("normalizes an unrecognized level word (e.g. a log CATEGORY, not a severity) to info, never promoting it to error", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "debug", event: "x" })); + // info < the default "error" threshold, so this stays suppressed -- proves debug did NOT get promoted. + expect(mocks.captureException).not.toHaveBeenCalled(); + }); + + it("normalizes fatal to critical severity", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "fatal", event: "x", message: "very bad" })); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + }); + + it("uses a repository-only field for the location suffix (no repo field)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToPostHog(JSON.stringify({ level: "error", repository: "owner/repo" })); + expect(lastCapturedException().message).toContain("(owner/repo)"); + }); +}); + +describe("installPostHogStructuredLogForwarding", () => { + it("forwards console.error lines as error-sink by default", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const target = { log: vi.fn(), error: vi.fn() }; + installPostHogStructuredLogForwarding(target); + target.error(JSON.stringify({ event: "x" })); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + }); + + it("forwards console.log lines only when they carry an explicit level:error", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const target = { log: vi.fn(), error: vi.fn() }; + installPostHogStructuredLogForwarding(target); + target.log(JSON.stringify({ event: "x" })); + expect(mocks.captureException).not.toHaveBeenCalled(); + target.log(JSON.stringify({ level: "error", event: "y" })); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + }); + + it("still calls the original console method", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const originalLog = vi.fn(); + const target = { log: originalLog, error: vi.fn() }; + installPostHogStructuredLogForwarding(target); + target.log("plain text"); + expect(originalLog).toHaveBeenCalledWith("plain text"); + }); + + it("guards against re-entrant forwarding when capture itself logs", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + mocks.captureException.mockImplementationOnce(() => { + target.error(JSON.stringify({ event: "recursive" })); + }); + const target = { log: vi.fn(), error: vi.fn() }; + installPostHogStructuredLogForwarding(target); + expect(() => target.error(JSON.stringify({ level: "error", event: "outer" }))).not.toThrow(); + }); +}); + +describe("withPostHogMonitor", () => { + it("just runs the callback when PostHog is unconfigured", async () => { + const result = await withPostHogMonitor("orb-export", undefined, async () => "ok"); + expect(result).toBe("ok"); + expect(mocks.capture).not.toHaveBeenCalled(); + }); + + it("emits an ok heartbeat on success", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const result = await withPostHogMonitor("orb-export", { jobType: "orb-export" }, async () => "done"); + expect(result).toBe("done"); + expect(mocks.capture).toHaveBeenCalledWith( + expect.objectContaining({ event: POSTHOG_MONITOR_HEARTBEAT_EVENT, properties: expect.objectContaining({ monitor: "orb-export", status: "ok" }) }), + ); + }); + + it("emits an error heartbeat AND captures the exception on failure, then rethrows", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const boom = new Error("orb export failed"); + await expect(withPostHogMonitor("orb-export", { jobType: "orb-export" }, async () => { throw boom; })).rejects.toThrow("orb export failed"); + expect(mocks.capture).toHaveBeenCalledWith( + expect.objectContaining({ event: POSTHOG_MONITOR_HEARTBEAT_EVENT, properties: expect.objectContaining({ monitor: "orb-export", status: "error" }) }), + ); + expect(mocks.captureException).toHaveBeenCalledTimes(1); + }); + + it("wraps a non-Error thrown value on failure", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + await expect(withPostHogMonitor("orb-export", undefined, async () => { throw "a string failure"; })).rejects.toBe("a string failure"); + expect(lastCapturedException()).toBeInstanceOf(Error); + expect(lastCapturedException().message).toBe("a string failure"); + }); +}); + +describe("flushPostHog / shutdownPostHog", () => { + it("flushPostHog is a no-op when unconfigured", async () => { + await flushPostHog(); + expect(mocks.flush).not.toHaveBeenCalled(); + }); + + it("flushPostHog calls the client's flush when configured", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + await flushPostHog(); + expect(mocks.flush).toHaveBeenCalledTimes(1); + }); + + it("flushPostHog swallows a flush rejection", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + mocks.flush.mockRejectedValueOnce(new Error("flush failed")); + await expect(flushPostHog()).resolves.toBeUndefined(); + }); + + it("shutdownPostHog is a no-op when unconfigured", async () => { + await shutdownPostHog(); + expect(mocks.shutdown).not.toHaveBeenCalled(); + }); + + it("shutdownPostHog calls the client's shutdown when configured", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + await shutdownPostHog(); + expect(mocks.shutdown).toHaveBeenCalledTimes(1); + }); + + it("shutdownPostHog swallows a shutdown rejection", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + mocks.shutdown.mockRejectedValueOnce(new Error("shutdown failed")); + await expect(shutdownPostHog()).resolves.toBeUndefined(); + }); +}); + +describe("resetPostHogForTest", () => { + it("resets active state so a subsequent capture is a no-op again", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + resetPostHogForTest(); + capturePostHogError(new Error("boom")); + expect(mocks.captureException).not.toHaveBeenCalled(); + }); +});