feat(cli): telemetry consent controls, redaction hardening, retry queue, www endpoint

docs/telemetry-enhancements.md

@@ -57,6 +57,19 @@ Possible improvements to the ChaiBuilder CLI telemetry (CLI events, the
 
 **No endpoint change required:** `distinct_id`/`run_id` are existing top-level fields (values only changed); `is_ci`, `code`, `duration_ms` ride inside `properties`, which the endpoint forwards to PostHog wholesale.
 
+### Implemented — phase 2 (robustness, endpoint, debug)
+
+- **2.5 redaction hardening** — key=value credentials (incl. `FOO_SECRET=`), Bearer tokens, prefixed keys (`ghp_`/`sk-`/`AKIA`/`xox…`), more URL schemes; + a fuzz test.
+- **3.2 www endpoint** — default is `https://www.chaibuilder.com/in/cli/track` (skips the 308 hop; verified non-www 308→www and www responds directly).
+- **3.6 retry queue** — failed sends persist to `telemetry-queue.jsonl` (capped 50) and drain + resend on the next run (transient/5xx only; 4xx dropped).
+- **6.1 `CHAIBUILDER_TELEMETRY_DEBUG`** — echoes each event to stderr.
+
+**Deliberately not done — consent UI (2.1 / 2.2 / 2.3 / 2.4):** telemetry is anonymous, so there is no first-run notice, no `telemetry` subcommand, and no persisted config opt-out. Opt-out is env-only: `DO_NOT_TRACK=1` or `CHAIBUILDER_TELEMETRY_DISABLED=1`.
+
+**N/A — 1.7 homepage source:** the CLI has no remote homepage fetch (the first page always uses bundled `DEFAULT_BLOCKS`), so there's no remote-vs-fallback to segment. Revisit only if a remote homepage fetch is added.
+
+**Still not in this repo:** 3.1 batching / 3.3 rate-limit / 3.4 geo (cbpl endpoint), 4.x dashboards (PostHog UI), 5.2/5.4 CI (cli-e2e has no remote), 1.6 extra dimensions (needs a data-scope decision), 3.5 posthog-node (would bypass the server-side endpoint — not recommended).
+
 ---
 
 ## 2. Privacy & consent

src/core/real-adapters.ts

@@ -9,6 +9,7 @@ import path from 'node:path'
 import { execa } from 'execa'
 import { downloadTemplate } from 'giget'
 import { createTelemetry, resolveDistinctId, telemetryDisabled } from '../lib/telemetry.js'
+import { telemetryQueuePath } from '../lib/telemetry-queue.js'
 import type {
   ExecPort,
   FsPort,
@@ -227,6 +228,7 @@ function createLog(): LoggerPort {
 }
 
 export function createRealPorts(): Ports {
+  const optedOut = telemetryDisabled()
   return {
     prompts: createPrompts(),
     fs: createFs(),
@@ -235,7 +237,9 @@ export function createRealPorts(): Ports {
     log: createLog(),
     telemetry: createTelemetry({
       cliVersion: cliVersion(),
-      distinctId: telemetryDisabled() ? undefined : resolveDistinctId(),
+      disabled: optedOut,
+      distinctId: optedOut ? undefined : resolveDistinctId(),
+      queueFile: telemetryQueuePath(),
     }),
     randomBytes: (size) => nodeRandomBytes(size),
   }

src/lib/telemetry-queue.ts

@@ -0,0 +1,39 @@
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { dirname, join } from 'node:path'
+
+const MAX_QUEUED = 50
+
+export function telemetryQueuePath(): string {
+  return join(homedir(), '.config', 'chaibuilder', 'telemetry-queue.jsonl')
+}
+
+function readQueue(file: string): string[] {
+  try {
+    return readFileSync(file, 'utf8')
+      .split('\n')
+      .filter((l) => l.trim())
+  } catch {
+    return []
+  }
+}
+
+/** Append a failed event body, keeping only the most recent MAX_QUEUED. Best-effort. */
+export function enqueueFailed(body: string, file: string = telemetryQueuePath()): void {
+  try {
+    const next = [...readQueue(file), body].slice(-MAX_QUEUED)
+    mkdirSync(dirname(file), { recursive: true })
+    writeFileSync(file, next.join('\n') + '\n')
+  } catch {}
+}
+
+/** Read and remove all queued bodies. Best-effort; returns [] when empty. */
+export function drainQueue(file: string = telemetryQueuePath()): string[] {
+  const items = readQueue(file)
+  if (items.length) {
+    try {
+      rmSync(file, { force: true })
+    } catch {}
+  }
+  return items
+}

src/lib/telemetry.ts

@@ -3,8 +3,9 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
 import { homedir } from 'node:os'
 import { dirname, join } from 'node:path'
 import type { TelemetryPort } from '../core/adapters.js'
+import { drainQueue, enqueueFailed } from './telemetry-queue.js'
 
-const DEFAULT_ENDPOINT = 'https://chaibuilder.com/in/cli/track'
+const DEFAULT_ENDPOINT = 'https://www.chaibuilder.com/in/cli/track'
 const SEND_TIMEOUT_MS = 2000
 
 export type TelemetryOptions = {
@@ -17,6 +18,8 @@ export type TelemetryOptions = {
   fetchImpl?: typeof fetch
   /** Persistent anonymous id (per machine). Falls back to a per-run id if omitted. */
   distinctId?: string
+  /** File for the failed-send retry queue. When set, failed sends are queued and drained on init. */
+  queueFile?: string
 }
 
 export type StepAction = 'start' | 'finish'
@@ -27,6 +30,11 @@ export function telemetryDisabled(): boolean {
   return Boolean(process.env.CHAIBUILDER_TELEMETRY_DISABLED || process.env.DO_NOT_TRACK)
 }
 
+/** Debug mode: echo each event to stderr instead of/alongside sending. */
+export function telemetryDebug(): boolean {
+  return Boolean(process.env.CHAIBUILDER_TELEMETRY_DEBUG)
+}
+
 /** `cmd:step:start` for a start, `cmd:step:finish:result` for a finish. */
 export function stepEvent(cmd: string, step: string, action: StepAction, result?: StepResult): string {
   return result ? `${cmd}:${step}:${action}:${result}` : `${cmd}:${step}:${action}`
@@ -106,10 +114,21 @@ export function createTelemetry(opts: TelemetryOptions): TelemetryPort {
     os: process.platform,
   }
   const baseProps = { is_ci: isCI() }
+  const debug = telemetryDebug()
+  const queueFile = opts.queueFile
   const inflight = new Set<Promise<void>>()
 
+  function post(body: string): void {
+    if (typeof doFetch !== 'function') return
+    const promise = send(doFetch, endpoint, body).then((result) => {
+      if (result === 'retry' && queueFile) enqueueFailed(body, queueFile)
+    })
+    inflight.add(promise)
+    void promise.finally(() => inflight.delete(promise))
+  }
+
   function capture(event: string, properties: Record<string, unknown> = {}): void {
-    if (disabled || typeof doFetch !== 'function') return
+    if (disabled) return
     let body: string
     try {
       body = JSON.stringify({
@@ -121,9 +140,13 @@ export function createTelemetry(opts: TelemetryOptions): TelemetryPort {
     } catch {
       return
     }
-    const promise = send(doFetch, endpoint, body)
-    inflight.add(promise)
-    void promise.finally(() => inflight.delete(promise))
+    if (debug) process.stderr.write(`[telemetry] ${body}\n`)
+    post(body)
+  }
+
+  // Retry events that failed to send on a previous run.
+  if (!disabled && queueFile && typeof doFetch === 'function') {
+    for (const body of drainQueue(queueFile)) post(body)
   }
 
   async function flush(timeoutMs = SEND_TIMEOUT_MS): Promise<void> {
@@ -148,28 +171,42 @@ export function errorDetails(err: unknown): Record<string, unknown> {
   const e = err as { name?: string; message?: unknown; constructor?: { name?: string } }
   return {
     error_type: e?.name || e?.constructor?.name || 'Error',
-    error: redact(String(e?.message ?? err)).slice(0, 300),
+    error: redact(String(e?.message ?? err).slice(0, 2000)).slice(0, 300),
   }
 }
 
 /** Strip anything potentially identifying/secret from a free-text error message. */
 export function redact(s: string): string {
   return s
     .replace(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi, '<email>')
-    .replace(/\b(?:libsql|https?|wss?|postgres(?:ql)?|file):\/\/\S+/gi, '<url>')
+    .replace(/\b(?:libsql|https?|wss?|postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqps?|file):\/\/\S+/gi, '<url>')
+    .replace(/\bbearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer <token>')
+    .replace(
+      /\b(\w*(?:password|passwd|pwd|secret|token|api[_-]?key|auth[_-]?token|access[_-]?key|client[_-]?secret))(["']?\s*[:=]\s*["']?)([^\s"'&,;]+)/gi,
+      '$1$2<redacted>',
+    )
+    .replace(/\b(?:ghp|gho|ghs|ghr|ghu|xox[baprs])[_-][A-Za-z0-9_-]{6,}/g, '<token>')
+    .replace(/\b(?:sk|pk|rk)[_-][A-Za-z0-9_-]{10,}/g, '<token>')
+    .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{12,}\b/g, '<token>')
     .replace(/[A-Za-z]:\\[^\s"']+/g, '<path>')
     .replace(/(?:\/[^\s/:"']+){2,}/g, '<path>')
     .replace(/\b[A-Za-z0-9_-]{24,}\b/g, '<token>')
 }
 
-/** POST one event; never throws, and always settles within SEND_TIMEOUT_MS. */
-async function send(doFetch: typeof fetch, endpoint: string, body: string): Promise<void> {
+type SendResult = 'ok' | 'retry' | 'drop'
+
+/**
+ * POST one event; never throws, settles within SEND_TIMEOUT_MS.
+ * `ok` = delivered, `retry` = transient (network/timeout/5xx) so re-queue,
+ * `drop` = permanent (4xx) so don't re-queue.
+ */
+async function send(doFetch: typeof fetch, endpoint: string, body: string): Promise<SendResult> {
   const controller = new AbortController()
   let timer: ReturnType<typeof setTimeout> | undefined
-  const timeout = new Promise<void>((resolve) => {
+  const timeout = new Promise<SendResult>((resolve) => {
     timer = setTimeout(() => {
       controller.abort()
-      resolve()
+      resolve('retry')
     }, SEND_TIMEOUT_MS)
     timer.unref?.()
   })
@@ -178,9 +215,16 @@ async function send(doFetch: typeof fetch, endpoint: string, body: string): Prom
     headers: { 'content-type': 'application/json' },
     body,
     signal: controller.signal,
-  }).then(() => undefined, () => undefined)
+  }).then(
+    (res): SendResult => {
+      if (!res || typeof res.status !== 'number') return 'ok'
+      if (res.ok) return 'ok'
+      return res.status >= 500 ? 'retry' : 'drop'
+    },
+    (): SendResult => 'retry',
+  )
   try {
-    await Promise.race([attempt, timeout])
+    return await Promise.race([attempt, timeout])
   } finally {
     if (timer) clearTimeout(timer)
   }

tests/lib/telemetry-queue.test.ts

@@ -0,0 +1,30 @@
+import { mkdtempSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { drainQueue, enqueueFailed } from '../../src/lib/telemetry-queue.js'
+
+const tmpQueue = () => join(mkdtempSync(join(tmpdir(), 'chai-q-')), 'queue.jsonl')
+
+describe('telemetry-queue', () => {
+  it('enqueue then drain returns items and clears the file', () => {
+    const f = tmpQueue()
+    enqueueFailed('{"a":1}', f)
+    enqueueFailed('{"b":2}', f)
+    expect(drainQueue(f)).toEqual(['{"a":1}', '{"b":2}'])
+    expect(drainQueue(f)).toEqual([]) // cleared after drain
+  })
+
+  it('caps the queue at 50, keeping the most recent', () => {
+    const f = tmpQueue()
+    for (let i = 0; i < 60; i++) enqueueFailed(`{"i":${i}}`, f)
+    const items = drainQueue(f)
+    expect(items).toHaveLength(50)
+    expect(items[0]).toBe('{"i":10}') // oldest 10 dropped
+    expect(items[49]).toBe('{"i":59}')
+  })
+
+  it('draining a missing file is empty', () => {
+    expect(drainQueue(join(tmpdir(), 'chai-nope-xyz.jsonl'))).toEqual([])
+  })
+})