Mob.Audio output probes: verify sound is actually working

android/jni/mob_nif.zig

@@ -263,6 +263,11 @@ pub const BridgeMethods = extern struct {
     audio_play_at: jni.JMethodID = null,
     audio_stop_playback: jni.JMethodID = null,
     audio_set_volume: jni.JMethodID = null,
+    // Output probes — optional (cacheOptional); a drifted MobBridge.kt that
+    // predates them simply leaves these null and the NIFs return an error
+    // atom instead of crashing nif_load.
+    audio_output_status: jni.JMethodID = null,
+    audio_output_level: jni.JMethodID = null,
     motion_start: jni.JMethodID = null,
     motion_stop: jni.JMethodID = null,
     take_launch_notification: jni.JMethodID = null,
@@ -2127,6 +2132,90 @@ export fn nif_open_settings(
     return erts.ok(env);
 }
 
+// nif_audio_output_status/0 — {Volume, Muted, RouteCode, OtherAudio} as four
+// doubles (decoded by Mob.Audio.output_status/0). MobBridge.audioOutputStatus()
+// returns float[4] = [volume0..1, muted(0/1), routeCode, otherAudio(0/1)].
+// Optional bridge method: an older MobBridge.kt leaves it null → return all
+// zeros (Mob.Audio decodes that as route :none, which reads as "unknown-ish"
+// rather than crashing).
+export fn nif_audio_output_status(
+    env: ?*erts.ErlNifEnv,
+    argc: c_int,
+    argv: [*]const erts.ERL_NIF_TERM,
+) callconv(.c) erts.ERL_NIF_TERM {
+    _ = argc;
+    _ = argv;
+    var vals: [4]f32 = @splat(0);
+    if (Bridge.audio_output_status != null) {
+        var attached: c_int = 0;
+        const jenv = get_jenv(&attached) orelse return erts.atom(env, "error");
+        const arr = jenv.*.CallStaticObjectMethod.?(jenv, Bridge.cls, Bridge.audio_output_status);
+        if (arr != null) {
+            jni.getFloatArrayRegion(jenv, arr, 0, 4, &vals);
+            jni.deleteLocalRef(jenv, arr);
+        }
+        detachIfAttached(attached);
+    }
+    return erts.makeTuple(env, .{
+        erts.enif_make_double(env, @floatCast(vals[0])),
+        erts.enif_make_double(env, @floatCast(vals[1])),
+        erts.enif_make_double(env, @floatCast(vals[2])),
+        erts.enif_make_double(env, @floatCast(vals[3])),
+    });
+}
+
+// nif_audio_output_level/1 — {RmsDb, PeakDb} as two doubles, or an error atom.
+// Source is "mob" (Mob's own player session) | "mix". The global output mix is
+// privileged on modern Android (a normal app gets ERROR_NO_INIT attaching a
+// Visualizer to session 0), so "mix" is unsupported here — global device-audio
+// capture lives in a separate MediaProjection-based plugin. MobBridge returns:
+//   float[2] = [rms_db, peak_db]  → success
+//   float[1] = [code]             → 1 unsupported_on_platform, 2 needs_record_audio,
+//                                    3 not_playing (no active Mob.Audio player)
+//   null                          → generic error
+export fn nif_audio_output_level(
+    env: ?*erts.ErlNifEnv,
+    argc: c_int,
+    argv: [*]const erts.ERL_NIF_TERM,
+) callconv(.c) erts.ERL_NIF_TERM {
+    _ = argc;
+    if (Bridge.audio_output_level == null) return erts.atom(env, "unsupported_on_platform");
+    const bin = getBinOrIolist(env, argv[0]) orelse return erts.badarg(env);
+    const source = binToCString(bin) orelse return erts.atom(env, "error");
+    defer freeCString(source);
+    var attached: c_int = 0;
+    const jenv = get_jenv(&attached) orelse return erts.atom(env, "error");
+    const jsource = jni.newStringUTF(jenv, source);
+    const arr = jenv.*.CallStaticObjectMethod.?(jenv, Bridge.cls, Bridge.audio_output_level, jsource);
+    jni.deleteLocalRef(jenv, jsource);
+    if (arr == null) {
+        detachIfAttached(attached);
+        return erts.atom(env, "error");
+    }
+    const len = jni.getArrayLength(jenv, arr);
+    if (len >= 2) {
+        var vals: [2]f32 = @splat(0);
+        jni.getFloatArrayRegion(jenv, arr, 0, 2, &vals);
+        jni.deleteLocalRef(jenv, arr);
+        detachIfAttached(attached);
+        return erts.makeTuple(env, .{
+            erts.enif_make_double(env, @floatCast(vals[0])),
+            erts.enif_make_double(env, @floatCast(vals[1])),
+        });
+    }
+    // Length-1 array carries an error code the Kotlin couldn't express otherwise.
+    var code: [1]f32 = @splat(0);
+    if (len == 1) jni.getFloatArrayRegion(jenv, arr, 0, 1, &code);
+    jni.deleteLocalRef(jenv, arr);
+    detachIfAttached(attached);
+    return switch (@as(i32, @intFromFloat(code[0]))) {
+        1 => erts.atom(env, "unsupported_on_platform"),
+        2 => erts.atom(env, "needs_record_audio"),
+        3 => erts.atom(env, "not_playing"),
+        else => erts.atom(env, "error"),
+    };
+}
+
 // nif_share_text/1 — system share sheet (Intent ACTION_SEND text/plain).
 export fn nif_share_text(
     env: ?*erts.ErlNifEnv,
@@ -3699,6 +3788,10 @@ fn nifLoad(env: ?*erts.ErlNifEnv, priv: *?*anyopaque, info: erts.ERL_NIF_TERM) c
     if (!cacheRequired(jenv, "audio_play_at", "(JLjava/lang/String;Ljava/lang/String;Ljava/lang/String;)V", &Bridge.audio_play_at)) return -1;
     if (!cacheRequired(jenv, "audio_stop_playback", "()V", &Bridge.audio_stop_playback)) return -1;
     if (!cacheRequired(jenv, "audio_set_volume", "(Ljava/lang/String;)V", &Bridge.audio_set_volume)) return -1;
+    // Output probes are optional so a drifted MobBridge.kt that predates them
+    // no-ops (NIF returns an error atom) instead of failing nif_load.
+    cacheOptional(jenv, "audioOutputStatus", "()[F", &Bridge.audio_output_status);
+    cacheOptional(jenv, "audioOutputLevel", "(Ljava/lang/String;)[F", &Bridge.audio_output_level);
     if (!cacheRequired(jenv, "storage_dir", "(Ljava/lang/String;)Ljava/lang/String;", &Bridge.storage_dir)) return -1;
     if (!cacheRequired(jenv, "storage_save_to_media_store", "(JLjava/lang/String;Ljava/lang/String;)V", &Bridge.storage_save_to_media_store)) return -1;
     if (!cacheRequired(jenv, "storage_external_files_dir", "(Ljava/lang/String;)Ljava/lang/String;", &Bridge.storage_external_files_dir)) return -1;
@@ -3827,6 +3920,11 @@ const nif_funcs = [_]erts.ErlNifFunc{
     .{ .name = "audio_play_at", .arity = 3, .fptr = nif_audio_play_at, .flags = 0 },
     .{ .name = "audio_stop_playback", .arity = 0, .fptr = nif_audio_stop_playback, .flags = 0 },
     .{ .name = "audio_set_volume", .arity = 1, .fptr = nif_audio_set_volume, .flags = 0 },
+    .{ .name = "audio_output_status", .arity = 0, .fptr = nif_audio_output_status, .flags = 0 },
+    // Dirty IO: the Android side briefly settles a Visualizer measurement
+    // window, and iOS dispatch_syncs to the main queue — keep it off the
+    // regular schedulers.
+    .{ .name = "audio_output_level", .arity = 1, .fptr = nif_audio_output_level, .flags = erts.ERL_NIF_DIRTY_JOB_IO_BOUND },
     .{ .name = "motion_start", .arity = 2, .fptr = nif_motion_start, .flags = 0 },
     .{ .name = "motion_stop", .arity = 0, .fptr = nif_motion_stop, .flags = 0 },
     .{ .name = "take_launch_notification", .arity = 0, .fptr = nif_take_launch_notification, .flags = 0 },

decisions/2026-06-29-audio-output-probes.md

@@ -0,0 +1,103 @@
+# Audio output probes — verify sound is actually working
+
+- Date: 2026-06-29
+- Status: accepted
+
+## Context
+
+Mob can verify *visual* output in-process via the `screenshot/3` NIF (reads the
+composited framebuffer; see `Mob.Test`). There was no equivalent for *audio*:
+nothing could answer "is sound actually coming out right now." This surfaced
+bringing up Doom (the `mob_doom` plugin) in a mob app — Doom drives its own
+`AudioTrack` at 11025 Hz from a polling thread, and there was no programmatic
+way to tell working audio from silence.
+
+`adb shell dumpsys audio` / `dumpsys media.audio_flinger` already answer much of
+this from outside the app on Android (active players + state, stream volume +
+mute, mixer-track underrun counters). But they cannot distinguish a live signal
+from pushed silence, and they do not exist on iOS. These probes are the
+in-process, cross-platform layer on top of that.
+
+Audio has no single "final surface" the way the framebuffer is for video, so we
+expose two probes at two vantage points rather than one screenshot-equivalent.
+
+## Decision
+
+Add two read-only NIF-backed functions to `Mob.Audio`:
+
+- `output_status/0` → `%{volume, muted, route, other_audio}`. Cheap,
+  synchronous, no permission. Catches the common "no sound" causes (muted,
+  volume 0, dead route). iOS: `AVAudioSession`. Android: `AudioManager`.
+- `output_level/1` → `{rms_db, peak_db}` | `:silent` | `{:error, reason}`. Reads
+  actual signal energy — the part `output_status` and `adb` cannot answer. Takes
+  a `:source`:
+  - `:mob` (default) — meters `Mob.Audio`'s own player. iOS: `AVAudioPlayer`
+    metering (free, no permission). Android: `Visualizer` on the player's **own
+    audio session** (needs `RECORD_AUDIO`, runtime-granted). `{:error,
+    :not_playing}` when no `Mob.Audio` playback is active.
+  - `:mix` — *would* tap the global output mix to observe audio that bypasses
+    `Mob.Audio` (a game's own `AudioTrack`, another app). **Not available to a
+    normal app** on either platform → `{:error, :unsupported_on_platform}`.
+    Device-verified: a session-0 `Visualizer` on Android 11 fails with
+    `ERROR_NO_INIT` even with `RECORD_AUDIO` + `MODIFY_AUDIO_SETTINGS` (global
+    output capture is privileged); iOS forbids it by sandbox. Global
+    device-audio capture belongs in a separate MediaProjection-based plugin
+    intended as a **test-environment dependency**, not the core framework.
+
+Native wiring mirrors `screenshot/3` and `open_settings/1`: `-export` + `-nifs` +
+stub in `src/mob_nif.erl`; native table entries in `android/jni/mob_nif.zig` and
+`ios/mob_nif.m`; and **`cacheOptional` + null-guard** for the app-owned Android
+bridge methods so a drifted `MobBridge.kt` no-ops (NIF returns an error atom)
+instead of failing `nif_load` and crash-looping boot (the 0.7.6 lesson). The
+Android bridge methods ship in the `mob_new` template; iOS level-2 is
+self-contained in `mob_nif.m`. `output_level` is a dirty IO NIF (Android settles
+a Visualizer window; iOS `dispatch_sync`s to the main queue).
+
+The NIFs return only doubles / bare atoms (no term-building in C/Zig); `Mob.Audio`
+decodes route codes and the `:silent`/`:error` shapes in pure Elixir
+(`decode_status/1`, `decode_level/1`, unit-tested on host).
+
+## Consequences
+
+- **Honest scope:** the in-app probes verify *your own* audio only. Metering a
+  foreign native player (a bundled game's `AudioTrack`, another app) is not
+  possible for a normal app on either platform — the global-mix tap is privileged
+  on Android and forbidden on iOS. That capability is deferred to a separate
+  capture plugin (below). For the immediate "is the bundled game's audio working"
+  question, `adb shell dumpsys media.audio_flinger` (active track + underruns) is
+  the answer, no in-app probe needed.
+- Both probes observe only the device's own output, never "did a human hear it"
+  (that would need a mic loopback, deliberately out of scope).
+- Verification idiom is `play → sleep a beat → output_level`; metering is
+  instantaneous and only valid while audio plays. The Android `Visualizer`
+  occasionally returns its `-96 dB` floor if a measurement window hasn't filled,
+  so sample a few times.
+
+## Device verification (moto g power 2021, Android 11) — 2026-06-29
+
+Verified on hardware via `mix mob.connect` dist-RPC into a `doom_demo` build:
+
+- App **boots** with the new NIF table (stable pid) — the boot-critical check for
+  a native-table mismatch.
+- `output_status/0` → `%{route: :speaker, volume: 0.2, muted: false, other_audio:
+  false}`.
+- `output_level(:mob)` while a local tone looped → `{-34.8, -31.8}` (real signal);
+  idle / after stop → `{:error, :not_playing}`.
+- `output_level(:mix)` → `{:error, :unsupported_on_platform}`.
+- Without `RECORD_AUDIO` granted at runtime → `{:error, :needs_record_audio}`.
+- Disproved the original design: a session-0 `Visualizer` returns `ERROR_NO_INIT`
+  even with `RECORD_AUDIO` + `MODIFY_AUDIO_SETTINGS`. This is why `:mix` is
+  unsupported in core.
+
+## Follow-up: separate device-audio capture plugin (test-env dependency)
+
+True global/foreign-app output capture on Android is achievable only via
+`MediaProjection` + `AudioPlaybackCaptureConfiguration` (API 29+), which pops a
+one-time system consent dialog and can capture other apps' output. That UX is
+unacceptable in a shipped app but fine in a dev/test harness. Plan: a separate
+`mob_*` capture plugin, added as a test-environment dep, exposing a
+`capture_level/0`-style probe backed by `AudioPlaybackCapture`. This is where the
+"meter Doom's own audio" capability lives. iOS has no equivalent (no
+inter-app/system output capture), so that plugin is Android-only. Pairs with the
+in-process screenshot work for agent-driven testing, and gives `mob_midi`'s
+pending tone primitive something to assert against.

deps

@@ -0,0 +1 @@
+/Users/kevin/code/mob/deps

ios/mob_nif.m

@@ -2217,6 +2217,71 @@ static ERL_NIF_TERM nif_open_settings(ErlNifEnv *env, int argc, const ERL_NIF_TE
     return enif_make_atom(env, "ok");
 }
 
+// nif_audio_output_status/0 — {Volume, Muted, RouteCode, OtherAudio} as four
+// doubles (decoded by Mob.Audio.output_status/0). iOS has no direct mute flag,
+// so Muted is inferred from outputVolume == 0. RouteCode mirrors the Android
+// encoding: 1=speaker, 2=headphones, 3=bluetooth, 4=receiver, 0=none.
+static ERL_NIF_TERM nif_audio_output_status(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    AVAudioSession *session = [AVAudioSession sharedInstance];
+    double volume = (double)session.outputVolume;
+    double other = session.isOtherAudioPlaying ? 1.0 : 0.0;
+    double route = 0.0;
+    for (AVAudioSessionPortDescription *out in session.currentRoute.outputs) {
+        NSString *t = out.portType;
+        if ([t isEqualToString:AVAudioSessionPortBuiltInSpeaker])
+            route = 1.0;
+        else if ([t isEqualToString:AVAudioSessionPortHeadphones])
+            route = 2.0;
+        else if ([t isEqualToString:AVAudioSessionPortBluetoothA2DP] ||
+                 [t isEqualToString:AVAudioSessionPortBluetoothLE] ||
+                 [t isEqualToString:AVAudioSessionPortBluetoothHFP])
+            route = 3.0;
+        else if ([t isEqualToString:AVAudioSessionPortBuiltInReceiver])
+            route = 4.0;
+        if (route != 0.0)
+            break;
+    }
+    double muted = (volume <= 0.0) ? 1.0 : 0.0;
+    return enif_make_tuple4(env, enif_make_double(env, volume), enif_make_double(env, muted),
+                            enif_make_double(env, route), enif_make_double(env, other));
+}
+
+// nif_audio_output_level/1 — {RmsDb, PeakDb} as two doubles, or an error atom.
+// iOS cannot tap the global output mix (sandbox), so "mix" is unsupported;
+// "mob" meters Mob.Audio's own AVAudioPlayer (metering is enabled when the
+// player is created).
+//
+// Forward-declare the play/1 player: it lives with the audio-playback globals
+// defined further below, but output_level/1 (here) meters that same player.
+static AVAudioPlayer *g_audio_player;
+static ERL_NIF_TERM nif_audio_output_level(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
+    ErlNifBinary bin;
+    if (!enif_inspect_binary(env, argv[0], &bin) &&
+        !enif_inspect_iolist_as_binary(env, argv[0], &bin))
+        return enif_make_badarg(env);
+    NSString *source = [[NSString alloc] initWithBytes:bin.data
+                                                length:bin.size
+                                              encoding:NSUTF8StringEncoding];
+    if (![source isEqualToString:@"mob"])
+        return enif_make_atom(env, "unsupported_on_platform");
+
+    __block double rms = -160.0;
+    __block double peak = -160.0;
+    __block BOOL playing = NO;
+    dispatch_sync(dispatch_get_main_queue(), ^{
+      AVAudioPlayer *player = g_audio_player;
+      if (player && player.playing) {
+          playing = YES;
+          [player updateMeters];
+          rms = (double)[player averagePowerForChannel:0];
+          peak = (double)[player peakPowerForChannel:0];
+      }
+    });
+    if (!playing)
+        return enif_make_atom(env, "not_playing");
+    return enif_make_tuple2(env, enif_make_double(env, rms), enif_make_double(env, peak));
+}
+
 // ── NIF: share_text/1 ─────────────────────────────────────────────────────────
 // Opens the iOS share sheet with plain text. Fire-and-forget.
 
@@ -2998,6 +3063,9 @@ static ERL_NIF_TERM nif_audio_play(ErlNifEnv *env, int argc, const ERL_NIF_TERM
       player.delegate = g_player_delegate;
       player.volume = (float)volume;
       player.numberOfLoops = loop ? -1 : 0;
+      // Enable metering so Mob.Audio.output_level(source: :mob) can read the
+      // signal level without a separate tap. Cheap; off by default otherwise.
+      player.meteringEnabled = YES;
       g_audio_player = player;
       [player play];
     });
@@ -6248,6 +6316,8 @@ static ERL_NIF_TERM nif_vendor_usb_close(ErlNifEnv *env, int argc, const ERL_NIF
     {"audio_play_at", 3, nif_audio_play_at, 0},
     {"audio_stop_playback", 0, nif_audio_stop_playback, 0},
     {"audio_set_volume", 1, nif_audio_set_volume, 0},
+    {"audio_output_status", 0, nif_audio_output_status, 0},
+    {"audio_output_level", 1, nif_audio_output_level, ERL_NIF_DIRTY_JOB_IO_BOUND},
     {"tts_speak", 2, nif_tts_speak, 0},
     {"tts_stop", 0, nif_tts_stop, 0},
     {"motion_start", 2, nif_motion_start, 0},