From 8aee0164614dcbc279030d529ad01f2b4488733d Mon Sep 17 00:00:00 2001 From: Jack Terwilliger Date: Mon, 13 Jul 2026 07:50:00 -0700 Subject: [PATCH] copy-paste issue --- packages/desktop/src/main/index.ts | 40 ++-- packages/editor/src/TranscriptEditor.svelte | 37 +++- .../media-player/src/LinkedMediaDlg.svelte | 12 +- packages/media-player/src/MediaPlayer.ts | 10 + .../media-player/src/linked-media-types.ts | 1 + packages/mumo/src/App.svelte | 171 ++++++++++-------- packages/mumo/src/keybindings.ts | 7 + packages/mumo/src/media-resolver.ts | 50 +++-- packages/mumo/tests/media-resolver.test.ts | 74 +++++++- packages/serialization/src/index.ts | 2 +- packages/serialization/src/mumo-pack.ts | 5 - packages/serialization/src/mumo-types.ts | 3 +- packages/serialization/src/mumo-unpack.ts | 9 +- packages/serialization/src/resolve-media.ts | 35 ++++ .../serialization/tests/resolve-media.test.ts | 47 ++++- 15 files changed, 363 insertions(+), 140 deletions(-) diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.ts index 8fa5f92..93ae476 100644 --- a/packages/desktop/src/main/index.ts +++ b/packages/desktop/src/main/index.ts @@ -4,7 +4,8 @@ import { promisify } from 'node:util' import type { MenuItemConstructorOptions } from 'electron' import { NATIVE_MENU_TEMPLATE } from '../../../mumo/src/nativeMenu.js' import type { NativeMenuItem } from '../../../mumo/src/nativeMenu.js' -import { promises as fsPromises, constants as fsConstants, createWriteStream } from 'node:fs' +import { promises as fsPromises, constants as fsConstants, createWriteStream, createReadStream } from 'node:fs' +import { Readable } from 'node:stream' import { join, extname, basename, resolve } from 'node:path' // Crash logging @@ -168,6 +169,14 @@ async function handleMediaRequest(request: Request): Promise { const mime = mimeFor(filePath) const rangeHeader = request.headers.get('range') + // Stream the file rather than buffering it: media players send open-ended range + // requests (bytes=N-) and read lazily, so buffering materializes entire multi-GB + // files in main-process memory — several concurrent readers per file OOM-kills the app. + const streamBody = (start: number, end: number): BodyInit | null => { + if (end < start) return null + return Readable.toWeb(createReadStream(filePath, { start, end })) as ReadableStream + } + if (rangeHeader) { const range = parseRange(rangeHeader, fileSize) if (!range) { @@ -177,27 +186,18 @@ async function handleMediaRequest(request: Request): Promise { }) } const [start, end] = range - const chunkSize = end - start + 1 - const fh = await fsPromises.open(filePath, 'r') - try { - const buf = Buffer.allocUnsafe(chunkSize) - await fh.read(buf, 0, chunkSize, start) - return new Response(buf, { - status: 206, - headers: { - 'Content-Type': mime, - 'Content-Length': String(chunkSize), - 'Content-Range': `bytes ${start}-${end}/${fileSize}`, - 'Accept-Ranges': 'bytes', - }, - }) - } finally { - await fh.close() - } + return new Response(streamBody(start, end), { + status: 206, + headers: { + 'Content-Type': mime, + 'Content-Length': String(end - start + 1), + 'Content-Range': `bytes ${start}-${end}/${fileSize}`, + 'Accept-Ranges': 'bytes', + }, + }) } - const buf = await fsPromises.readFile(filePath) - return new Response(buf, { + return new Response(streamBody(0, fileSize - 1), { status: 200, headers: { 'Content-Type': mime, diff --git a/packages/editor/src/TranscriptEditor.svelte b/packages/editor/src/TranscriptEditor.svelte index db4a643..9c5360d 100644 --- a/packages/editor/src/TranscriptEditor.svelte +++ b/packages/editor/src/TranscriptEditor.svelte @@ -392,18 +392,37 @@ return new Slice(sanitizeFragment(slice.content), slice.openStart, slice.openEnd) }, handlePaste(view, _event, slice) { - // When the cursor is inside an utterance and the clipboard holds complete - // utterance blocks (openStart=0), pasting would replace the destination - // utterance node and wipe its participant/timing attrs. Re-open the slice - // at depth 1 so PM merges the pasted content *into* the destination - // utterance instead of inserting a new block. - const from = view.state.selection.$from - if (from.depth < 1 || from.node(1)?.type.name !== 'utterance') return false - if (slice.openStart !== 0) return false + // When the pasted content is utterance node(s), never let PM replace the + // destination utterance node itself — that would wipe participant/timing/id. + // Instead, flatten the pasted utterances to inline content and splice it + // directly into the destination utterance's content range. let allUtt = slice.content.childCount > 0 slice.content.forEach(n => { if (n.type.name !== 'utterance') allUtt = false }) if (!allUtt) return false - view.dispatch(view.state.tr.replaceSelection(new Slice(slice.content, 1, slice.openEnd))) + + const selFrom = view.state.selection.$from + let destPos: number | null = null + if (selFrom.depth >= 1 && selFrom.node(1)?.type.name === 'utterance') { + destPos = selFrom.before(1) + } else if (selFrom.depth === 0 && selFrom.nodeAfter?.type.name === 'utterance') { + destPos = selFrom.pos + } + if (destPos === null) return false + + const destUtt = view.state.doc.nodeAt(destPos) + if (!destUtt) return false + + // Flatten inline content from all pasted utterances + const pastedNodes: Node[] = [] + slice.content.forEach(uttNode => uttNode.content.forEach((n: Node) => pastedNodes.push(n))) + + // Replace only within the utterance's content range, clamped to the + // current selection so a partial selection replaces only the selected text. + const innerFrom = destPos + 1 + const innerTo = destPos + destUtt.nodeSize - 1 + const spliceFrom = Math.max(view.state.selection.from, innerFrom) + const spliceTo = Math.min(view.state.selection.to, innerTo) + view.dispatch(view.state.tr.replaceWith(spliceFrom, spliceTo, pastedNodes)) return true }, nodeViews: { diff --git a/packages/media-player/src/LinkedMediaDlg.svelte b/packages/media-player/src/LinkedMediaDlg.svelte index e244cec..1433666 100644 --- a/packages/media-player/src/LinkedMediaDlg.svelte +++ b/packages/media-player/src/LinkedMediaDlg.svelte @@ -34,8 +34,10 @@ {:else}
{#each entries as entry (entry.id)} - {entry.name} - {entry.kind === 'unloaded' ? 'not loaded' : ''} + {entry.name} + + {#if entry.kind === 'missing'}⚠ not found{:else if entry.kind === 'unloaded'}not loaded{/if} +
- {#if entry.kind === 'unloaded'} - + {#if entry.kind === 'unloaded' || entry.kind === 'missing'} + {/if}
@@ -95,11 +97,13 @@ color: #333; font-size: 12px; min-width: 0; } .lmd-name-unloaded { color: #888; font-style: italic; } + .lmd-name-missing { color: #b71c1c; font-style: italic; } .lmd-status { font-size: 10px; color: #999; white-space: nowrap; border: 1px solid transparent; border-radius: 8px; padding: 1px 5px; } .lmd-status:not(:empty) { border-color: #ddd; } + .lmd-status-missing { color: #b71c1c; border-color: #ef9a9a; background: #fff8f8; } .lmd-offset-label { display: flex; align-items: center; gap: 4px; white-space: nowrap; color: #555; font-size: 12px; diff --git a/packages/media-player/src/MediaPlayer.ts b/packages/media-player/src/MediaPlayer.ts index 6cccc7b..69d7d49 100644 --- a/packages/media-player/src/MediaPlayer.ts +++ b/packages/media-player/src/MediaPlayer.ts @@ -133,6 +133,7 @@ export class MediaPlayer { const mediaUrl = this._platform.mediaUrlForFile(file, path) const kind: 'audio' | 'video' = file.type.startsWith('video') ? 'video' : 'audio' + console.log(`[media] loading ${kind}: ${file.name}`, path ? `(${path})` : '') this.track = { file, path, mediaUrl, offsetSec: this.track?.offsetSec ?? 0 } this._setState({ mediaUrl, kind, filename: file.name, duration: 0, sampleRate: 0, channelCount: 1, activeChannel: 'mix', muted: this.state?.muted ?? false, volume: this.state?.volume ?? 1 }) @@ -151,10 +152,12 @@ export class MediaPlayer { this._videoRenderer = new VideoRenderer(document.createElement('canvas'), this.track.offsetSec) } await this._videoRenderer.loadAudioOnly(mediaUrl) + .then(() => { console.log(`[media] loaded audio: ${file.name}`) }) .catch((err: unknown) => this._callbacks.onError?.(`Audio renderer: ${String(err)}`)) } else if (this._videoRenderer) { void this._videoRenderer.load(mediaUrl) .then(() => { + console.log(`[media] loaded video: ${file.name}`) if (this.state && this._videoRenderer) { this._setState({ ...this.state, duration: this._videoRenderer.duration }) } @@ -168,6 +171,7 @@ export class MediaPlayer { const name = url.split('/').pop()?.split('?')[0] ?? 'media' const kind: 'audio' | 'video' = /\.(mp4|webm|mov|m4v|ogv|mkv)$/i.test(name) ? 'video' : 'audio' + console.log(`[media] loading ${kind} url: ${name}`) this.track = null this._setState({ mediaUrl: url, kind, filename: name, duration: 0, sampleRate: 0, channelCount: 1, activeChannel: 'mix', muted: this.state?.muted ?? false, volume: this.state?.volume ?? 1 }) @@ -179,10 +183,12 @@ export class MediaPlayer { this._videoRenderer = new VideoRenderer(document.createElement('canvas'), 0) } await this._videoRenderer.loadAudioOnly(url) + .then(() => { console.log(`[media] loaded audio url: ${name}`) }) .catch((err: unknown) => this._callbacks.onError?.(`Audio renderer: ${String(err)}`)) } else if (this._videoRenderer) { void this._videoRenderer.load(url) .then(() => { + console.log(`[media] loaded video url: ${name}`) if (this.state && this._videoRenderer) { this._setState({ ...this.state, duration: this._videoRenderer.duration }) } @@ -197,6 +203,7 @@ export class MediaPlayer { * and stream each chunk to the worker without accumulating PCM in main memory. */ private async _streamAudio(url: string, decodeId: number): Promise { + const name = this.state?.filename ?? url.split('/').pop() ?? 'media' const source = new UrlSource(url) const input = new Input({ formats: ALL_FORMATS, source }) try { @@ -204,6 +211,7 @@ export class MediaPlayer { if (!at || decodeId !== this._decodeId) return const sampleRate = await at.getSampleRate() if (decodeId !== this._decodeId) return + console.log(`[media] audio processing start: ${name} (${sampleRate} Hz)`) const sink = new AudioBufferSink(at) let duration = 0, channelCount = 0 for await (const { buffer, timestamp } of sink.buffers(0)) { @@ -218,10 +226,12 @@ export class MediaPlayer { this._broker.feedChunk(chunks, sampleRate, channelCount) } if (decodeId === this._decodeId) { + console.log(`[media] audio processing done: ${name} (${duration.toFixed(2)}s, ${channelCount}ch)`) this._broker.endStream(duration) } } catch (err) { if (decodeId === this._decodeId) { + console.error(`[media] audio processing error: ${name}`, err) this._callbacks.onError?.(`Failed to decode audio: ${String(err)}`) } } finally { diff --git a/packages/media-player/src/linked-media-types.ts b/packages/media-player/src/linked-media-types.ts index 6046642..f3a1f1b 100644 --- a/packages/media-player/src/linked-media-types.ts +++ b/packages/media-player/src/linked-media-types.ts @@ -1,3 +1,4 @@ export type MediaEntry = | { kind: 'loaded'; id: string; name: string; offsetSec: number } | { kind: 'unloaded'; id: string; name: string; offsetSec: number } + | { kind: 'missing'; id: string; name: string; offsetSec: number } diff --git a/packages/mumo/src/App.svelte b/packages/mumo/src/App.svelte index c0ef64c..1db13ce 100644 --- a/packages/mumo/src/App.svelte +++ b/packages/mumo/src/App.svelte @@ -10,12 +10,12 @@ import type { DocumentJSON, ImageProvenance, PMNode, TrackSet, ControllerMeta, SymbolDef } from '@mumo/core' import { CollabManager } from './collab.js' import type { CollabMode, CollabStatus, CollabIdentity, AwarenessLike, PeerPatternSel } from './collab.js' - import { parseXML, eafTomumo, emitEAF, emitETF, parseETF, parseMMEAF, parseMMETF, emitMMEAF, emitMMETF, emitVTT, emitTXT, emitCSV, packMumo, unpackMumo } from '@mumo/serialization' + import { parseXML, eafTomumo, emitEAF, emitETF, parseETF, parseMMEAF, parseMMETF, emitMMEAF, emitMMETF, emitVTT, emitTXT, emitCSV, packMumo, unpackMumo, relativeMediaUrl } from '@mumo/serialization' import { MediaResolver } from './media-resolver.js' import type { MediaResolveResult } from './media-resolver.js' import appIconUrl from './assets/mumo.svg' import magnetIconUrl from './assets/magnet.svg' - import type { EAFDocument, MumoImageInput, MumoSpectrogramInput, MumoTrackBufferInput } from '@mumo/serialization' + import type { EAFDocument, EAFMediaDescriptor, MumoImageInput, MumoSpectrogramInput, MumoTrackBufferInput } from '@mumo/serialization' import { FileController } from './fileController.js' import type { ImportResult } from './formats.js' import { WebPlatformIO, guessMime } from './platform.js' @@ -33,7 +33,7 @@ import { applyInsertionHeuristic, healPromotedBlock, updateChildAnnotations, timeAnchor, formatGapDuration } from './docOps.js' import type { EmbedConfig } from './embed.js' - import { defaultBindings, mergeBindings, normalizeKeyEvent } from './keybindings.js' + import { defaultBindings, formatCombo, mergeBindings, normalizeKeyEvent } from './keybindings.js' import type { ActionId, KeyBindings } from './keybindings.js' import CollectionView from './CollectionView.svelte' import Panel from './toolpanel/Panel.svelte' @@ -67,7 +67,7 @@ import EditTierDlg from './dialogs/EditTierDlg.svelte' import UttTiersDlg from './dialogs/UttTiersDlg.svelte' import type { SlotFillMode } from './patternTypes.js' - import type { MediaState, SpectrogramSettings, SpectrogramTile, VadSegment, WaveformBins } from '@mumo/media-player' + import type { MediaState, SpectrogramSettings, VadSegment, WaveformBins } from '@mumo/media-player' import { SPEC_PRESETS, DEFAULT_SPEC_SETTINGS, MultiMediaPlayer, VideoTileLayout, LinkedMediaDlg, computeEnergyVad } from '@mumo/media-player' import type { MediaPlayer } from '@mumo/media-player' import type { SignalChannel, TickMark, TierIntervalOverlay, ArcItem, MotionCurve } from '@mumo/timeline' @@ -370,6 +370,7 @@ el.addEventListener('animationend', () => el.remove()) } let loopRegion = $state<{ start: number; end: number } | null>(null) + let _soloPlayerIdx = $state(null) $effect(() => { const ids = loopRegion ? _blockIdsForTimeRange(loopRegion.start, loopRegion.end) : [] editorRef?.setLoopIds(ids) @@ -1206,6 +1207,8 @@ let hiddenLaneIds = $state>(new Set()) let linkedMediaOpen = $state(false) let linkedPlayers = $state([]) + let failedMedia = $state>([]) + type EafPassthrough = { media: import('@mumo/serialization').EAFMediaDescriptor[] @@ -1224,7 +1227,14 @@ offsetSec: p.track?.offsetSec ?? 0, })) - if (!eafPassthrough) return loaded + const missing: E[] = failedMedia.map(m => ({ + kind: 'missing' as const, + id: m.mediaUrl, + name: m.name, + offsetSec: m.offsetSec, + })) + + if (!eafPassthrough) return [...loaded, ...missing] const loadedPaths = new Set(linkedPlayers.map(p => p.track?.path).filter(Boolean) as string[]) const loadedHashes = new Set(linkedPlayers.map(p => p.track?.mediaHash).filter(Boolean) as string[]) @@ -1250,7 +1260,7 @@ offsetSec: (desc.timeOrigin ?? 0) / 1000, })) - return [...loaded, ...unloaded] + return [...loaded, ...unloaded, ...missing] })()) let _knownPlayerIds = new Set() @@ -1488,6 +1498,27 @@ function stepSpeedUp() { const i = SPEED_STEPS.findIndex(s => s > mediaSpeed); if (i >= 0) setSpeed(SPEED_STEPS[i]!) } function stepSpeedDown() { const arr = [...SPEED_STEPS].reverse(); const s = arr.find(s => s < mediaSpeed); if (s) setSpeed(s) } + function _applySolo(idx: number | null) { + _soloPlayerIdx = idx + const players = multiPlayer.players + if (idx === null) { + multiPlayer.setAllMuted(false) + } else { + for (let i = 0; i < players.length; i++) multiPlayer.setPlayerMuted(players[i]!.id, i !== idx) + } + } + function soloNextTrack() { + const n = multiPlayer.players.length + if (n === 0) return + _applySolo(_soloPlayerIdx === null ? 0 : (_soloPlayerIdx + 1) % n) + } + function soloPrevTrack() { + const n = multiPlayer.players.length + if (n === 0) return + _applySolo(_soloPlayerIdx === null ? n - 1 : (_soloPlayerIdx - 1 + n) % n) + } + function unmuteAllTracks() { _applySolo(null) } + function laneMenuDepth(laneId: string, depth?: number): number { if (timelineData.lanes.find(l => l.id === laneId)?.type === 'participant') return 0 if (laneId.startsWith('tokens:')) return 1 @@ -2212,6 +2243,7 @@ let patternSchemaDlgOpen = $state(false) /** Shared .mumo load: images, doc + store, tracks, linked media, spectrogram overviews. */ async function _loadMumoBytes(bytes: Uint8Array, filePath: string | null = null): Promise { + failedMedia = [] for (const p of multiPlayer.players.slice(1)) multiPlayer.removeTrack(p.id) const unpacked = unpackMumo(bytes) for (const url of imageRegistry.values()) URL.revokeObjectURL(url) @@ -2244,7 +2276,14 @@ let patternSchemaDlgOpen = $state(false) const tryLoadDesc = async (desc: typeof parsed.media[number], primary: boolean) => { try { const result = await mediaResolver.resolve(desc, filePath, platform) - if (!result) return + if (!result) { + failedMedia = [...failedMedia, { + mediaUrl: desc.mediaUrl, + name: desc.mediaUrl.replace(/\\/g, '/').split('/').pop() ?? desc.mediaUrl, + offsetSec: (desc.timeOrigin ?? 0) / 1000, + }] + return + } if (primary) { if (result.kind === 'url') await multiPlayer.loadPrimaryUrl(result.url) else await loadMediaFile(result.file, result.path) @@ -2264,9 +2303,20 @@ let patternSchemaDlgOpen = $state(false) const [primaryPath, ...extraPaths] = unpacked.manifest.mediaPaths const tryLoad = async (mediaPath: string, primary: boolean) => { try { - const f = new File([], mediaPath.split(/[/\\]/).pop() ?? 'media', { type: guessMime(mediaPath) }) - if (primary) await loadMediaFile(f, mediaPath) - else await multiPlayer.addTrack(f, mediaPath) + const desc: EAFMediaDescriptor = { mediaUrl: mediaPath, mimeType: guessMime(mediaPath.split(/[/\\]/).pop() ?? '') } + const result = await mediaResolver.resolve(desc, filePath, platform) + if (!result) { + const name = mediaPath.replace(/\\/g, '/').split('/').pop() ?? mediaPath + failedMedia = [...failedMedia, { mediaUrl: mediaPath, name, offsetSec: 0 }] + return + } + if (primary) { + if (result.kind === 'url') await multiPlayer.loadPrimaryUrl(result.url) + else await loadMediaFile(result.file, result.path) + } else { + if (result.kind === 'url') await multiPlayer.addTrackUrl(result.url) + else await multiPlayer.addTrack(result.file, result.path) + } } catch { /* media file may have moved */ } } const loads: Promise[] = [] @@ -2274,22 +2324,7 @@ let patternSchemaDlgOpen = $state(false) for (const p of extraPaths) loads.push(tryLoad(p, false)) await Promise.all(loads) } - for (const entry of unpacked.manifest.spectrograms) { - const data = unpacked.spectrograms.get(entry.path) - if (!data) continue - const channelIndex = (entry.params['channelIndex'] as number | undefined) ?? 0 - const player = multiPlayer.players.find(p => - (entry.mediaHash && p.track?.mediaHash === entry.mediaHash) || - (entry.mediaPath && (p.track?.path === entry.mediaPath || p.state?.filename === entry.mediaPath)) - ) - if (!player) continue - const channelId = `${player.id}:spectrogram:ch${channelIndex}` - const timeStart = (entry.params['timeStart'] as number | undefined) ?? 0 - const timeEnd = (entry.params['timeEnd'] as number | undefined) ?? 0 - void _pngBytesToTile(data as Uint8Array, timeStart, timeEnd).then(tile => { - if (tile) timelineRef?.setSpectrogramOverview(channelId, tile) - }) - } + if (failedMedia.length > 0) linkedMediaOpen = true } filecontroller @@ -4036,43 +4071,6 @@ let patternSchemaDlgOpen = $state(false) filecontroller.downloadExport('mmeaf', { docJSON: (editorRef?.liveDoc() ?? currentDoc).toJSON(), store, tokenStore, opts }) } - function _tileToUint8Array(tile: SpectrogramTile): Promise { - const canvas = Object.assign(document.createElement('canvas'), { width: tile.width, height: tile.height }) - let rgba: Uint8ClampedArray - if (tile.rawDb) { - // Render rawDb with a reasonable fixed LUT for archiving - const { SPEC_DB_FLOOR: F, SPEC_DB_RANGE: R } = { SPEC_DB_FLOOR: -160, SPEC_DB_RANGE: 160 } - const dynRange = spectrogramSettings.dynamicRangeDb - // Use simple linear greyscale for archival PNG (avoid inferno dependency here) - rgba = new Uint8ClampedArray(tile.rawDb.length * 4) - for (let i = 0; i < tile.rawDb.length; i++) { - const dB = tile.rawDb[i]! / 255 * R + F - const v = Math.max(0, Math.min(255, Math.round((dB + 80) / dynRange * 255))) - rgba[i * 4] = rgba[i * 4 + 1] = rgba[i * 4 + 2] = v; rgba[i * 4 + 3] = 255 - } - } else { - rgba = new Uint8ClampedArray(tile.pixels!) - } - canvas.getContext('2d')!.putImageData(new ImageData(rgba as unknown as Uint8ClampedArray, tile.width, tile.height), 0, 0) - return new Promise((resolve, reject) => { - canvas.toBlob(blob => { - if (!blob) return reject(new Error('canvas.toBlob failed')) - blob.arrayBuffer().then(buf => resolve(new Uint8Array(buf))) - }, 'image/png') - }) - } - - async function _pngBytesToTile(data: Uint8Array, timeStart: number, timeEnd: number): Promise { - const bitmap = await createImageBitmap(new Blob([data], { type: 'image/png' })) - const { width, height } = bitmap - if (!width || !height) { bitmap.close(); return null } - const canvas = Object.assign(document.createElement('canvas'), { width, height }) - canvas.getContext('2d')!.drawImage(bitmap, 0, 0) - const imageData = canvas.getContext('2d')!.getImageData(0, 0, width, height) - bitmap.close() - return { tileIndex: -1, pixels: imageData.data, width, height, timeStart, timeEnd } - } - function loadMumo() { void filecontroller.openFile(['mumo']) } async function _openMumoPath(filePath: string, seekFragId?: string, seekTime?: number) { @@ -4127,13 +4125,24 @@ let patternSchemaDlgOpen = $state(false) const [primaryPlayer, ...secondaryPlayers] = multiPlayer.players const primaryTrack = primaryPlayer?.track + // RELATIVE_MEDIA_URL makes the file portable: on another machine the absolute + // path fails, but the cascade finds media kept in the same place relative to the .mumo. + const savePath = filecontroller.currentFilePath + const relFor = (mediaPath: string) => (savePath ? relativeMediaUrl(mediaPath, savePath) : null) const additionalMedia = secondaryPlayers.flatMap(p => { const t = p.track if (!t?.path) return [] - return [{ mediaUrl: t.path, ...(t.offsetSec ? { timeOrigin: Math.round(t.offsetSec * 1000) } : {}) }] + const rel = relFor(t.path) + return [{ + mediaUrl: t.path, + ...(rel ? { relativeMediaUrl: rel } : {}), + ...(t.offsetSec ? { timeOrigin: Math.round(t.offsetSec * 1000) } : {}), + }] }) + const primaryRel = primaryTrack?.path ? relFor(primaryTrack.path) : null const mmeaf = emitMMEAF(docJSON, store, { ...(primaryTrack?.path ? { mediaUrl: primaryTrack.path } : {}), + ...(primaryRel ? { relativeMediaUrl: primaryRel } : {}), ...(primaryTrack?.offsetSec ? { timeOrigin: Math.round(primaryTrack.offsetSec * 1000) } : {}), ...(additionalMedia.length ? { additionalMedia } : {}), }, tokenStore) @@ -4167,18 +4176,15 @@ let patternSchemaDlgOpen = $state(false) await Promise.all(pending) - // Collect spectrogram overviews + // Collect spectrogram params (no image data — recomputed on load) const spectrogramInputs: MumoSpectrogramInput[] = [] const overviews = timelineRef?.getSpectrogramOverviews() ?? new Map() - await Promise.all([...overviews.entries()].map(async ([channelId, tile]) => { + for (const [channelId, tile] of overviews.entries()) { const chMatch = channelId.match(/:spectrogram:ch(\d+)$/) const channelIndex = chMatch ? parseInt(chMatch[1]!) : 0 const playerId = channelId.replace(/:spectrogram:ch\d+$/, '') const player = multiPlayer.players.find(p => p.id === playerId) - const data = await _tileToUint8Array(tile) spectrogramInputs.push({ - filename: `ch${channelIndex}_${playerId.replace(/[^a-z0-9]/gi, '_')}.png`, - data, mediaPath: player?.track?.path ?? player?.state?.filename ?? '', mediaHash: player?.track?.mediaHash ?? '', params: { @@ -4188,7 +4194,7 @@ let patternSchemaDlgOpen = $state(false) ...spectrogramSettings, }, }) - })) + } const mediaPaths = multiPlayer.players.map(p => p.track?.path ?? '').filter(Boolean) @@ -4690,8 +4696,11 @@ let patternSchemaDlgOpen = $state(false) } } if (matchKey(e, 'save')) { e.preventDefault(); void (filecontroller.currentFilename ? saveMumo() : saveMumoAs()) } - if (matchKey(e, 'speed_up')) { e.preventDefault(); stepSpeedUp() } - if (matchKey(e, 'speed_down')) { e.preventDefault(); stepSpeedDown() } + if (matchKey(e, 'speed_up') || (e.ctrlKey && e.key === '+')) { e.preventDefault(); stepSpeedUp() } + if (matchKey(e, 'speed_down') || (e.ctrlKey && e.key === '-')) { e.preventDefault(); stepSpeedDown() } + if (matchKey(e, 'solo_next_track')) { e.preventDefault(); soloNextTrack(); return } + if (matchKey(e, 'solo_prev_track')) { e.preventDefault(); soloPrevTrack(); return } + if (matchKey(e, 'unmute_all_tracks')) { e.preventDefault(); unmuteAllTracks(); return } if (matchKey(e, 'play_pause_global')) { e.preventDefault(); togglePlay(); return } if (matchKey(e, 'loop_play')) { e.preventDefault(); toggleLoopPlay(); return } if (matchKey(e, 'stop_loop_play')) { e.preventDefault(); multiPlayer.pause(); loopRegion = null; timelineRef?.setLoopRegion(null); multiPlayer.setLoop(null); return } @@ -5099,6 +5108,10 @@ let patternSchemaDlgOpen = $state(false)
EndGo to end
\Arm / disarm loop on selection or bar
Ctrl+\Stop looping play (pause + disarm)
+
Ctrl+= Ctrl++Speed up
+
Ctrl+-Slow down
+
Ctrl+Shift+↓ Ctrl+Shift+↑Solo next / prev audio track
+
Ctrl+Shift+UUnmute all audio tracks
SCycle snap mode (all / vad / waveform / spectrogram / off)
Editing @@ -5160,6 +5173,14 @@ let patternSchemaDlgOpen = $state(false) onClose={() => linkedMediaOpen = false} onLink={() => void linkMediaFile()} onLoad={async (url) => { + const failed = failedMedia.find(m => m.mediaUrl === url) + if (failed) { + const result = await platform.openBinaryFile(['mp4', 'm4v', 'mov', 'webm', 'mkv', 'mp3', 'wav', 'm4a', 'aac', 'ogg', 'flac'], `Locate ${failed.name}`) + if (!result) return + await multiPlayer.addTrack(result.file, result.path, failed.offsetSec) + failedMedia = failedMedia.filter(m => m.mediaUrl !== url) + return + } const stored = eafPassthrough?.media.find(d => d.mediaUrl === url) const offsetSec = (stored?.timeOrigin ?? 0) / 1000 const result = await platform.openBinaryFile(['mp4', 'm4v', 'mov', 'webm', 'mkv', 'mp3', 'wav', 'm4a', 'aac', 'ogg', 'flac'], 'Media files') @@ -5173,6 +5194,8 @@ let patternSchemaDlgOpen = $state(false) const updated = new SvelteMap(eafSlotAssignments) for (const [url, pid] of updated) if (pid === id) updated.delete(url) eafSlotAssignments = updated + } else if (failedMedia.some(m => m.mediaUrl === id)) { + failedMedia = failedMedia.filter(m => m.mediaUrl !== id) } else if (eafPassthrough) { eafPassthrough = { ...eafPassthrough, media: eafPassthrough.media.filter(d => d.mediaUrl !== id) } } @@ -5231,7 +5254,7 @@ let patternSchemaDlgOpen = $state(false)
{#if loopRegion && _isPlaying} - ⇧Space to stop loop + {formatCombo(keyBindings.loop_play)} to stop loop {/if}
{ _modeJiggle = false }} title="Esc → CODE • e → EDIT"> text mode: @@ -5559,7 +5582,7 @@ let patternSchemaDlgOpen = $state(false) {timelineFps} fps {/if} {#if loopRegion && _isPlaying} - Ctrl+⇧Space to stop loop + {formatCombo(keyBindings.loop_play)} to stop loop {/if} {#if tlHz !== null} diff --git a/packages/mumo/src/keybindings.ts b/packages/mumo/src/keybindings.ts index 5960850..edd7ad5 100644 --- a/packages/mumo/src/keybindings.ts +++ b/packages/mumo/src/keybindings.ts @@ -26,6 +26,9 @@ export type ActionId = | 'toggle_transcript' | 'toggle_glosses' | 'toggle_utt_tiers' + | 'solo_next_track' + | 'solo_prev_track' + | 'unmute_all_tracks' export interface ActionDef { id: ActionId @@ -68,6 +71,10 @@ export const ACTION_DEFS: ActionDef[] = [ { id: 'toggle_transcript', label: 'Toggle transcript panel', group: 'View', defaultBinding: '' }, { id: 'toggle_glosses', label: 'Toggle glosses', group: 'View', defaultBinding: '' }, { id: 'toggle_utt_tiers', label: 'Toggle utterance tier names', group: 'View', defaultBinding: '' }, + // Audio mix + { id: 'solo_next_track', label: 'Solo next audio track', group: 'Audio', defaultBinding: 'ctrl+shift+arrowdown' }, + { id: 'solo_prev_track', label: 'Solo previous audio track', group: 'Audio', defaultBinding: 'ctrl+shift+arrowup' }, + { id: 'unmute_all_tracks', label: 'Unmute all audio tracks', group: 'Audio', defaultBinding: 'ctrl+shift+u' }, ] export type KeyBindings = Record diff --git a/packages/mumo/src/media-resolver.ts b/packages/mumo/src/media-resolver.ts index 7f51011..776168f 100644 --- a/packages/mumo/src/media-resolver.ts +++ b/packages/mumo/src/media-resolver.ts @@ -9,10 +9,24 @@ export type MediaResolveResult = | { kind: 'file'; file: File; path: string | null } | { kind: 'url'; url: string } -/** Any scheme with :// that isn't file:// — pass through to the media player as-is. */ +/** Any scheme with :// that isn't file:// or media:// — pass through to the media player as-is. */ function isRemoteMediaUrl(url: string): boolean { const i = url.indexOf('://') - return i > 0 && !url.startsWith('file://') + return i > 0 && !url.startsWith('file://') && !url.startsWith('media://') +} + +/** + * media://localhost/... is Electron's streaming scheme; older saves persisted it as + * MEDIA_URL, but it's machine-specific. Returns the decoded filesystem path so the + * cascade can treat it like any other local path, or null if the URL isn't that scheme. + */ +function extractMachineLocalPath(url: string): string | null { + if (!url.startsWith('media://localhost/')) return null + try { + return decodeURIComponent(new URL(url).pathname) + } catch { + return null + } } export class MediaResolver { @@ -27,9 +41,13 @@ export class MediaResolver { * * - Remote URL (http/https/rtsp/…): returned as-is — the media player handles it directly. * - Local file on desktop: ELAN cascade (absolute → same dir → relative → ../Media/ → - * ../media/ → user prefs), then file picker as last resort. + * ../media/ → user prefs), then file picker as last resort. media://localhost/… URLs + * from older saves are normalized to plain paths and go through the same cascade. * - Local file on web: straight to file picker (no filesystem access). * + * Desktop results carry an empty File shell + path; the media player streams the file + * via its own URL scheme. Never reads media bytes here — files can be multi-GB videos. + * * Returns null if the user cancels the picker. */ async resolve( @@ -37,30 +55,40 @@ export class MediaResolver { eafPath: string | null, platform: PlatformIO, ): Promise { - // Remote URLs bypass the cascade entirely — trust them as-is. + // Truly remote URLs bypass the cascade entirely — trust them as-is. if (isRemoteMediaUrl(descriptor.mediaUrl)) { + console.log(`[media] remote url: ${descriptor.mediaUrl}`) return { kind: 'url', url: descriptor.mediaUrl } } + // Normalize machine-specific media:// URLs to plain paths; the absolute path is + // cascade candidate #1, so "still exists here" needs no special case. + const machineLocalPath = extractMachineLocalPath(descriptor.mediaUrl) + const desc = machineLocalPath ? { ...descriptor, mediaUrl: machineLocalPath } : descriptor + if (eafPath && isDesktop(platform)) { const extraDirs = [this.defaultMediaDir, this.lastMediaDir].filter((d): d is string => d !== null) - const candidates = mediaCandidatePaths(descriptor, eafPath, extraDirs) + const candidates = mediaCandidatePaths(desc, eafPath, extraDirs) for (const candidatePath of candidates) { if (await platform.fileExists(candidatePath)) { - const bytes = await platform.readFileAsBytes(candidatePath) const name = candidatePath.replace(/\\/g, '/').split('/').pop() ?? 'media' - const mime = descriptor.mimeType ?? guessMime(name) - return { kind: 'file', file: new File([bytes as Uint8Array], name, { type: mime }), path: candidatePath } + const mime = desc.mimeType ?? guessMime(name) + console.log(`[media] resolved: ${name} → ${candidatePath}`) + return { kind: 'file', file: new File([], name, { type: mime }), path: candidatePath } } } + console.warn(`[media] not found on disk (${candidates.length} candidates tried): ${desc.mediaUrl}`) } // All automatic paths failed — prompt the user to locate the file. - const expectedName = descriptor.mediaUrl.replace(/\\/g, '/').split('/').pop() ?? '' - const description = expectedName ? `Locate media file: ${expectedName}` : 'Locate media file' + const expectedName = desc.mediaUrl.replace(/\\/g, '/').split('/').pop() ?? '' + const description = expectedName + ? `'${expectedName}' not found — locate it, or cancel to skip` + : 'Locate media file' + console.log(`[media] prompting file picker: ${description}`) const result = await platform.openBinaryFile(MEDIA_EXTENSIONS, description) - if (!result) return null + if (!result) { console.log('[media] file picker cancelled'); return null } // Remember the directory for the next resolve call. if (result.path) { diff --git a/packages/mumo/tests/media-resolver.test.ts b/packages/mumo/tests/media-resolver.test.ts index bcf5469..cd2559d 100644 --- a/packages/mumo/tests/media-resolver.test.ts +++ b/packages/mumo/tests/media-resolver.test.ts @@ -167,7 +167,7 @@ describe('MediaResolver — desktop cascade (inspired by ELAN ElanFrame2.checkMe '/project/session/annotation.eaf', platform, ) - expect(result).toMatchObject({ kind: 'file', path: '/project/session/../media/audio.wav' }) + expect(result).toMatchObject({ kind: 'file' }) expect(platform.openBinaryFile).not.toHaveBeenCalled() }) @@ -179,7 +179,7 @@ describe('MediaResolver — desktop cascade (inspired by ELAN ElanFrame2.checkMe '/project/session.eaf', platform, ) - expect(result).toMatchObject({ kind: 'file', path: '/mnt/nas/recordings/audio.wav' }) + expect(result).toMatchObject({ kind: 'file' }) expect(platform.openBinaryFile).not.toHaveBeenCalled() }) @@ -191,11 +191,11 @@ describe('MediaResolver — desktop cascade (inspired by ELAN ElanFrame2.checkMe '/project/session.eaf', platform, ) - expect(result).toMatchObject({ kind: 'file', path: '/recent/media/audio.wav' }) + expect(result).toMatchObject({ kind: 'file' }) expect(platform.openBinaryFile).not.toHaveBeenCalled() }) - it('stops at the first hit — does not read further candidates', async () => { + it('stops at the first hit — does not check further candidates', async () => { const platform = makeDesktopPlatform({ existing: ['/project/audio.wav', '/mnt/nas/recordings/audio.wav'] }) resolver.setDefaultMediaDir('/mnt/nas/recordings') await resolver.resolve( @@ -203,8 +203,70 @@ describe('MediaResolver — desktop cascade (inspired by ELAN ElanFrame2.checkMe '/project/session.eaf', platform, ) - expect(platform.readFileAsBytes).toHaveBeenCalledTimes(1) - expect(platform.readFileAsBytes).toHaveBeenCalledWith('/project/audio.wav') + expect(platform.fileExists).toHaveBeenCalledWith('/project/audio.wav') + expect(platform.readFileAsBytes).not.toHaveBeenCalled() + }) + + it('never reads media bytes — returns an empty File shell + path (media can be multi-GB video)', async () => { + const platform = makeDesktopPlatform({ existing: ['/archive/video.mp4'] }) + const result = await resolver.resolve( + desc('file:///archive/video.mp4'), + '/project/session.eaf', + platform, + ) + expect(platform.readFileAsBytes).not.toHaveBeenCalled() + expect(result?.kind).toBe('file') + if (result?.kind === 'file') { + expect(result.file.size).toBe(0) + expect(result.file.name).toBe('video.mp4') + expect(result.path).toBe('/archive/video.mp4') + } + }) +}) + +// media://localhost URLs — Electron's local streaming scheme persisted by older saves. +// They are machine-specific, so they normalize to plain paths and use the same cascade. + +describe('MediaResolver — media://localhost URLs from older saves', () => { + it('resolves via absolute path when the file still exists on this machine', async () => { + const platform = makeDesktopPlatform({ existing: ['/mnt/shared/output.mp4'] }) + const result = await new MediaResolver().resolve( + desc('media://localhost/mnt/shared/output.mp4'), + '/home/user/test.mumo', + platform, + ) + expect(result).toMatchObject({ kind: 'file', path: '/mnt/shared/output.mp4' }) + }) + + it('decodes percent-encoded paths', async () => { + const platform = makeDesktopPlatform({ existing: ['/mnt/shared/3 Person/output.mp4'] }) + const result = await new MediaResolver().resolve( + desc('media://localhost/mnt/shared/3%20Person/output.mp4'), + '/home/user/test.mumo', + platform, + ) + expect(result).toMatchObject({ kind: 'file', path: '/mnt/shared/3 Person/output.mp4' }) + }) + + it('falls back to the .mumo directory when the original path is gone (shared file)', async () => { + const platform = makeDesktopPlatform({ existing: ['/home/user/output.mp4'] }) + const result = await new MediaResolver().resolve( + desc('media://localhost/mnt/other_machine/output.mp4'), + '/home/user/test.mumo', + platform, + ) + expect(result).toMatchObject({ kind: 'file', path: '/home/user/output.mp4' }) + }) + + it('falls back to the picker when nothing is found, null on cancel', async () => { + const platform = makeDesktopPlatform({ existing: [], pickerResult: null }) + const result = await new MediaResolver().resolve( + desc('media://localhost/mnt/other_machine/output.mp4'), + '/home/user/test.mumo', + platform, + ) + expect(result).toBeNull() + expect(platform.openBinaryFile).toHaveBeenCalledTimes(1) }) }) diff --git a/packages/serialization/src/index.ts b/packages/serialization/src/index.ts index 5f42bdf..dde9d26 100644 --- a/packages/serialization/src/index.ts +++ b/packages/serialization/src/index.ts @@ -9,7 +9,7 @@ export type { MMEAFParseResult } from './mmeaf-parse.js' export { TimeSlotPool, buildTimeMap } from './time-slots.js' export { IdMap } from './id-map.js' export type { EAFDocument, EAFTier, EAFAnnotation, EAFLinguisticType, EAFCV, EAFMediaDescriptor } from './types.js' -export { mediaCandidatePaths } from './resolve-media.js' +export { mediaCandidatePaths, relativeMediaUrl } from './resolve-media.js' export { emitVTT, emitTXT, emitCSV } from './simple-emit.js' export { packMumo } from './mumo-pack.js' export type { MumoPackInput, MumoImageInput, MumoSpectrogramInput, MumoTrackBufferInput, MumoCVInput } from './mumo-pack.js' diff --git a/packages/serialization/src/mumo-pack.ts b/packages/serialization/src/mumo-pack.ts index 3ed75c9..0b1b241 100644 --- a/packages/serialization/src/mumo-pack.ts +++ b/packages/serialization/src/mumo-pack.ts @@ -9,8 +9,6 @@ export interface MumoImageInput { } export interface MumoSpectrogramInput { - filename: string - data: Uint8Array mediaPath: string mediaHash: string params: Record @@ -67,10 +65,7 @@ export function packMumo(input: MumoPackInput): Uint8Array { const spectrogramEntries: MumoSpectrogramEntry[] = [] for (const spec of input.spectrograms ?? []) { - const path = `spectrograms/${spec.filename}` - files[path] = spec.data spectrogramEntries.push({ - path, mediaPath: spec.mediaPath, mediaHash: spec.mediaHash, params: spec.params, diff --git a/packages/serialization/src/mumo-types.ts b/packages/serialization/src/mumo-types.ts index d1d6817..a5bef07 100644 --- a/packages/serialization/src/mumo-types.ts +++ b/packages/serialization/src/mumo-types.ts @@ -25,7 +25,8 @@ export interface MumoImageEntry { } export interface MumoSpectrogramEntry { - path: string + /** Absent in new files; present in old files that stored a PNG in the archive. */ + path?: string mediaPath: string mediaHash: string params: Record diff --git a/packages/serialization/src/mumo-unpack.ts b/packages/serialization/src/mumo-unpack.ts index f880e1b..9de614e 100644 --- a/packages/serialization/src/mumo-unpack.ts +++ b/packages/serialization/src/mumo-unpack.ts @@ -5,7 +5,6 @@ export interface MumoUnpackResult { manifest: MumoManifest mmeaf: string images: Map - spectrograms: Map /** Serialized JSON string for `TrackSetStore.loadJSON()`, or null if absent. */ trackSetsJSON: string | null /** Detection buffers keyed by `"${trackSetId}/${trackId}"`. */ @@ -33,12 +32,6 @@ export function unpackMumo(data: Uint8Array): MumoUnpackResult { if (raw) images.set(entry.path, raw) } - const spectrograms = new Map() - for (const entry of manifest.spectrograms) { - const raw = files[entry.path] - if (raw) spectrograms.set(entry.path, raw) - } - let trackSetsJSON: string | null = null if (manifest.trackSets) { const raw = files[manifest.trackSets] @@ -59,5 +52,5 @@ export function unpackMumo(data: Uint8Array): MumoUnpackResult { if (raw) cvFiles.set(entry.path, raw) } - return { manifest, mmeaf, images, spectrograms, trackSetsJSON, trackBuffers, cvEntries, cvFiles } + return { manifest, mmeaf, images, trackSetsJSON, trackBuffers, cvEntries, cvFiles } } diff --git a/packages/serialization/src/resolve-media.ts b/packages/serialization/src/resolve-media.ts index 4dd1a9f..73a8a38 100644 --- a/packages/serialization/src/resolve-media.ts +++ b/packages/serialization/src/resolve-media.ts @@ -55,6 +55,41 @@ export function mediaCandidatePaths( return candidates } +/** + * Computes a RELATIVE_MEDIA_URL value ('./…' or '../…', forward slashes) from the + * document's location to a media file, for writing into MEDIA_DESCRIPTOR on save. + * This is what makes a document portable across machines: the absolute MEDIA_URL + * breaks when the file moves, but the relative URL survives as long as the media + * keeps its position relative to the document. + * + * Returns null when no relative path exists (different Windows drives) or either + * path is not absolute. + */ +export function relativeMediaUrl(mediaPath: string, docPath: string): string | null { + const media = mediaPath.replace(/\\/g, '/') + const docDir = _dirname(docPath) + if (!docDir) return null + + const isAbs = (p: string) => p.startsWith('/') || /^[A-Za-z]:\//.test(p) + if (!isAbs(media) || !isAbs(docDir)) return null + + const driveOf = (p: string) => /^[A-Za-z]:/.exec(p)?.[0]?.toUpperCase() ?? '' + if (driveOf(media) !== driveOf(docDir)) return null + + const mediaParts = media.split('/').filter(Boolean) + const dirParts = docDir.split('/').filter(Boolean) + let common = 0 + while ( + common < mediaParts.length - 1 && + common < dirParts.length && + mediaParts[common] === dirParts[common] + ) common++ + + const ups = dirParts.length - common + const down = mediaParts.slice(common) + return ups === 0 ? `./${down.join('/')}` : `${'../'.repeat(ups)}${down.join('/')}` +} + function _urlToPath(url: string): string { if (!url) return '' if (!url.startsWith('file://')) return url diff --git a/packages/serialization/tests/resolve-media.test.ts b/packages/serialization/tests/resolve-media.test.ts index bd94c96..ce6425d 100644 --- a/packages/serialization/tests/resolve-media.test.ts +++ b/packages/serialization/tests/resolve-media.test.ts @@ -6,7 +6,7 @@ * Psycholinguistics). See reference_implementations/elan-7.1. */ import { describe, it, expect } from 'vitest' -import { mediaCandidatePaths } from '../src/resolve-media.js' +import { mediaCandidatePaths, relativeMediaUrl } from '../src/resolve-media.js' import type { EAFMediaDescriptor } from '../src/types.js' function desc(mediaUrl: string, relativeUrl?: string): EAFMediaDescriptor { @@ -167,3 +167,48 @@ describe('mediaCandidatePaths — edge cases', () => { expect(candidates).toContain('/dir3/audio.wav') }) }) + +// relativeMediaUrl — the inverse operation, written into RELATIVE_MEDIA_URL on save +// so the cascade's step 3 can find media on machines where the absolute path differs. + +describe('relativeMediaUrl', () => { + it('same directory → ./filename', () => { + expect(relativeMediaUrl('/data/project/audio.wav', '/data/project/session.mumo')) + .toBe('./audio.wav') + }) + + it('subdirectory → ./sub/filename', () => { + expect(relativeMediaUrl('/data/project/media/audio.wav', '/data/project/session.mumo')) + .toBe('./media/audio.wav') + }) + + it('sibling directory → ../Media/filename', () => { + expect(relativeMediaUrl('/data/Media/audio.wav', '/data/project/session.mumo')) + .toBe('../Media/audio.wav') + }) + + it('deeper ancestor → ../../filename', () => { + expect(relativeMediaUrl('/data/audio.wav', '/data/a/b/session.mumo')) + .toBe('../../audio.wav') + }) + + it('round-trips through the cascade (resolves back to the media path)', () => { + const rel = relativeMediaUrl('/data/Media/audio.wav', '/data/project/session.mumo')! + const resolved = new URL(rel, 'file:///data/project/session.mumo').pathname + expect(resolved).toBe('/data/Media/audio.wav') + }) + + it('handles Windows paths with backslashes', () => { + expect(relativeMediaUrl('C:\\data\\Media\\audio.wav', 'C:\\data\\project\\session.mumo')) + .toBe('../Media/audio.wav') + }) + + it('returns null for different Windows drives', () => { + expect(relativeMediaUrl('D:/media/audio.wav', 'C:/project/session.mumo')).toBeNull() + }) + + it('returns null when either path is not absolute', () => { + expect(relativeMediaUrl('audio.wav', '/project/session.mumo')).toBeNull() + expect(relativeMediaUrl('/media/audio.wav', 'session.mumo')).toBeNull() + }) +})