Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 20 additions & 20 deletions packages/desktop/src/main/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ import { promisify } from 'node:util'
import type { MenuItemConstructorOptions } from 'electron'
import { NATIVE_MENU_TEMPLATE } from '../../../mumo/src/nativeMenu.js'
import type { NativeMenuItem } from '../../../mumo/src/nativeMenu.js'
import { promises as fsPromises, constants as fsConstants, createWriteStream } from 'node:fs'
import { promises as fsPromises, constants as fsConstants, createWriteStream, createReadStream } from 'node:fs'
import { Readable } from 'node:stream'
import { join, extname, basename, resolve } from 'node:path'

// Crash logging
Expand Down Expand Up @@ -168,6 +169,14 @@ async function handleMediaRequest(request: Request): Promise<Response> {
const mime = mimeFor(filePath)
const rangeHeader = request.headers.get('range')

// Stream the file rather than buffering it: media players send open-ended range
// requests (bytes=N-) and read lazily, so buffering materializes entire multi-GB
// files in main-process memory — several concurrent readers per file OOM-kills the app.
const streamBody = (start: number, end: number): BodyInit | null => {
if (end < start) return null
return Readable.toWeb(createReadStream(filePath, { start, end })) as ReadableStream
}

if (rangeHeader) {
const range = parseRange(rangeHeader, fileSize)
if (!range) {
Expand All @@ -177,27 +186,18 @@ async function handleMediaRequest(request: Request): Promise<Response> {
})
}
const [start, end] = range
const chunkSize = end - start + 1
const fh = await fsPromises.open(filePath, 'r')
try {
const buf = Buffer.allocUnsafe(chunkSize)
await fh.read(buf, 0, chunkSize, start)
return new Response(buf, {
status: 206,
headers: {
'Content-Type': mime,
'Content-Length': String(chunkSize),
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
},
})
} finally {
await fh.close()
}
return new Response(streamBody(start, end), {
status: 206,
headers: {
'Content-Type': mime,
'Content-Length': String(end - start + 1),
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
},
})
}

const buf = await fsPromises.readFile(filePath)
return new Response(buf, {
return new Response(streamBody(0, fileSize - 1), {
status: 200,
headers: {
'Content-Type': mime,
Expand Down
37 changes: 28 additions & 9 deletions packages/editor/src/TranscriptEditor.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -392,18 +392,37 @@
return new Slice(sanitizeFragment(slice.content), slice.openStart, slice.openEnd)
},
handlePaste(view, _event, slice) {
// When the cursor is inside an utterance and the clipboard holds complete
// utterance blocks (openStart=0), pasting would replace the destination
// utterance node and wipe its participant/timing attrs. Re-open the slice
// at depth 1 so PM merges the pasted content *into* the destination
// utterance instead of inserting a new block.
const from = view.state.selection.$from
if (from.depth < 1 || from.node(1)?.type.name !== 'utterance') return false
if (slice.openStart !== 0) return false
// When the pasted content is utterance node(s), never let PM replace the
// destination utterance node itself — that would wipe participant/timing/id.
// Instead, flatten the pasted utterances to inline content and splice it
// directly into the destination utterance's content range.
let allUtt = slice.content.childCount > 0
slice.content.forEach(n => { if (n.type.name !== 'utterance') allUtt = false })
if (!allUtt) return false
view.dispatch(view.state.tr.replaceSelection(new Slice(slice.content, 1, slice.openEnd)))

const selFrom = view.state.selection.$from
let destPos: number | null = null
if (selFrom.depth >= 1 && selFrom.node(1)?.type.name === 'utterance') {
destPos = selFrom.before(1)
} else if (selFrom.depth === 0 && selFrom.nodeAfter?.type.name === 'utterance') {
destPos = selFrom.pos
}
if (destPos === null) return false

const destUtt = view.state.doc.nodeAt(destPos)
if (!destUtt) return false

// Flatten inline content from all pasted utterances
const pastedNodes: Node[] = []
slice.content.forEach(uttNode => uttNode.content.forEach((n: Node) => pastedNodes.push(n)))

// Replace only within the utterance's content range, clamped to the
// current selection so a partial selection replaces only the selected text.
const innerFrom = destPos + 1
const innerTo = destPos + destUtt.nodeSize - 1
const spliceFrom = Math.max(view.state.selection.from, innerFrom)
const spliceTo = Math.min(view.state.selection.to, innerTo)
view.dispatch(view.state.tr.replaceWith(spliceFrom, spliceTo, pastedNodes))
return true
},
nodeViews: {
Expand Down
12 changes: 8 additions & 4 deletions packages/media-player/src/LinkedMediaDlg.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,10 @@
{:else}
<div class="lmd-grid">
{#each entries as entry (entry.id)}
<span class="lmd-name" class:lmd-name-unloaded={entry.kind === 'unloaded'} title={entry.id}>{entry.name}</span>
<span class="lmd-status">{entry.kind === 'unloaded' ? 'not loaded' : ''}</span>
<span class="lmd-name" class:lmd-name-unloaded={entry.kind === 'unloaded'} class:lmd-name-missing={entry.kind === 'missing'} title={entry.id}>{entry.name}</span>
<span class="lmd-status" class:lmd-status-missing={entry.kind === 'missing'}>
{#if entry.kind === 'missing'}⚠ not found{:else if entry.kind === 'unloaded'}not loaded{/if}
</span>
<label class="lmd-offset-label">
Offset (s)
<input class="lmd-offset" type="number" step="0.01"
Expand All @@ -44,8 +46,8 @@
/>
</label>
<div class="lmd-actions">
{#if entry.kind === 'unloaded'}
<button class="lmd-load" onclick={() => onLoad(entry.id)}>Load</button>
{#if entry.kind === 'unloaded' || entry.kind === 'missing'}
<button class="lmd-load" onclick={() => onLoad(entry.id)}>Locate</button>
{/if}
<button class="lmd-remove" onclick={() => onRemove(entry.id)}>Remove</button>
</div>
Expand Down Expand Up @@ -95,11 +97,13 @@
color: #333; font-size: 12px; min-width: 0;
}
.lmd-name-unloaded { color: #888; font-style: italic; }
.lmd-name-missing { color: #b71c1c; font-style: italic; }
.lmd-status {
font-size: 10px; color: #999; white-space: nowrap;
border: 1px solid transparent; border-radius: 8px; padding: 1px 5px;
}
.lmd-status:not(:empty) { border-color: #ddd; }
.lmd-status-missing { color: #b71c1c; border-color: #ef9a9a; background: #fff8f8; }
.lmd-offset-label {
display: flex; align-items: center; gap: 4px;
white-space: nowrap; color: #555; font-size: 12px;
Expand Down
10 changes: 10 additions & 0 deletions packages/media-player/src/MediaPlayer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ export class MediaPlayer {
const mediaUrl = this._platform.mediaUrlForFile(file, path)
const kind: 'audio' | 'video' = file.type.startsWith('video') ? 'video' : 'audio'

console.log(`[media] loading ${kind}: ${file.name}`, path ? `(${path})` : '')
this.track = { file, path, mediaUrl, offsetSec: this.track?.offsetSec ?? 0 }
this._setState({ mediaUrl, kind, filename: file.name, duration: 0, sampleRate: 0, channelCount: 1, activeChannel: 'mix', muted: this.state?.muted ?? false, volume: this.state?.volume ?? 1 })

Expand All @@ -151,10 +152,12 @@ export class MediaPlayer {
this._videoRenderer = new VideoRenderer(document.createElement('canvas'), this.track.offsetSec)
}
await this._videoRenderer.loadAudioOnly(mediaUrl)
.then(() => { console.log(`[media] loaded audio: ${file.name}`) })
.catch((err: unknown) => this._callbacks.onError?.(`Audio renderer: ${String(err)}`))
} else if (this._videoRenderer) {
void this._videoRenderer.load(mediaUrl)
.then(() => {
console.log(`[media] loaded video: ${file.name}`)
if (this.state && this._videoRenderer) {
this._setState({ ...this.state, duration: this._videoRenderer.duration })
}
Expand All @@ -168,6 +171,7 @@ export class MediaPlayer {
const name = url.split('/').pop()?.split('?')[0] ?? 'media'
const kind: 'audio' | 'video' = /\.(mp4|webm|mov|m4v|ogv|mkv)$/i.test(name) ? 'video' : 'audio'

console.log(`[media] loading ${kind} url: ${name}`)
this.track = null
this._setState({ mediaUrl: url, kind, filename: name, duration: 0, sampleRate: 0, channelCount: 1, activeChannel: 'mix', muted: this.state?.muted ?? false, volume: this.state?.volume ?? 1 })

Expand All @@ -179,10 +183,12 @@ export class MediaPlayer {
this._videoRenderer = new VideoRenderer(document.createElement('canvas'), 0)
}
await this._videoRenderer.loadAudioOnly(url)
.then(() => { console.log(`[media] loaded audio url: ${name}`) })
.catch((err: unknown) => this._callbacks.onError?.(`Audio renderer: ${String(err)}`))
} else if (this._videoRenderer) {
void this._videoRenderer.load(url)
.then(() => {
console.log(`[media] loaded video url: ${name}`)
if (this.state && this._videoRenderer) {
this._setState({ ...this.state, duration: this._videoRenderer.duration })
}
Expand All @@ -197,13 +203,15 @@ export class MediaPlayer {
* and stream each chunk to the worker without accumulating PCM in main memory.
*/
private async _streamAudio(url: string, decodeId: number): Promise<void> {
const name = this.state?.filename ?? url.split('/').pop() ?? 'media'
const source = new UrlSource(url)
const input = new Input({ formats: ALL_FORMATS, source })
try {
const at = await input.getPrimaryAudioTrack()
if (!at || decodeId !== this._decodeId) return
const sampleRate = await at.getSampleRate()
if (decodeId !== this._decodeId) return
console.log(`[media] audio processing start: ${name} (${sampleRate} Hz)`)
const sink = new AudioBufferSink(at)
let duration = 0, channelCount = 0
for await (const { buffer, timestamp } of sink.buffers(0)) {
Expand All @@ -218,10 +226,12 @@ export class MediaPlayer {
this._broker.feedChunk(chunks, sampleRate, channelCount)
}
if (decodeId === this._decodeId) {
console.log(`[media] audio processing done: ${name} (${duration.toFixed(2)}s, ${channelCount}ch)`)
this._broker.endStream(duration)
}
} catch (err) {
if (decodeId === this._decodeId) {
console.error(`[media] audio processing error: ${name}`, err)
this._callbacks.onError?.(`Failed to decode audio: ${String(err)}`)
}
} finally {
Expand Down
1 change: 1 addition & 0 deletions packages/media-player/src/linked-media-types.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
export type MediaEntry =
| { kind: 'loaded'; id: string; name: string; offsetSec: number }
| { kind: 'unloaded'; id: string; name: string; offsetSec: number }
| { kind: 'missing'; id: string; name: string; offsetSec: number }
Loading
Loading