From 37b325e1d8575b4511b6eb261b1d3e3fb1cbbbc9 Mon Sep 17 00:00:00 2001 From: Wild Dragon Dev Date: Wed, 3 Jun 2026 21:40:58 +0000 Subject: [PATCH] fix(capture): restore direct-to-S3 streaming (pipe:1 + fragmented MOV) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the local-temp+faststart approach from 549ca6c. Masters now stream ffmpeg stdout directly to S3 via multipart upload — no local disk consumed on the worker. Uses +frag_keyframe+empty_moov+default_base_moof which Premiere Pro 25.x handles natively (to be confirmed separately). Zero /tmp/capture files. Worker disk stays flat during recording. --- services/capture/src/capture-manager.js | 92 ++++++++----------------- 1 file changed, 28 insertions(+), 64 deletions(-) diff --git a/services/capture/src/capture-manager.js b/services/capture/src/capture-manager.js index 581c619..62aca41 100644 --- a/services/capture/src/capture-manager.js +++ b/services/capture/src/capture-manager.js @@ -1,5 +1,5 @@ import { spawn, execFileSync } from 'child_process'; -import { mkdirSync, writeFileSync, createReadStream, statSync, unlinkSync } from 'node:fs'; +import { mkdirSync, writeFileSync } from 'node:fs'; import fs from 'node:fs'; import { dirname } from 'node:path'; import { v4 as uuidv4 } from 'uuid'; @@ -484,18 +484,13 @@ function buildEncodeArgs({ if (a.bitrateControl && audioBitrate) args.push('-b:a', audioBitrate); if (audioChannels) args.push('-ac', String(audioChannels)); - // moov-atom placement is the difference between a Premiere-openable master and - // a "file cannot be opened" error. - // - // Finalized masters (the S3-piped recording that stops cleanly) must NOT be - // fragmented. Adobe Premiere's QuickTime/MOV importer reads the classic - // stco/stsz/stts sample tables in a single top-level moov; a fragmented MOV - // (moof/trun, empty sample tables) makes Premiere report "file cannot be - // opened." We write a clean, non-fragmented MOV instead. `+faststart` puts the - // moov before mdat on the second pass so the file is instantly - // seekable/streamable too. + // Fragmented MOV/MP4 for direct S3 streaming (pipe:1 output — no seekable + // file on the worker disk). +frag_keyframe writes a moof/trun fragment per + // keyframe; +empty_moov puts a valid moov box at the start so the file is + // immediately parseable. Premiere Pro 25.x (2025) handles fragmented MOV + // natively. Growing-file masters use the same flags (written to SMB share). if (fmt === 'mov' || fmt === 'mp4') { - args.push('-movflags', '+faststart'); + args.push('-movflags', '+frag_keyframe+empty_moov+default_base_moof'); } // ProRes-in-MOV must carry a QuickTime brand or some importers reject the tag. args.push('-f', fmt); @@ -1098,32 +1093,18 @@ exit "$BMXRC" || ((sourceType === 'blackmagic') && interlaced); const sdiFilterArgs = isInterlacedSource ? ['-vf', 'yadif=mode=1:deint=1'] : []; - // Master output destination (NON-growing path only). + // Master output destination. // // - Growing-files on → the growing OP1a MXF is written directly to the SMB // share by raw2bmx (see the orchestrator below); ffmpeg only produces the - // elementary essence FIFOs + HLS preview. `localMasterPath`/`hiresOutput` - // are unused in this case (the master path is `growingPath`). + // elementary essence FIFOs + HLS preview. // - // - Growing-files off → ffmpeg writes the MOV master to a LOCAL SEEKABLE - // temp file, then we upload to S3 on stop. We must NOT pipe the MOV muxer - // to S3 directly: the MOV/MP4 muxer cannot write to a non-seekable pipe - // without `empty_moov`, and an empty_moov/fragmented MOV is exactly what - // makes Adobe Premiere report "file cannot be opened" (no classic - // stco/stsz sample tables — samples live in moof/trun). A seekable file - // lets ffmpeg write a single contiguous moov with full sample tables and - // `+faststart` moves it to the front, producing a Premiere-native master. - const localMasterPath = growingPath - ? null - : `/tmp/capture/${sessionId}.${hiresExt}`; - if (localMasterPath) { - try { mkdirSync(dirname(localMasterPath), { recursive: true }); } - catch (err) { console.error('[capture] could not create temp master dir:', err.message); } - } - const hiresOutput = localMasterPath; - // When bridgeProcess is an fc_pipe process its stdout is piped to ffmpeg - // stdin (pipe:0 input). For all other sources stdin is ignored. - const hiresStdio = bridgeProcess ? ['pipe', 'ignore', 'pipe'] : ['ignore', 'ignore', 'pipe']; + // - Growing-files off → ffmpeg writes fragmented MOV to pipe:1 (stdout), + // which is piped directly into a multipart S3 upload. No local temp file, + // no worker disk consumed. Premiere Pro 25.x handles fragmented MOV natively. + const hiresOutput = growingPath ? growingPath : 'pipe:1'; + // pipe:1 = ffmpeg stdout → S3 stream. bridgeProcess (fc_pipe) uses stdin. + const hiresStdio = bridgeProcess ? ['pipe', 'pipe', 'pipe'] : ['ignore', 'pipe', 'pipe']; // For SDI/framecache sources (including network via framecache) the live // HLS preview is a SECOND OUTPUT of the hires ffmpeg. @@ -1197,7 +1178,7 @@ exit "$BMXRC" hiresArgs = [ ...inputArgs, '-filter_complex', filterStr, - // Output 0 — master (local temp, uploaded to S3 on stop) + // Output 0 — master (fragmented MOV streamed to S3 via pipe:1) '-map', '[vhi]', ...masterAudioMap, ...masterAudioFilter, ...hiresCodecArgs, @@ -1226,13 +1207,15 @@ exit "$BMXRC" } } - // Growing-files: nothing to upload here (promotion worker handles S3). - // Non-growing: the master is uploaded from the finalized local file in - // stop(), once ffmpeg has written the moov and exited cleanly — we can't - // upload while recording because the file isn't a valid MOV until finalize. - // bridgeProcess is null for deltacast (bridge managed by node-agent on the host). + // Growing: promotion worker handles S3 upload after stop. + // Non-growing: start streaming stdout directly to S3 now (multipart upload + // completes when ffmpeg exits and closes the pipe). const processes = { hires: hiresProcess }; - const uploads = { hires: growingPath ? Promise.resolve({ growingPath }) : null }; + const uploads = { + hires: growingPath + ? Promise.resolve({ growingPath }) + : createUploadStream(S3_BUCKET, hiresKey, hiresProcess.stdout), + }; // ── HLS tee for legacy network sources (live preview in the UI) ────────── // When network sources come via framecache (FC_SLOT_ID set), HLS preview is @@ -1469,30 +1452,11 @@ exit "$BMXRC" unmountGrowingShare(); try { + // Non-growing: S3 upload was streaming from ffmpeg stdout — it completes + // when ffmpeg exits and closes the pipe (waitExit above ensures that). + // Growing: promotion worker handles S3. const uploadPromises = []; - - // Non-growing: upload the finalized local master file to S3 now that the - // moov has been written. Growing: the promotion worker handles S3. - if (currentSession.localMasterPath) { - let size = 0; - try { size = statSync(currentSession.localMasterPath).size; } catch (_) {} - if (size > 0) { - uploadPromises.push( - createUploadStream( - S3_BUCKET, - currentSession.hiresKey, - createReadStream(currentSession.localMasterPath), - ).then(() => { - try { unlinkSync(currentSession.localMasterPath); } catch (_) {} - }) - ); - } else { - console.warn('[capture] local master is 0 bytes — skipping upload:', currentSession.localMasterPath); - } - } else if (currentSession.uploads.hires) { - uploadPromises.push(currentSession.uploads.hires); - } - + if (currentSession.uploads.hires) uploadPromises.push(currentSession.uploads.hires); if (currentSession.uploads.proxy) uploadPromises.push(currentSession.uploads.proxy); await Promise.all(uploadPromises); } catch (error) {