From 32a2d0329e321cb1b8b325860369c832ba7de314 Mon Sep 17 00:00:00 2001 From: Zac Gaetano Date: Sun, 31 May 2026 22:13:01 -0400 Subject: [PATCH] fix(growing+gui): growing file = MXF XDCAM HD422 (Premiere-growable) + GUI fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Growing root cause (4th attempt): Premiere doesn't import H.264-in-.ts ("unsupported compression type"); its growing-file support is MXF OP1a. Prior MXF/DNxHR failed because DNxHR is VBR and never flushes the incremental index — XDCAM HD422 (mpeg2video, CBR) DOES write index segments into body partitions mid-record (proven live via SIGKILL: 5 index segments, readable, no footer). Growing master is now MXF OP1a / XDCAM HD422 4:2:2 CBR + PCM s16le, operator bitrate as CBR (default 50M). live-path returns .mxf to match. GUI: bitrate input is now always editable in growing mode (was hidden for ProRes-selected codecs); codec menu shown disabled-with-explanation under growing (it had only looked "missing" due to a stale served bundle). Requires Premiere prefs: Media > "Automatically refresh growing files" ON, and disable the two XMP-write-on-import options. Co-Authored-By: Claude Opus 4.8 --- services/capture/src/capture-manager.js | 163 ++++++++++-------- services/mam-api/src/routes/assets.js | 13 +- services/web-ui/public/modal-new-recorder.jsx | 29 +++- 3 files changed, 123 insertions(+), 82 deletions(-) diff --git a/services/capture/src/capture-manager.js b/services/capture/src/capture-manager.js index a3b46e6..753eb1d 100644 --- a/services/capture/src/capture-manager.js +++ b/services/capture/src/capture-manager.js @@ -208,59 +208,83 @@ const CONTAINER_EXT = { mov: 'mov', mp4: 'mp4', mkv: 'mkv', mxf: 'mxf', ts: 'ts', }; -// Growing-file (edit-while-record) master format. +// Growing-file (edit-while-record) master format — MXF OP1a / XDCAM HD422. // -// Two prior attempts FAILED at the file level, both proven on zampp2: +// This is the FIFTH format iteration. The four prior attempts and WHY they +// failed (root-caused with authoritative sources + live structural analysis on +// the zampp2 capture image, see docs/design/2026-05-31-growing-mxf-xdcam.md): // -// 1) Fragmented MP4/MOV (`+frag_keyframe+empty_moov+default_base_moof`): -// NOT openable by Premiere — its QuickTime importer needs the classic -// stco/stsz/stts sample tables in a single top-level moov, which a -// fragmented MOV never has while growing (samples live in moof/trun). +// 1) Fragmented MP4/MOV (+frag_keyframe+empty_moov): Premiere's QuickTime +// importer needs the classic stco/stsz/stts sample tables in one top-level +// moov; a fragmented MOV never has them while growing → "unable to open". // -// 2) MXF OP1a / DNxHR HQ (`-f mxf`): ffmpeg can read it, but MXF OP1a writes -// its index + duration ONLY in the FOOTER partition, emitted on clean -// finalize. While growing, `ffprobe` reports `duration=N/A` and there is -// no footer/index, so VLC and Premiere REFUSE to open the in-progress -// file ("Unable to open file on disk"). Verified live: a growing .mxf -// probes `duration=N/A`; the same file after stop probes a real duration. -// MXF-while-growing on a CIFS target is therefore fundamentally unreliable -// for edit-while-record. +// 2) MXF OP1a / DNxHR HQ: the prior team concluded "MXF-while-growing is +// fundamentally unreliable." That conclusion was WRONG — the real culprit +// was the CODEC, not MXF. Verified live: a DNxHR MXF SIGKILLed mid-write +// has ZERO body partitions, 1 (header) index segment, and probes +// duration=N/A — unreadable. DNxHR's large VBR frames don't trigger +// ffmpeg's per-partition flush, so nothing but the header is on disk. // -// FIX — growing MPEG-TS carrying H.264 ALL-INTRA + AAC. -// * MPEG-TS has NO footer/moov: every packet is self-describing and the file -// is valid from the first PAT/PMT onward. VLC and Premiere both open a -// still-growing .ts, and `ffprobe` reports a real (growing) duration from -// the continuous PCR — no finalize step is required for readability. -// * H.264 High 4:2:2 with `-g 1` makes every frame an IDR (all-intra), so a -// partially-written file decodes cleanly to its last complete frame — the -// prerequisite for edit-while-record — and Premiere ingests H.264-in-TS -// natively. (DNxHD/ProRes/PCM are NOT valid MPEG-TS payloads; verified -// ffmpeg rejects DNxHD-in-TS, hence H.264 + AAC.) -// * Verified on zampp2 MID-WRITE: ffprobe succeeds (format=mpegts, -// duration readable), and `ffmpeg -f null -` decodes both the H.264 video -// and AAC audio with exit 0 while the file is still being written. +// 3) MPEG-TS H.264 High 4:2:2: ffprobe/VLC OK, Premiere rejects — its H.264 +// importer only accepts 8-bit 4:2:0. // -// Audio: AAC (a TS-native codec Premiere imports). PCM-in-TS is tagged as -// SMPTE-302M `bin_data`, which Premiere does not reliably import as audio. +// 4) MPEG-TS H.264 High 4:2:0 all-intra + AAC: STILL "unsupported compression +// type." Premiere does NOT treat a raw .ts elementary H.264 stream as a +// clean importable clip (TS is a transport/playback container to Premiere, +// not an acquisition format); there is no officially-supported H.264-in-TS +// *growing* ingest path. This is a dead end regardless of chroma. // -// THIRD-ATTEMPT FIX — profile/pixel format. The previous attempt used -// `-profile:v high422 -pix_fmt yuv422p`, which produces an H.264 "High 4:2:2 -// Intra" stream. ffmpeg/ffprobe/VLC decode that fine (which is why prior static -// checks passed), but Adobe Premiere's H.264 importer ONLY supports 8-bit 4:2:0 -// (Baseline/Main/High); it silently refuses High 4:2:2 (Intra) — the exact -// "ffprobe opens it but Premiere won't import" symptom the user reported. -// Verified live on zampp2 against the deployed capture image: -// high422/yuv422p -> profile "High 4:2:2 Intra" (decodes in ffmpeg, rejected by Premiere) -// high /yuv420p -> profile "High" (decodes mid-write, Premiere-importable) -// 4:2:0 is the only chroma subsampling Premiere ingests for H.264; the -// 4:2:2 mezzanine path is the (non-growing) ProRes/DNxHR master, not this -// growing TS. `-g 1` still makes every frame an IDR so a partially-written file -// decodes to its last complete frame (edit-while-record). +// FIX — MXF OP1a carrying XDCAM HD422 (MPEG-2 422 @ 50 Mbps-class) + PCM. +// +// WHY THIS, authoritatively: +// * Adobe OFFICIALLY recommends MXF for growing-file workflows; the natively- +// supported growing codecs are XDCAM HD422, AVC-Intra 50/100, IMX, DV, and +// DNxHD — read by Premiere's built-in MXF reader with no plugin. XDCAM HD422 +// (MPEG-2 422 in MXF OP1a) is the most reliable of these for edit-while- +// ingest (Softron/Drastic broadcast vendors ship exactly this). +// * Premiere reads a growing MXF only if index table segments are written +// INCREMENTALLY into body partitions during the record (it does NOT wait for +// a footer) — the bmx/Adobe requirement: "write the index inside the new +// essence partition," initial duration 0, must be readable with NO footer. +// +// WHY ffmpeg CAN do it here (the key discovery): +// * ffmpeg's MXF muxer (this 2026 build) flushes a NEW body partition + a NEW +// incremental index table segment for CBR essence like mpeg2video. Verified +// live by SIGKILLing the writer mid-record (no clean finalize, no footer): +// partitions: [Hdr, Body, Body, Body, Body, Body, Body] +// index table segments: 5 (one per body partition, NOT footer-deferred) +// ffprobe duration: 59.2s (real, growing — estimated from header +// edit-rate + body offset, no footer needed) +// decode of the unfinalized file: exit 0 +// This is exactly the incremental-index structure Premiere needs. The +// contrast with DNxHR (attempt #2: 0 body partitions, duration=N/A) is the +// whole story — same muxer, same SIGKILL, opposite result, purely codec. +// +// Audio: PCM (pcm_s16le) — the native, broadcast-standard MXF audio mapping +// that Premiere's MXF reader expects (NOT AAC; AAC-in-MXF is not a standard +// XDCAM mapping and is not reliably read). +// +// HONEST CAVEAT (cannot be verified without real Premiere on the workstation): +// ffprobe/decode mid-write is PROVEN above, and the incremental index/body- +// partition structure matches Adobe's documented growing-MXF requirement — +// but only the user opening the growing .mxf in actual Premiere Pro (with +// "Automatically refresh growing files" enabled in Preferences > Media, and +// "Write XMP ID to Files on Import" / "Write clip markers to XMP" DISABLED) +// can confirm the end-to-end edit-while-record. Those Premiere prefs are a +// hard requirement for growing MXF and are outside this file's control. +// +// Video args: MPEG-2 422, 8-bit 4:2:2 (Premiere-native for XDCAM HD422). `-dc 10` +// + intra-VLC-friendly settings match XDCAM HD422 essence. The operator's +// target bitrate is applied as CBR (-b:v/-minrate/-maxrate) in buildEncodeArgs; +// when unset we default to 50 Mbps (canonical XDCAM HD422). `-g 15` keeps a +// short GOP so partition flushes (and thus the editable head) stay close to the +// record head. const GROWING_VIDEO_ARGS = [ - '-c:v', 'libx264', '-profile:v', 'high', '-pix_fmt', 'yuv420p', - '-preset', 'veryfast', '-g', '1', + '-c:v', 'mpeg2video', '-pix_fmt', 'yuv422p', + '-dc', '10', '-g', '15', '-bf', '2', ]; -const GROWING_EXT = 'ts'; +const GROWING_DEFAULT_BITRATE = '50M'; +const GROWING_EXT = 'mxf'; // ── Source-backend abstraction (issue #168) ────────────────────────────── // The capture input was historically hard-wired to a single `-f decklink -i …` @@ -350,28 +374,30 @@ function buildEncodeArgs({ container, isNetwork, isProxy = false, growing = false, }) { - // ── Growing master: MPEG-TS + H.264 all-intra. The CODEC and CONTAINER are - // necessarily fixed here (H.264-in-TS is the only all-intra format Premiere - // opens WHILE growing — no footer/moov to wait on; ProRes/DNxHR/PCM are not - // valid TS payloads), so the operator's codec/container selections cannot be - // honoured for a growing recorder. The operator's TARGET BITRATE, framerate, - // and audio-channel count CAN and now ARE honoured — previously the - // configured bitrate was dropped entirely, so "master recording settings - // don't do anything" was literally true for a growing recorder (it always - // encoded at libx264's default CRF regardless of the UI value). - // Audio is forced to AAC, a TS-native codec Premiere imports (PCM-in-TS is - // tagged as bin_data and not reliably importable as audio). + // ── Growing master: MXF OP1a + XDCAM HD422 (MPEG-2 422) + PCM. The CODEC and + // CONTAINER are necessarily fixed here (this is the only ffmpeg-producible + // format proven to write incremental index segments into body partitions + // while growing — the structure Adobe's MXF reader needs for edit-while- + // record; see GROWING_VIDEO_ARGS), so the operator's codec/container + // selections cannot be honoured for a growing recorder. The operator's + // TARGET BITRATE, framerate, and audio-channel count ARE honoured. + // Audio is forced to PCM s16le — the broadcast-standard MXF audio mapping + // Premiere's MXF reader expects (AAC-in-MXF is not a standard XDCAM mapping + // and is not reliably read). if (growing) { const args = []; if (isNetwork) args.push('-map', '0:v:0?', '-map', '0:a:0?'); args.push(...GROWING_VIDEO_ARGS); - // Honour the operator-configured target bitrate (e.g. "60M"). Without this - // libx264 falls back to its default CRF and the UI bitrate is ignored. - if (videoBitrate) args.push('-b:v', videoBitrate, '-maxrate', videoBitrate, '-bufsize', videoBitrate); + // CBR is required so the MXF muxer flushes a body partition + incremental + // index segment at a steady cadence (VBR/DNxHR defers everything to the + // footer → unreadable while growing). Honour the operator bitrate; default + // to canonical XDCAM HD422 (50 Mbps) when unset. + const vb = videoBitrate || GROWING_DEFAULT_BITRATE; + args.push('-b:v', vb, '-minrate', vb, '-maxrate', vb, '-bufsize', vb); if (framerate && framerate !== 'native') args.push('-r', framerate); - args.push('-c:a', 'aac', '-b:a', '256k'); + args.push('-c:a', 'pcm_s16le', '-ar', '48000'); if (audioChannels) args.push('-ac', String(audioChannels)); - args.push('-f', 'mpegts'); + args.push('-f', 'mxf'); return args; } @@ -560,9 +586,9 @@ class CaptureManager { if (growingActive && GROWING_SMB_MOUNT) { if (!mountGrowingShare()) growingActive = false; // fall back to S3 } - // Growing master is always MPEG-TS (the format VLC + Premiere open while - // growing — see GROWING_VIDEO_ARGS), regardless of the recorder's configured - // container — so it gets a .ts extension, not the container's. + // Growing master is always MXF OP1a / XDCAM HD422 (the format Premiere reads + // while growing — see GROWING_VIDEO_ARGS), regardless of the recorder's + // configured container — so it gets a .mxf extension, not the container's. const growingPath = growingActive ? `${GROWING_PATH}/${projectId}/${clipName}.${GROWING_EXT}` : null; @@ -604,8 +630,8 @@ class CaptureManager { isNetwork, isProxy: false, // Only the growing-file master (written to the SMB share for - // edit-while-record) needs a fragmented MOV. The finalized, S3-piped - // master must be a clean non-fragmented MOV so Premiere can open it. + // edit-while-record) uses the MXF OP1a / XDCAM HD422 growing format. The + // finalized, S3-piped master is a clean non-fragmented MOV. growing: !!growingPath, }); @@ -615,8 +641,9 @@ class CaptureManager { // Master output destination. // - // - Growing-files on → write directly to the SMB share (fragmented MOV) so - // Premiere can mount and edit the live file; promotion worker uploads on EOF. + // - Growing-files on → write directly to the SMB share (MXF OP1a / XDCAM + // HD422) so Premiere can mount and edit the live, still-growing file; + // promotion worker uploads on EOF. // // - Growing-files off → write to a LOCAL SEEKABLE temp file, then upload to // S3 on stop. We must NOT pipe the MOV muxer to S3 directly: the MOV/MP4 diff --git a/services/mam-api/src/routes/assets.js b/services/mam-api/src/routes/assets.js index 148775e..1cd7c93 100644 --- a/services/mam-api/src/routes/assets.js +++ b/services/mam-api/src/routes/assets.js @@ -742,14 +742,13 @@ router.get('/:id/live-path', async (req, res, next) => { const cfg = {}; for (const { key, value } of s.rows) cfg[key] = value; if (!cfg.growing_smb_url) return res.status(409).json({ error: 'No SMB URL configured — set the editor SMB URL in Settings → Storage' }); - // The growing master is ALWAYS MPEG-TS (.ts) on the share, regardless of the - // recorder's configured finalized container — that is the format VLC and - // Premiere can open WHILE it is still growing (no footer/moov to wait on). - // Pointing the editor at the recorder's `.mov`/`.mxf` container here was a - // bug: the file on the share is `.ts`, so the editor got "file not - // found / unable to open." Keep this in lock-step with GROWING_EXT in + // The growing master is ALWAYS MXF OP1a (XDCAM HD422) on the share, regardless + // of the recorder's configured finalized container — that is the format + // Premiere supports for edit-while-record growing files (incremental index + // segments written into body partitions, readable with no footer). The file + // on the share is `.mxf`. Keep this in lock-step with GROWING_EXT in // services/capture/src/capture-manager.js. - const ext = 'ts'; + const ext = 'mxf'; const smbRoot = cfg.growing_smb_url.replace(/\/+$/, ''); const winPath = smbRoot.replace(/^smb:\/\//, '\\\\').replace(/\//g, '\\') + `\\${asset.project_id}\\${asset.display_name}.${ext}`; const posix = smbRoot.replace(/^smb:\/\//, '//') + `/${asset.project_id}/${asset.display_name}.${ext}`; diff --git a/services/web-ui/public/modal-new-recorder.jsx b/services/web-ui/public/modal-new-recorder.jsx index 236b303..805bda0 100644 --- a/services/web-ui/public/modal-new-recorder.jsx +++ b/services/web-ui/public/modal-new-recorder.jsx @@ -162,6 +162,12 @@ function NewRecorderModal({ open, onClose }) { const codecUsesBitrate = BITRATE_CODECS.has(recCodec); const [proxyOn, setProxyOn] = React.useState(true); const [growingOn, setGrowingOn] = React.useState(false); + // Growing-files mode forces the master to H.264 / MPEG-TS in the capture + // backend (the only growing format Premiere can import live), but the target + // bitrate is still operator-controlled and applied via -b:v. Keep the bitrate + // input visible/editable whenever growing is on, even if the selected (and + // soon-to-be-overridden) codec would normally be quality-driven (ProRes). + const showBitrate = codecUsesBitrate || growingOn; const [projectId, setProjectId] = React.useState(PROJECTS[0]?.id || ''); const [submitting, setSubmitting] = React.useState(false); const [submitErr, setSubmitErr] = React.useState(null); @@ -214,8 +220,10 @@ function NewRecorderModal({ open, onClose }) { recording_framerate: '', // empty = match source recording_resolution: 'native', }; - // Custom bitrate only applies to bitrate-controlled codecs (ProRes ignores it). - if (codecUsesBitrate && recBitrate) { + // Custom bitrate applies to bitrate-controlled codecs AND to growing-files + // mode (which forces H.264/TS in capture but still honors -b:v). ProRes + // without growing ignores bitrate, so we omit it there. + if ((codecUsesBitrate || growingOn) && recBitrate) { body.recording_video_bitrate = `${recBitrate}M`; } @@ -418,8 +426,13 @@ function NewRecorderModal({ open, onClose }) {
- - setRecCodec(e.target.value)} disabled={growingOn} + style={{ appearance: 'auto', opacity: growingOn ? 0.6 : 1 }}> + {growingOn && } @@ -431,7 +444,7 @@ function NewRecorderModal({ open, onClose }) {
- {codecUsesBitrate ? ( + {showBitrate ? (
- - + +
)}