diff --git a/docker-compose.yml b/docker-compose.yml index 7b35400..c92e57c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,11 @@ services: dockerfile: Dockerfile.gpu image: wild-dragon-worker-gpu:latest runtime: nvidia + # Privileged so the promotion scanner can mount the growing-files CIFS share + # at /growing (same Approach A as the capture sidecar). Without the share + # mounted the scanner watches an empty local dir and never promotes growing + # captures to S3. + privileged: true depends_on: - queue - db @@ -136,7 +141,9 @@ services: WORKER_LABEL: "zampp1 / Tesla P4" NVIDIA_DRIVER_CAPABILITIES: video,compute,utility volumes: - - /mnt/NVME/MAM/wild-dragon-growing:/growing + # NOTE: /growing is NOT a host bind anymore — the promotion scanner mounts + # the CIFS landing-zone share there itself (a bind would shadow it). The + # mount needs rshared propagation so the in-container CIFS mount is visible. - /mnt/NVME/MAM/wild-dragon-media:/media networks: - wild-dragon diff --git a/services/capture/src/capture-manager.js b/services/capture/src/capture-manager.js index c2f7dab..09df4e0 100644 --- a/services/capture/src/capture-manager.js +++ b/services/capture/src/capture-manager.js @@ -210,30 +210,43 @@ const CONTAINER_EXT = { // Growing-file (edit-while-record) master format. // -// Premiere's "open capture in progress" / grow-on-disk support is FORMAT- -// SPECIFIC. A fragmented MP4/MOV (`+frag_keyframe+empty_moov+default_base_moof`) -// is NOT openable by Premiere as a growing file — its QuickTime importer needs -// the classic stco/stsz/stts sample tables in a single top-level moov, which a -// fragmented MOV never has while growing (samples live in moof/trun fragments). -// Symptom: "Unable to open file on disk." (Confirmed via ffprobe on zampp2: the -// growing .mov is ftyp + empty moov + repeating moof/mdat pairs, no sample -// tables.) +// Two prior attempts FAILED at the file level, both proven on zampp2: // -// The robust, broadcast-standard growing format Premiere DOES ingest is -// MXF OP1a (`-f mxf`) carrying a Premiere-native intra codec. We use DNxHR HQ -// (4:2:2 8-bit) which ffmpeg's MXF muxer accepts (HEVC/ProRes-in-MXF are -// rejected by this build), every frame is intra so a partially-written file is -// decodable to its last complete frame, and MXF writes header + body partitions -// incrementally so readers see valid essence mid-write. The same finalized .mxf -// is also a clean, Premiere-native asset, so the promotion/finalized path stays -// valid. +// 1) Fragmented MP4/MOV (`+frag_keyframe+empty_moov+default_base_moof`): +// NOT openable by Premiere — its QuickTime importer needs the classic +// stco/stsz/stts sample tables in a single top-level moov, which a +// fragmented MOV never has while growing (samples live in moof/trun). // -// Trade-off: DNxHR HQ is large (~22 GB/min at 1080p). Switch the profile to -// dnxhr_sq below (~half the bitrate) if disk is the constraint. +// 2) MXF OP1a / DNxHR HQ (`-f mxf`): ffmpeg can read it, but MXF OP1a writes +// its index + duration ONLY in the FOOTER partition, emitted on clean +// finalize. While growing, `ffprobe` reports `duration=N/A` and there is +// no footer/index, so VLC and Premiere REFUSE to open the in-progress +// file ("Unable to open file on disk"). Verified live: a growing .mxf +// probes `duration=N/A`; the same file after stop probes a real duration. +// MXF-while-growing on a CIFS target is therefore fundamentally unreliable +// for edit-while-record. +// +// FIX — growing MPEG-TS carrying H.264 ALL-INTRA + AAC. +// * MPEG-TS has NO footer/moov: every packet is self-describing and the file +// is valid from the first PAT/PMT onward. VLC and Premiere both open a +// still-growing .ts, and `ffprobe` reports a real (growing) duration from +// the continuous PCR — no finalize step is required for readability. +// * H.264 High 4:2:2 with `-g 1` makes every frame an IDR (all-intra), so a +// partially-written file decodes cleanly to its last complete frame — the +// prerequisite for edit-while-record — and Premiere ingests H.264-in-TS +// natively. (DNxHD/ProRes/PCM are NOT valid MPEG-TS payloads; verified +// ffmpeg rejects DNxHD-in-TS, hence H.264 + AAC.) +// * Verified on zampp2 MID-WRITE: ffprobe succeeds (format=mpegts, +// duration readable), and `ffmpeg -f null -` decodes both the H.264 video +// and AAC audio with exit 0 while the file is still being written. +// +// Audio: AAC (a TS-native codec Premiere imports). PCM-in-TS is tagged as +// SMPTE-302M `bin_data`, which Premiere does not reliably import as audio. const GROWING_VIDEO_ARGS = [ - '-c:v', 'dnxhd', '-profile:v', 'dnxhr_hq', '-pix_fmt', 'yuv422p', + '-c:v', 'libx264', '-profile:v', 'high422', '-pix_fmt', 'yuv422p', + '-preset', 'veryfast', '-g', '1', ]; -const GROWING_EXT = 'mxf'; +const GROWING_EXT = 'ts'; // ── Source-backend abstraction (issue #168) ────────────────────────────── // The capture input was historically hard-wired to a single `-f decklink -i …` @@ -323,18 +336,19 @@ function buildEncodeArgs({ container, isNetwork, isProxy = false, growing = false, }) { - // ── Growing master: force MXF OP1a + DNxHR, ignoring the configured MOV/ - // ProRes container/codec. This is the only combination Premiere opens as a - // growing file (see GROWING_VIDEO_ARGS above). Audio is forced to PCM, - // which MXF carries natively and Premiere ingests. + // ── Growing master: force MPEG-TS + H.264 all-intra, ignoring the configured + // MOV/ProRes container/codec. This is a format VLC and Premiere both open + // WHILE GROWING (no footer/moov to wait on — see GROWING_VIDEO_ARGS above). + // Audio is forced to AAC, a TS-native codec Premiere imports (PCM-in-TS is + // tagged as bin_data and not reliably importable as audio). if (growing) { const args = []; if (isNetwork) args.push('-map', '0:v:0?', '-map', '0:a:0?'); args.push(...GROWING_VIDEO_ARGS); if (framerate && framerate !== 'native') args.push('-r', framerate); - args.push('-c:a', 'pcm_s24le'); + args.push('-c:a', 'aac', '-b:a', '256k'); if (audioChannels) args.push('-ac', String(audioChannels)); - args.push('-f', 'mxf'); + args.push('-f', 'mpegts'); return args; } @@ -511,12 +525,10 @@ class CaptureManager { } const sessionId = uuidv4(); - const hiresExt = CONTAINER_EXT[container] || 'mov'; const proxyExt = CONTAINER_EXT[proxyContainer] || 'mp4'; - const hiresKey = `projects/${projectId}/masters/${clipName}.${hiresExt}`; - // Growing-files: write master to the local SMB share instead of streaming - // to S3. Path is relative to the container's GROWING_PATH mount. + // Growing-files: write master to the SMB share instead of streaming to S3. + // Path is relative to the container's GROWING_PATH mount. // // Approach A: if a CIFS source is configured, mount it now. A mount failure // is non-fatal — we fall back to S3 streaming so the recording is never @@ -525,12 +537,21 @@ class CaptureManager { if (growingActive && GROWING_SMB_MOUNT) { if (!mountGrowingShare()) growingActive = false; // fall back to S3 } - // Growing master is always MXF OP1a (the only Premiere-growable format here), - // regardless of the recorder's configured container — so it gets a .mxf - // extension, not hiresExt. + // Growing master is always MPEG-TS (the format VLC + Premiere open while + // growing — see GROWING_VIDEO_ARGS), regardless of the recorder's configured + // container — so it gets a .ts extension, not the container's. const growingPath = growingActive ? `${GROWING_PATH}/${projectId}/${clipName}.${GROWING_EXT}` : null; + + // hiresKey MUST match the actual master format/destination: + // - growing active → the promotion worker uploads the on-share .ts to this + // key, so it has the .ts extension. (A stale .mov key here would make the + // proxy job download a nonexistent object → "unable to open the file on + // disk".) + // - growing fell back to S3 → the normal container extension. + const hiresExt = growingPath ? GROWING_EXT : (CONTAINER_EXT[container] || 'mov'); + const hiresKey = `projects/${projectId}/masters/${clipName}.${hiresExt}`; if (growingPath) { try { mkdirSync(dirname(growingPath), { recursive: true }); } catch (err) { console.error('[capture] could not create growing dir:', err.message); } diff --git a/services/capture/src/index.js b/services/capture/src/index.js index 59d0cbe..678e073 100644 --- a/services/capture/src/index.js +++ b/services/capture/src/index.js @@ -135,6 +135,15 @@ async function gracefulShutdown(signal) { console.error('[shutdown] failed to flag empty asset:', e.message); } } + } else if (completed.growingPath) { + // Growing-files recorder: the master lives on the SMB share as a .ts, + // NOT in S3 yet. The promotion worker (which watches the same share) + // uploads it to S3 and enqueues the proxy from the real, finalized key. + // We must NOT call /finalize here: that sets original_s3_key to a key + // that doesn't exist yet and enqueues a proxy that instantly fails with + // "unable to open the file on disk." Leave the asset 'live' for the + // promotion worker to flip to 'ready'. + console.log(`[shutdown] growing capture finalized on share (${completed.growingPath}); leaving promotion worker to upload + proxy`); } else if (liveAssetId) { // Finalise the pre-created live asset by id (avoids POST / 409 collision). try { diff --git a/services/mam-api/src/routes/assets.js b/services/mam-api/src/routes/assets.js index 8e3463d..148775e 100644 --- a/services/mam-api/src/routes/assets.js +++ b/services/mam-api/src/routes/assets.js @@ -742,11 +742,14 @@ router.get('/:id/live-path', async (req, res, next) => { const cfg = {}; for (const { key, value } of s.rows) cfg[key] = value; if (!cfg.growing_smb_url) return res.status(409).json({ error: 'No SMB URL configured — set the editor SMB URL in Settings → Storage' }); - const rec = await pool.query( - `SELECT recording_container FROM recorders WHERE current_session_id = $1 ORDER BY updated_at DESC LIMIT 1`, - [asset.id] - ); - const ext = rec.rows[0]?.recording_container || 'mov'; + // The growing master is ALWAYS MPEG-TS (.ts) on the share, regardless of the + // recorder's configured finalized container — that is the format VLC and + // Premiere can open WHILE it is still growing (no footer/moov to wait on). + // Pointing the editor at the recorder's `.mov`/`.mxf` container here was a + // bug: the file on the share is `.ts`, so the editor got "file not + // found / unable to open." Keep this in lock-step with GROWING_EXT in + // services/capture/src/capture-manager.js. + const ext = 'ts'; const smbRoot = cfg.growing_smb_url.replace(/\/+$/, ''); const winPath = smbRoot.replace(/^smb:\/\//, '\\\\').replace(/\//g, '\\') + `\\${asset.project_id}\\${asset.display_name}.${ext}`; const posix = smbRoot.replace(/^smb:\/\//, '//') + `/${asset.project_id}/${asset.display_name}.${ext}`; diff --git a/services/worker/Dockerfile.gpu b/services/worker/Dockerfile.gpu index 56d122b..e80be18 100644 --- a/services/worker/Dockerfile.gpu +++ b/services/worker/Dockerfile.gpu @@ -16,8 +16,10 @@ FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 # track YouTube's frequent changes, so we pull the latest self-contained # release binary at build time. /usr/local/bin precedes /usr/bin on PATH, so # `yt-dlp` resolves to this one. Rebuild the worker image to refresh it. +# cifs-utils: the promotion scanner mounts the growing-files SMB landing zone +# at /growing so it watches the SAME share the capture sidecars write to. RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates ffmpeg python3 \ + curl ca-certificates ffmpeg python3 cifs-utils \ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux \ diff --git a/services/worker/src/workers/promotion.js b/services/worker/src/workers/promotion.js index d5a5429..5ad63df 100644 --- a/services/worker/src/workers/promotion.js +++ b/services/worker/src/workers/promotion.js @@ -6,7 +6,8 @@ // Why a poll loop and not chokidar: NFS/SMB mounts don't reliably surface // inotify events through the kernel; mtime polling is the boring-but-works // answer for fairness across all storage backends. -import { readdir, stat, unlink } from 'node:fs/promises'; +import { readdir, stat, unlink, mkdir, writeFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; import { join, relative, basename } from 'node:path'; import { createReadStream } from 'node:fs'; import { Queue } from 'bullmq'; @@ -16,6 +17,66 @@ import { uploadStreamToS3 } from '../s3/client.js'; const GROWING_PATH = process.env.GROWING_PATH || '/growing'; const S3_BUCKET = process.env.S3_BUCKET || 'wild-dragon'; const POLL_MS = 5000; +const SMB_CREDS_FILE = '/run/promotion-smb-creds'; + +// Normalize a Windows / smb:// share path to the //host/share UNC that +// mount.cifs accepts (mirrors services/capture/src/capture-manager.js). +function toUncShare(raw) { + if (!raw) return ''; + let s = String(raw).trim().replace(/\\/g, '/'); + s = s.replace(/^smb:\/\//i, '//'); + if (!s.startsWith('//')) s = '//' + s.replace(/^\/+/, ''); + return s; +} + +function isMounted(path) { + try { execFileSync('mountpoint', ['-q', path]); return true; } + catch { return false; } +} + +// Mount the growing-files CIFS share at GROWING_PATH so the promotion scanner +// sees the SAME files the capture sidecar writes on the remote node. Without +// this the worker was watching a LOCAL empty /growing and never promoted any +// growing capture — the master never reached S3 and the only proxy that fired +// was the bogus one from capture's finalize call (against a key that doesn't +// exist) → "unable to open the file on disk". Best-effort + idempotent. +async function ensureGrowingShareMounted() { + const r = await query( + `SELECT key, value FROM settings WHERE key = ANY($1)`, + [['growing_smb_mount', 'growing_smb_username', 'growing_smb_password', 'growing_smb_vers']] + ).catch(() => ({ rows: [] })); + const cfg = {}; + for (const { key, value } of r.rows) cfg[key] = value; + + const share = toUncShare(cfg.growing_smb_mount || ''); + if (!share) { + console.log('[promotion] no growing_smb_mount configured — using local GROWING_PATH'); + return; + } + try { + if (isMounted(GROWING_PATH)) { + console.log('[promotion] growing share already mounted at', GROWING_PATH); + return; + } + await mkdir(GROWING_PATH, { recursive: true }).catch(() => {}); + await writeFile( + SMB_CREDS_FILE, + `username=${cfg.growing_smb_username || ''}\npassword=${cfg.growing_smb_password || ''}\n`, + { mode: 0o600 } + ); + const opts = [ + `credentials=${SMB_CREDS_FILE}`, + 'uid=0', 'gid=0', 'file_mode=0664', 'dir_mode=0775', + `vers=${cfg.growing_smb_vers || '3.0'}`, + ].join(','); + execFileSync('mount', ['-t', 'cifs', share, GROWING_PATH, '-o', opts], + { stdio: ['ignore', 'ignore', 'pipe'] }); + console.log('[promotion] mounted CIFS growing share', share, '->', GROWING_PATH); + } catch (err) { + const stderr = err.stderr ? err.stderr.toString().trim() : err.message; + console.error('[promotion] CIFS mount failed (promotion will not see growing files):', stderr); + } +} let inflight = new Set(); let idleThresholdMs = 8000; @@ -137,6 +198,10 @@ async function scan() { // and close the queue connection during SIGTERM. export function startPromotionWorker() { loadIdleThreshold(); + // Mount the SMB landing zone before the first scan so we watch the SAME share + // the capture sidecars write to (best-effort; falls back to local GROWING_PATH). + ensureGrowingShareMounted().catch((e) => + console.error('[promotion] mount bootstrap failed:', e.message)); thresholdInterval = setInterval(loadIdleThreshold, 60_000); scanInterval = setInterval(scan, POLL_MS); console.log(`[promotion] watching ${GROWING_PATH} (idle threshold ${idleThresholdMs}ms)`);