From bbed2a7059806ac5b9d21a864f0e825741ff0f45 Mon Sep 17 00:00:00 2001 From: ZGaetano Date: Thu, 21 May 2026 18:51:10 -0400 Subject: [PATCH] fix(decklink): mount /dev/blackmagic in sidecar + remote node routing via node-agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs fixed: 1. SDI capture sidecar never had /dev/blackmagic bound — ffmpeg opened the decklink input inside a container with no device nodes, so frame=0. Fix: local spawns now push '/dev/blackmagic:/dev/blackmagic' onto Binds when source_type='sdi'. 2. recorders.js always spawned sidecars against the local Docker socket (zampp1), even when a recorder's node_id pointed at zampp2 (where the card is). Fix: resolveNodeTarget() looks up the recorder's cluster node; if it's a different hostname the sidecar is spawned via a new POST /sidecar/start endpoint on the remote node-agent. node-agent gains three new routes (all talk to the local Docker socket): POST /sidecar/start — create + start container (host network, privileged, /dev/blackmagic bind for sdi) DELETE /sidecar/:id — stop + remove GET /sidecar/:id/status — inspect + poll capture service docker-compose.worker.yml: add /var/run/docker.sock and LIVE_DIR to node-agent so it can spawn sidecars, and document build-capture prerequisite.: recorders.js --- services/mam-api/src/routes/recorders.js | 246 +++++++++++++++-------- 1 file changed, 166 insertions(+), 80 deletions(-) diff --git a/services/mam-api/src/routes/recorders.js b/services/mam-api/src/routes/recorders.js index ca89c10..751a439 100644 --- a/services/mam-api/src/routes/recorders.js +++ b/services/mam-api/src/routes/recorders.js @@ -8,6 +8,10 @@ const router = express.Router(); router.use(requireAuth); +// Base port for on-demand SDI sidecar containers on remote worker nodes. +// Device index 0 → 7438, index 1 → 7439, etc. +const SIDECAR_PORT_BASE = 7438; + // Docker API helper function function dockerApi(method, path, body = null) { return new Promise((resolve, reject) => { @@ -34,6 +38,22 @@ function dockerApi(method, path, body = null) { }); } +// Look up the cluster node for a recorder and decide if it is remote. +// Returns { remote: false } when the node is local or unset; +// { remote: true, apiUrl, ip } when it is a different host. +async function resolveNodeTarget(nodeId) { + if (!nodeId) return { remote: false }; + const r = await pool.query( + 'SELECT hostname, ip_address, api_url FROM cluster_nodes WHERE id = $1', + [nodeId] + ); + if (r.rows.length === 0) return { remote: false }; + const node = r.rows[0]; + const localHostname = process.env.NODE_HOSTNAME || ''; + if (!node.api_url || node.hostname === localHostname) return { remote: false }; + return { remote: true, apiUrl: node.api_url, ip: node.ip_address }; +} + // Helper function to generate clip name with timestamp function generateClipName(recorderName) { const now = new Date(); @@ -284,8 +304,6 @@ router.post('/:id/start', async (req, res, next) => { const sourceType = recorder.source_type; const deviceIndex = recorder.device_index ?? sourceConfig.device ?? 0; - const { portBindings, exposedPorts } = buildPortConfig(sourceType, sourceConfig); - // Build container env — all codec controls flow through here. const env = [ `S3_ENDPOINT=${s3Endpoint}`, @@ -337,42 +355,68 @@ router.post('/:id/start', async (req, res, next) => { } } - const alias = `recorder-${id}`; - const containerConfig = { - Image: 'wild-dragon-capture:latest', - Env: env, - ExposedPorts: Object.keys(exposedPorts).length > 0 ? exposedPorts : undefined, - HostConfig: { - Privileged: true, - NetworkMode: dockerNetwork, - PortBindings: Object.keys(portBindings).length > 0 ? portBindings : undefined, - Binds: ['/mnt/NVME/MAM/wild-dragon-live:/live'], - }, - NetworkingConfig: { - EndpointsConfig: { - [dockerNetwork]: { Aliases: [alias] }, + // Determine whether to spawn locally or via a remote node-agent. + const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id); + + let containerId; + + if (isRemote) { + // Remote node: delegate container lifecycle to that node's agent. + const capturePort = SIDECAR_PORT_BASE + (deviceIndex || 0); + const sidecarRes = await fetch(`${targetNodeApiUrl}/sidecar/start`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ image: 'wild-dragon-capture:latest', env, capturePort, sourceType }), + signal: AbortSignal.timeout(15000), + }); + if (!sidecarRes.ok) { + const details = await sidecarRes.json().catch(() => ({})); + return res.status(502).json({ error: 'Remote node failed to start sidecar', details }); + } + const sidecarData = await sidecarRes.json(); + containerId = sidecarData.containerId; + } else { + // Local spawn via Docker socket. + const { portBindings, exposedPorts } = buildPortConfig(sourceType, sourceConfig); + const alias = `recorder-${id}`; + + const hostBinds = ['/mnt/NVME/MAM/wild-dragon-live:/live']; + if (sourceType === 'sdi') hostBinds.push('/dev/blackmagic:/dev/blackmagic'); + + const containerConfig = { + Image: 'wild-dragon-capture:latest', + Env: env, + ExposedPorts: Object.keys(exposedPorts).length > 0 ? exposedPorts : undefined, + HostConfig: { + Privileged: true, + NetworkMode: dockerNetwork, + PortBindings: Object.keys(portBindings).length > 0 ? portBindings : undefined, + Binds: hostBinds, }, - }, - Hostname: alias, - }; + NetworkingConfig: { + EndpointsConfig: { + [dockerNetwork]: { Aliases: [alias] }, + }, + }, + Hostname: alias, + }; - const createRes = await dockerApi('POST', '/containers/create', containerConfig); + const createRes = await dockerApi('POST', '/containers/create', containerConfig); + if (createRes.status !== 201) { + return res.status(500).json({ + error: 'Failed to create container', + details: createRes.data, + }); + } - if (createRes.status !== 201) { - return res.status(500).json({ - error: 'Failed to create container', - details: createRes.data, - }); - } - - const containerId = createRes.data.Id; - const startRes = await dockerApi('POST', `/containers/${containerId}/start`); - - if (startRes.status !== 204) { - return res.status(500).json({ - error: 'Failed to start container', - details: startRes.data, - }); + containerId = createRes.data.Id; + const startRes = await dockerApi('POST', `/containers/${containerId}/start`); + if (startRes.status !== 204) { + return res.status(500).json({ + error: 'Failed to start container', + details: startRes.data, + }); + } } const updateResult = await pool.query( @@ -409,28 +453,40 @@ router.post('/:id/stop', async (req, res, next) => { return res.status(400).json({ error: 'No container running' }); } - const stopRes = await dockerApi( - 'POST', - `/containers/${recorder.container_id}/stop` - ); + const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id); - if (stopRes.status !== 204 && stopRes.status !== 304) { - return res.status(500).json({ - error: 'Failed to stop container', - details: stopRes.data, + if (isRemote) { + const stopRes = await fetch(`${targetNodeApiUrl}/sidecar/${recorder.container_id}`, { + method: 'DELETE', + signal: AbortSignal.timeout(15000), }); - } + if (!stopRes.ok && stopRes.status !== 404) { + return res.status(502).json({ error: 'Remote node failed to stop sidecar' }); + } + } else { + const stopRes = await dockerApi( + 'POST', + `/containers/${recorder.container_id}/stop` + ); - const removeRes = await dockerApi( - 'DELETE', - `/containers/${recorder.container_id}` - ); + if (stopRes.status !== 204 && stopRes.status !== 304) { + return res.status(500).json({ + error: 'Failed to stop container', + details: stopRes.data, + }); + } - if (removeRes.status !== 204 && removeRes.status !== 404) { - return res.status(500).json({ - error: 'Failed to remove container', - details: removeRes.data, - }); + const removeRes = await dockerApi( + 'DELETE', + `/containers/${recorder.container_id}` + ); + + if (removeRes.status !== 204 && removeRes.status !== 404) { + return res.status(500).json({ + error: 'Failed to remove container', + details: removeRes.data, + }); + } } const updateResult = await pool.query( @@ -471,37 +527,59 @@ router.get('/:id/status', async (req, res, next) => { }); } - const inspectRes = await dockerApi( - 'GET', - `/containers/${recorder.container_id}/json` - ); + const deviceIndex = recorder.device_index ?? (recorder.source_config?.device ?? 0); + const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id); - if (inspectRes.status !== 200) { - return res.json({ - status: 'unknown', - duration: 0, - containerId: recorder.container_id, - }); - } - - const container = inspectRes.data; - const startedAt = new Date(container.State.StartedAt).getTime(); - const now = Date.now(); - const duration = Math.floor((now - startedAt) / 1000); - - let signal = container.State.Running ? 'receiving' : 'stopped'; + let isRunning = false; + let duration = 0; + let signal = 'connecting'; let signalKnown = false; let live = null; - try { - const captureRes = await fetch(`http://recorder-${id}:3001/capture/status`, { signal: AbortSignal.timeout(2000) }); - if (captureRes.ok) { - live = await captureRes.json(); - if (live && live.signal) { signal = live.signal; signalKnown = true; } + + if (isRemote) { + try { + const statusRes = await fetch(`${targetNodeApiUrl}/sidecar/${recorder.container_id}/status`, { + signal: AbortSignal.timeout(4000), + }); + if (statusRes.ok) { + const data = await statusRes.json(); + isRunning = data.running; + if (data.startedAt) { + duration = Math.floor((Date.now() - new Date(data.startedAt).getTime()) / 1000); + } + live = data.live; + } + } catch (_) { /* node unreachable */ } + } else { + const inspectRes = await dockerApi( + 'GET', + `/containers/${recorder.container_id}/json` + ); + + if (inspectRes.status !== 200) { + return res.json({ + status: 'unknown', + duration: 0, + containerId: recorder.container_id, + }); } - } catch (_) { /* not ready yet */ } + + const container = inspectRes.data; + isRunning = container.State.Running; + duration = Math.floor((Date.now() - new Date(container.State.StartedAt).getTime()) / 1000); + + try { + const captureRes = await fetch(`http://recorder-${id}:3001/capture/status`, { signal: AbortSignal.timeout(2000) }); + if (captureRes.ok) live = await captureRes.json(); + } catch (_) { /* not ready yet */ } + } + + if (isRunning) signal = 'receiving'; + if (!isRunning) signal = 'stopped'; + if (live && live.signal) { signal = live.signal; signalKnown = true; } res.json({ - status: container.State.Running ? 'recording' : 'stopped', + status: isRunning ? 'recording' : 'stopped', duration, containerId: recorder.container_id, signal, @@ -533,9 +611,17 @@ router.delete('/:id', async (req, res, next) => { const recorder = recorderResult.rows[0]; if (recorder.container_id) { + const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id); try { - await dockerApi('POST', `/containers/${recorder.container_id}/stop`); - await dockerApi('DELETE', `/containers/${recorder.container_id}`); + if (isRemote) { + await fetch(`${targetNodeApiUrl}/sidecar/${recorder.container_id}`, { + method: 'DELETE', + signal: AbortSignal.timeout(10000), + }); + } else { + await dockerApi('POST', `/containers/${recorder.container_id}/stop`); + await dockerApi('DELETE', `/containers/${recorder.container_id}`); + } } catch (err) { console.error('Error stopping container during delete:', err); }