fix(decklink): mount /dev/blackmagic in sidecar + remote node routing via node-agent

Two bugs fixed:
1. SDI capture sidecar never had /dev/blackmagic bound — ffmpeg opened the
   decklink input inside a container with no device nodes, so frame=0.
   Fix: local spawns now push '/dev/blackmagic:/dev/blackmagic' onto Binds
   when source_type='sdi'.

2. recorders.js always spawned sidecars against the local Docker socket
   (zampp1), even when a recorder's node_id pointed at zampp2 (where the
   card is). Fix: resolveNodeTarget() looks up the recorder's cluster node;
   if it's a different hostname the sidecar is spawned via a new
   POST /sidecar/start endpoint on the remote node-agent.

node-agent gains three new routes (all talk to the local Docker socket):
  POST   /sidecar/start         — create + start container (host network,
                                   privileged, /dev/blackmagic bind for sdi)
  DELETE /sidecar/:id           — stop + remove
  GET    /sidecar/:id/status    — inspect + poll capture service

docker-compose.worker.yml: add /var/run/docker.sock and LIVE_DIR to
node-agent so it can spawn sidecars, and document build-capture prerequisite.: recorders.js

services/mam-api/src/routes/recorders.js

@@ -8,6 +8,10 @@ const router = express.Router();
 
 router.use(requireAuth);
 
+// Base port for on-demand SDI sidecar containers on remote worker nodes.
+// Device index 0 → 7438, index 1 → 7439, etc.
+const SIDECAR_PORT_BASE = 7438;
+
 // Docker API helper function
 function dockerApi(method, path, body = null) {
   return new Promise((resolve, reject) => {
@@ -34,6 +38,22 @@ function dockerApi(method, path, body = null) {
   });
 }
 
+// Look up the cluster node for a recorder and decide if it is remote.
+// Returns { remote: false } when the node is local or unset;
+// { remote: true, apiUrl, ip } when it is a different host.
+async function resolveNodeTarget(nodeId) {
+  if (!nodeId) return { remote: false };
+  const r = await pool.query(
+    'SELECT hostname, ip_address, api_url FROM cluster_nodes WHERE id = $1',
+    [nodeId]
+  );
+  if (r.rows.length === 0) return { remote: false };
+  const node = r.rows[0];
+  const localHostname = process.env.NODE_HOSTNAME || '';
+  if (!node.api_url || node.hostname === localHostname) return { remote: false };
+  return { remote: true, apiUrl: node.api_url, ip: node.ip_address };
+}
+
 // Helper function to generate clip name with timestamp
 function generateClipName(recorderName) {
   const now = new Date();
@@ -284,8 +304,6 @@ router.post('/:id/start', async (req, res, next) => {
     const sourceType = recorder.source_type;
     const deviceIndex = recorder.device_index ?? sourceConfig.device ?? 0;
 
-    const { portBindings, exposedPorts } = buildPortConfig(sourceType, sourceConfig);
-
     // Build container env — all codec controls flow through here.
     const env = [
       `S3_ENDPOINT=${s3Endpoint}`,
@@ -337,42 +355,68 @@ router.post('/:id/start', async (req, res, next) => {
       }
     }
 
-    const alias = `recorder-${id}`;
+    // Determine whether to spawn locally or via a remote node-agent.
-    const containerConfig = {
+    const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id);
-      Image: 'wild-dragon-capture:latest',
+
-      Env: env,
+    let containerId;
-      ExposedPorts: Object.keys(exposedPorts).length > 0 ? exposedPorts : undefined,
+
-      HostConfig: {
+    if (isRemote) {
-        Privileged: true,
+      // Remote node: delegate container lifecycle to that node's agent.
-        NetworkMode: dockerNetwork,
+      const capturePort = SIDECAR_PORT_BASE + (deviceIndex || 0);
-        PortBindings: Object.keys(portBindings).length > 0 ? portBindings : undefined,
+      const sidecarRes = await fetch(`${targetNodeApiUrl}/sidecar/start`, {
-        Binds: ['/mnt/NVME/MAM/wild-dragon-live:/live'],
+        method: 'POST',
-      },
+        headers: { 'Content-Type': 'application/json' },
-      NetworkingConfig: {
+        body: JSON.stringify({ image: 'wild-dragon-capture:latest', env, capturePort, sourceType }),
-        EndpointsConfig: {
+        signal: AbortSignal.timeout(15000),
-          [dockerNetwork]: { Aliases: [alias] },
+      });
+      if (!sidecarRes.ok) {
+        const details = await sidecarRes.json().catch(() => ({}));
+        return res.status(502).json({ error: 'Remote node failed to start sidecar', details });
+      }
+      const sidecarData = await sidecarRes.json();
+      containerId = sidecarData.containerId;
+    } else {
+      // Local spawn via Docker socket.
+      const { portBindings, exposedPorts } = buildPortConfig(sourceType, sourceConfig);
+      const alias = `recorder-${id}`;
+
+      const hostBinds = ['/mnt/NVME/MAM/wild-dragon-live:/live'];
+      if (sourceType === 'sdi') hostBinds.push('/dev/blackmagic:/dev/blackmagic');
+
+      const containerConfig = {
+        Image: 'wild-dragon-capture:latest',
+        Env: env,
+        ExposedPorts: Object.keys(exposedPorts).length > 0 ? exposedPorts : undefined,
+        HostConfig: {
+          Privileged: true,
+          NetworkMode: dockerNetwork,
+          PortBindings: Object.keys(portBindings).length > 0 ? portBindings : undefined,
+          Binds: hostBinds,
         },
-      },
+        NetworkingConfig: {
-      Hostname: alias,
+          EndpointsConfig: {
-    };
+            [dockerNetwork]: { Aliases: [alias] },
+          },
+        },
+        Hostname: alias,
+      };
 
-    const createRes = await dockerApi('POST', '/containers/create', containerConfig);
+      const createRes = await dockerApi('POST', '/containers/create', containerConfig);
+      if (createRes.status !== 201) {
+        return res.status(500).json({
+          error: 'Failed to create container',
+          details: createRes.data,
+        });
+      }
 
-    if (createRes.status !== 201) {
+      containerId = createRes.data.Id;
-      return res.status(500).json({
+      const startRes = await dockerApi('POST', `/containers/${containerId}/start`);
-        error: 'Failed to create container',
+      if (startRes.status !== 204) {
-        details: createRes.data,
+        return res.status(500).json({
-      });
+          error: 'Failed to start container',
-    }
+          details: startRes.data,
-
+        });
-    const containerId = createRes.data.Id;
+      }
-    const startRes = await dockerApi('POST', `/containers/${containerId}/start`);
-
-    if (startRes.status !== 204) {
-      return res.status(500).json({
-        error: 'Failed to start container',
-        details: startRes.data,
-      });
     }
 
     const updateResult = await pool.query(
@@ -409,28 +453,40 @@ router.post('/:id/stop', async (req, res, next) => {
       return res.status(400).json({ error: 'No container running' });
     }
 
-    const stopRes = await dockerApi(
+    const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id);
-      'POST',
-      `/containers/${recorder.container_id}/stop`
-    );
 
-    if (stopRes.status !== 204 && stopRes.status !== 304) {
+    if (isRemote) {
-      return res.status(500).json({
+      const stopRes = await fetch(`${targetNodeApiUrl}/sidecar/${recorder.container_id}`, {
-        error: 'Failed to stop container',
+        method: 'DELETE',
-        details: stopRes.data,
+        signal: AbortSignal.timeout(15000),
       });
-    }
+      if (!stopRes.ok && stopRes.status !== 404) {
+        return res.status(502).json({ error: 'Remote node failed to stop sidecar' });
+      }
+    } else {
+      const stopRes = await dockerApi(
+        'POST',
+        `/containers/${recorder.container_id}/stop`
+      );
 
-    const removeRes = await dockerApi(
+      if (stopRes.status !== 204 && stopRes.status !== 304) {
-      'DELETE',
+        return res.status(500).json({
-      `/containers/${recorder.container_id}`
+          error: 'Failed to stop container',
-    );
+          details: stopRes.data,
+        });
+      }
 
-    if (removeRes.status !== 204 && removeRes.status !== 404) {
+      const removeRes = await dockerApi(
-      return res.status(500).json({
+        'DELETE',
-        error: 'Failed to remove container',
+        `/containers/${recorder.container_id}`
-        details: removeRes.data,
+      );
-      });
+
+      if (removeRes.status !== 204 && removeRes.status !== 404) {
+        return res.status(500).json({
+          error: 'Failed to remove container',
+          details: removeRes.data,
+        });
+      }
     }
 
     const updateResult = await pool.query(
@@ -471,37 +527,59 @@ router.get('/:id/status', async (req, res, next) => {
       });
     }
 
-    const inspectRes = await dockerApi(
+    const deviceIndex = recorder.device_index ?? (recorder.source_config?.device ?? 0);
-      'GET',
+    const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id);
-      `/containers/${recorder.container_id}/json`
-    );
 
-    if (inspectRes.status !== 200) {
+    let isRunning = false;
-      return res.json({
+    let duration = 0;
-        status: 'unknown',
+    let signal = 'connecting';
-        duration: 0,
-        containerId: recorder.container_id,
-      });
-    }
-
-    const container = inspectRes.data;
-    const startedAt = new Date(container.State.StartedAt).getTime();
-    const now = Date.now();
-    const duration = Math.floor((now - startedAt) / 1000);
-
-    let signal = container.State.Running ? 'receiving' : 'stopped';
     let signalKnown = false;
     let live = null;
-    try {
+
-      const captureRes = await fetch(`http://recorder-${id}:3001/capture/status`, { signal: AbortSignal.timeout(2000) });
+    if (isRemote) {
-      if (captureRes.ok) {
+      try {
-        live = await captureRes.json();
+        const statusRes = await fetch(`${targetNodeApiUrl}/sidecar/${recorder.container_id}/status`, {
-        if (live && live.signal) { signal = live.signal; signalKnown = true; }
+          signal: AbortSignal.timeout(4000),
+        });
+        if (statusRes.ok) {
+          const data = await statusRes.json();
+          isRunning = data.running;
+          if (data.startedAt) {
+            duration = Math.floor((Date.now() - new Date(data.startedAt).getTime()) / 1000);
+          }
+          live = data.live;
+        }
+      } catch (_) { /* node unreachable */ }
+    } else {
+      const inspectRes = await dockerApi(
+        'GET',
+        `/containers/${recorder.container_id}/json`
+      );
+
+      if (inspectRes.status !== 200) {
+        return res.json({
+          status: 'unknown',
+          duration: 0,
+          containerId: recorder.container_id,
+        });
       }
-    } catch (_) { /* not ready yet */ }
+
+      const container = inspectRes.data;
+      isRunning = container.State.Running;
+      duration = Math.floor((Date.now() - new Date(container.State.StartedAt).getTime()) / 1000);
+
+      try {
+        const captureRes = await fetch(`http://recorder-${id}:3001/capture/status`, { signal: AbortSignal.timeout(2000) });
+        if (captureRes.ok) live = await captureRes.json();
+      } catch (_) { /* not ready yet */ }
+    }
+
+    if (isRunning) signal = 'receiving';
+    if (!isRunning) signal = 'stopped';
+    if (live && live.signal) { signal = live.signal; signalKnown = true; }
 
     res.json({
-      status: container.State.Running ? 'recording' : 'stopped',
+      status: isRunning ? 'recording' : 'stopped',
       duration,
       containerId: recorder.container_id,
       signal,
@@ -533,9 +611,17 @@ router.delete('/:id', async (req, res, next) => {
     const recorder = recorderResult.rows[0];
 
     if (recorder.container_id) {
+      const { remote: isRemote, apiUrl: targetNodeApiUrl } = await resolveNodeTarget(recorder.node_id);
       try {
-        await dockerApi('POST', `/containers/${recorder.container_id}/stop`);
+        if (isRemote) {
-        await dockerApi('DELETE', `/containers/${recorder.container_id}`);
+          await fetch(`${targetNodeApiUrl}/sidecar/${recorder.container_id}`, {
+            method: 'DELETE',
+            signal: AbortSignal.timeout(10000),
+          });
+        } else {
+          await dockerApi('POST', `/containers/${recorder.container_id}/stop`);
+          await dockerApi('DELETE', `/containers/${recorder.container_id}`);
+        }
       } catch (err) {
         console.error('Error stopping container during delete:', err);
       }