From 27a868aa5cfc78b3dd9d65b00e0c126ed16b98e1 Mon Sep 17 00:00:00 2001
From: Zac Gaetano <Zgaetano@wilddragon.net>
Date: Sun, 31 May 2026 13:55:14 -0400
Subject: [PATCH] fix(playout): clean video-only HLS preview via standalone
 ffmpeg re-mux

CasparCG's bundled FFMPEG/HLS consumer muxes a broken audio track
(aac sample_rate=0, time_base 1/0) into the preview, and silently drops
every arg that would remove it (-an, -codec:a, -g, -r all "Unused
option"). That corrupt audio black-screens the browser preview because
neither ffmpeg nor hls.js can decode the playlist.

Re-architect the preview path: CasparCG now STREAMs plain mpegts to a
UDP loopback port, and a Node-spawned STANDALONE ffmpeg (where -an
actually works) re-muxes it to clean, video-only HLS with -c:v copy.
The child process is tracked, auto-respawned while running, and killed
in stopChannel(). The PRIMARY SRT/RTMP/SDI/NDI output (with program
audio) is untouched.

Also fix the Dockerfile to match the working image: ubuntu:22.04 base +
CasparCG 2.4.0 ubuntu22 zip + NodeSource Node 20, and add a standalone
ffmpeg CLI. The old 2.3.3 tarball URL 404s. entrypoint.sh updated for
the 2.4.x bin/casparcg layout + bundled lib/ LD_LIBRARY_PATH.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 services/playout/Dockerfile             |  60 ++++++----
 services/playout/entrypoint.sh          |  34 ++++--
 services/playout/src/playout-manager.js | 145 ++++++++++++++++++------
 3 files changed, 172 insertions(+), 67 deletions(-)

diff --git a/services/playout/Dockerfile b/services/playout/Dockerfile
index 9b53179..f93bcba 100644
--- a/services/playout/Dockerfile
+++ b/services/playout/Dockerfile
@@ -6,36 +6,58 @@
 # --privileged by mam-api (same as capture) so DeckLink / NDI hardware is
 # reachable when present.
 #
+# CasparCG 2.4.x no longer ships a self-contained Linux tarball — the GitHub
+# release provides either Ubuntu .deb packages or an "ubuntu22" zip that bundles
+# the binary + its .so files under bin/ and lib/. We use the zip on an
+# ubuntu:22.04 base so the bundled libs match the host glibc/abi, then install
+# Node 20 from NodeSource on top.
+#
 # NDI + DeckLink SDKs are NOT redistributable. They are fetched at build time
-# from URLs supplied as build args (mirror them into your own artifact store);
-# the build still succeeds without them (NDI/DeckLink consumers simply won't be
+# from a URL supplied as a build arg (mirror it into your own artifact store);
+# the build still succeeds without it (NDI/DeckLink consumers simply won't be
 # available — SRT/RTMP/test output still work).
 
-FROM node:20-bookworm
+FROM ubuntu:22.04
 
-ARG CASPAR_VERSION=2.3.3-stable
-ARG CASPAR_URL=https://github.com/CasparCG/server/releases/download/v2.3.3-stable/CasparCG-Server-2.3.3-stable-Linux.tar.gz
+ARG CASPAR_VERSION=2.4.0-stable
+ARG CASPAR_URL=https://github.com/CasparCG/server/releases/download/v2.4.0-stable/casparcg-server-v2.4.0-stable-ubuntu22.zip
 ARG NDI_SDK_URL=
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# CasparCG 2.3 Linux runtime deps + Xvfb for headless GL + ffmpeg libs for the
-# FFMPEG consumer (SRT/RTMP output).
+# CasparCG 2.4 runtime deps + Xvfb for headless GL + CEF (HTML producer) deps +
+# Node 20 (NodeSource) + a STANDALONE ffmpeg CLI. The standalone ffmpeg is what
+# the Node shim spawns to re-mux the CasparCG mpegts preview stream into clean,
+# video-only HLS (CasparCG's own FFMPEG consumer silently drops -an and muxes a
+# broken audio track, which black-screens the browser preview).
 RUN apt-get update && apt-get install -y --no-install-recommends \
-      ca-certificates curl tar xz-utils \
-      xvfb libgl1-mesa-glx libgl1-mesa-dri libglu1-mesa \
-      libx11-6 libxext6 libxrandr2 libxcursor1 libxinerama1 libxi6 \
-      libopenal1 libsndfile1 libavformat59 libavcodec59 libavfilter8 \
-      libswscale6 libswresample4 libpostproc56 fonts-dejavu-core \
+      ca-certificates curl unzip tar xz-utils gnupg ffmpeg \
+      xvfb libgl1-mesa-dri libglu1-mesa fonts-dejavu-core \
+      libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
+      libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \
+      libgbm1 libpango-1.0-0 libcairo2 libasound2 libatspi2.0-0 \
+  && mkdir -p /etc/apt/keyrings \
+  && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+       | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+  && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" \
+       > /etc/apt/sources.list.d/nodesource.list \
+  && apt-get update && apt-get install -y --no-install-recommends nodejs \
   && rm -rf /var/lib/apt/lists/*
 
-# ── CasparCG Server ──────────────────────────────────────────────────────────
-WORKDIR /opt
-RUN curl -fsSL "$CASPAR_URL" -o caspar.tar.gz \
-  && mkdir -p /opt/casparcg \
-  && tar xzf caspar.tar.gz -C /opt/casparcg --strip-components=1 \
-  && rm caspar.tar.gz \
-  && (test -f /opt/casparcg/casparcg || test -f /opt/casparcg/CasparCG\ Server || true)
+# ── CasparCG Server (ubuntu22 zip bundle) ────────────────────────────────────
+# The zip extracts to /opt/casparcg_server with the binary at bin/casparcg and
+# its bundled .so files under lib/ (added to LD_LIBRARY_PATH by entrypoint.sh).
+# Symlink to /opt/casparcg so the config/entrypoint paths stay stable.
+WORKDIR /tmp/caspar
+RUN set -eux; \
+    curl -fsSL "$CASPAR_URL" -o caspar.zip; \
+    unzip -q caspar.zip -d /opt; \
+    chmod +x /opt/casparcg_server/bin/casparcg /opt/casparcg_server/scanner 2>/dev/null || true; \
+    ls /opt/casparcg_server/; \
+    test -x /opt/casparcg_server/bin/casparcg; \
+    ln -sfn /opt/casparcg_server /opt/casparcg; \
+    echo "caspar binary: /opt/casparcg_server/bin/casparcg"; \
+    cd /; rm -rf /tmp/caspar
 
 # ── NDI runtime (optional) ───────────────────────────────────────────────────
 # If an NDI SDK tarball URL is provided, extract its libs to /opt/ndi-lib and
diff --git a/services/playout/entrypoint.sh b/services/playout/entrypoint.sh
index 7f432d5..7cde7f1 100644
--- a/services/playout/entrypoint.sh
+++ b/services/playout/entrypoint.sh
@@ -7,29 +7,42 @@ if [ -z "${DISPLAY:-}" ]; then
   echo "[entrypoint] starting Xvfb on :99"
   Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
   export DISPLAY=:99
-  # Give Xvfb a moment to create the socket.
   for i in $(seq 1 20); do
     [ -e /tmp/.X11-unix/X99 ] && break
     sleep 0.25
   done
 fi
 
-# Ensure the HLS preview directory exists before CasparCG attaches its second
-# FFMPEG consumer (mam-api serves /live/<channel_id>/* from the shared volume).
+# Ensure the HLS preview directory exists before the re-mux ffmpeg writes to it
+# (mam-api serves /live/<channel_id>/* from the shared media volume).
 if [ -n "${CHANNEL_ID:-}" ]; then
   mkdir -p "/media/live/${CHANNEL_ID}"
 fi
 
-# Launch CasparCG Server from its install dir (it reads ./casparcg.config and
-# resolves relative media paths against the configured media folder).
+mkdir -p /media/casparcg/log /media/casparcg/data /media/templates
+
+# CEF (HTML producer) initialises an NSS database at /root/.pki/nssdb and
+# Chrome caches under HOME. Pre-create writable dirs so CEF doesn't SIGABRT
+# ~30s into the run when it first lazily inits.
+mkdir -p /root/.pki/nssdb /root/.cache /tmp/cef-cache
+chmod 700 /root/.pki/nssdb
+export HOME=/root
+
+# 2.4.x zip bundles its own .so files under lib/ — add to LD_LIBRARY_PATH.
+export LD_LIBRARY_PATH="/opt/casparcg/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
 cd /opt/casparcg
-CASPAR_BIN="./casparcg"
-[ -x "$CASPAR_BIN" ] || CASPAR_BIN="./CasparCG Server"
-echo "[entrypoint] launching CasparCG: $CASPAR_BIN"
-"$CASPAR_BIN" &
+CASPAR_CFG=/opt/casparcg/casparcg.config
+# 2.4.x: binary at bin/casparcg. 2.5.x: symlinked to casparcg at root.
+if [ -x "./bin/casparcg" ]; then CASPAR_BIN="./bin/casparcg";
+elif [ -x "./casparcg" ]; then CASPAR_BIN="./casparcg";
+elif [ -x "./CasparCG Server" ]; then CASPAR_BIN="./CasparCG Server";
+elif command -v casparcg >/dev/null; then CASPAR_BIN="casparcg";
+else echo "[entrypoint] ERROR: casparcg binary not found"; exit 1; fi
+echo "[entrypoint] launching CasparCG: $CASPAR_BIN $CASPAR_CFG"
+"$CASPAR_BIN" "$CASPAR_CFG" &
 CASPAR_PID=$!
 
-# Forward termination to CasparCG so the channel closes cleanly.
 term() {
   echo "[entrypoint] terminating CasparCG ($CASPAR_PID)"
   kill -TERM "$CASPAR_PID" 2>/dev/null || true
@@ -38,7 +51,6 @@ term() {
 }
 trap term SIGTERM SIGINT
 
-# Launch the Node control shim (foreground). If it exits, stop the container.
 cd /app
 node src/index.js &
 NODE_PID=$!
diff --git a/services/playout/src/playout-manager.js b/services/playout/src/playout-manager.js
index 32c4eda..bef98e4 100644
--- a/services/playout/src/playout-manager.js
+++ b/services/playout/src/playout-manager.js
@@ -1,4 +1,6 @@
 import { AmcpClient } from './amcp.js';
+import { spawn } from 'node:child_process';
+import { mkdirSync } from 'node:fs';
 
 // Playout manager — owns one CasparCG channel's lifecycle inside this sidecar.
 //
@@ -23,6 +25,12 @@ const MEDIA_ROOT = process.env.CASPAR_MEDIA_ROOT || '/media';
 const CHANNEL_ID = process.env.CHANNEL_ID || '';
 const HLS_DIR    = CHANNEL_ID ? `${MEDIA_ROOT}/live/${CHANNEL_ID}` : '';
 
+// Loopback UDP port CasparCG's preview STREAM consumer publishes mpegts to, and
+// the standalone ffmpeg re-muxer reads from. One CasparCG per sidecar, so a
+// fixed port is fine; allow override for parallel local testing.
+const PREVIEW_UDP_PORT = parseInt(process.env.PREVIEW_UDP_PORT || '9710', 10);
+const PREVIEW_UDP_URL  = `udp://127.0.0.1:${PREVIEW_UDP_PORT}`;
+
 // CasparCG SEEK / LENGTH are in frames, not seconds. Capture standard is 59.94;
 // SD/film modes need their own values. Default 60000/1001 matches both
 // '1080p5994' and '1080i5994'.
@@ -77,6 +85,8 @@ export class PlayoutManager {
       lastError: null,
     };
     this._advanceTimer = null;
+    this._hlsProc = null;     // standalone ffmpeg re-mux child process
+    this._hlsRestartTimer = null;
   }
 
   async _consumerCommand(outputType, cfg) {
@@ -158,50 +168,111 @@ export class PlayoutManager {
     return this.getStatus();
   }
 
-  // Low-bitrate HLS for the web UI preview. Segments land in the shared media
-  // volume; the mam-api serves /media/live/<channel_id>/* from there.
+  // HLS preview for the web UI confidence monitor.
+  //
+  // ── Why not CasparCG's own HLS (FILE/STREAM ".../index.m3u8") ──────────────
+  // CasparCG's bundled FFMPEG consumer muxes a BROKEN audio track into the HLS:
+  // ffprobe reports `aac, sample_rate=0` and ffmpeg decoding the playlist fails
+  // with "Invalid data ... abuffer: Value inf for parameter 'time_base' ...
+  // time_base 1/0". That corrupt audio prevents BOTH ffmpeg and hls.js from
+  // decoding, so the browser <video> sits at readyState 0 and the preview stays
+  // black. The video track itself is perfectly clean h264. Critically, the
+  // consumer IGNORES every arg that would fix it — `-an`, `-codec:a`, `-g`,
+  // `-r`, `-force_key_frames` are all silently dropped ("Unused option"), so we
+  // CANNOT remove the audio from inside CasparCG.
+  //
+  // ── The fix: STREAM mpegts to UDP loopback, re-mux with a STANDALONE ffmpeg ─
+  // CasparCG outputs a plain mpegts elementary stream to a local UDP port (its
+  // STREAM/mpegts path is fine — the breakage is specific to its HLS muxer). A
+  // Node-spawned standalone ffmpeg (where `-an` actually works) reads that UDP
+  // stream, drops audio, copies the clean h264 video, and writes proper HLS.
+  // `-c:v copy` avoids re-encoding. The program audio is untouched — it rides
+  // the PRIMARY SRT/RTMP/SDI/NDI consumer, which we never modify.
   async _addHlsConsumer() {
-    // The CasparCG channel feeds this consumer in real time, and its frame
-    // timestamps are irregular ("packet with pts X has duration 0" warnings).
-    // With frame-count GOPs (-g 60) the HLS muxer split points drift, producing
-    // erratic segment durations (0.4s–4.2s) and TARGETDURATION violations. The
-    // result is a live playlist hls.js parses but can never sync to — it
-    // reloads forever ("sliding 0.00 / prev-sn na / MISSED") and never appends
-    // a fragment, so the preview stays black.
-    //
-    // The HLS preview is a VIDEO-ONLY confidence monitor. We deliberately drop
-    // audio (-an).
-    //
-    // Why: CasparCG's real-time channel feeds the FFMPEG consumer an audio
-    // stream whose muxed time_base comes out as 1/0 (infinity). ffmpeg itself
-    // can't decode the resulting playlist ("Invalid data ... abuffer: Value inf
-    // for parameter 'time_base'"), and hls.js silently fails to append the
-    // fragment after demux — the video element sits at readyState 0 and the
-    // preview stays black. Dropping audio removes the broken stream entirely;
-    // the remaining video track is clean h264 and plays in hls.js. A confidence
-    // monitor doesn't need audio — the real program audio rides the primary
-    // SRT/RTMP/SDI/NDI output, which is unaffected.
-    //
-    // NOTE: encoder options like -g / -r / -force_key_frames are NOT honored
-    // here — CasparCG's FFMPEG consumer applies args to the muxer, not the
-    // encoder (it logs "Unused option"). Segment cadence follows the channel's
-    // own keyframes; that's fine for a video-only preview.
-    const out = `${HLS_DIR}/index.m3u8`;
-    const args = [
-      `FILE "${out}"`,
-      '-format hls',
-      '-hls_time 2',
-      '-hls_list_size 8',
-      '-hls_flags delete_segments+append_list+independent_segments',
-      '-codec:v libx264 -preset:v veryfast -tune:v zerolatency -b:v 800k -maxrate 1M -bufsize 2M',
-      '-an',
+    // 1) CasparCG → mpegts over UDP loopback. The channel feeds RGBA, so a
+    //    format=yuv420p filter is required before libx264.
+    const streamArgs = [
+      `STREAM "${PREVIEW_UDP_URL}?pkt_size=1316"`,
+      '-format mpegts',
+      '-codec:v libx264 -preset:v veryfast -tune:v zerolatency',
+      '-b:v 2M -maxrate 2M -bufsize 4M',
+      '-codec:a aac -b:a 96k',
       '-filter:v format=yuv420p',
     ].join(' ');
-    await this.amcp.send(`ADD ${CHANNEL} ${args}`);
+    await this.amcp.send(`ADD ${CHANNEL} ${streamArgs}`);
+
+    // 2) Standalone ffmpeg re-mux: UDP mpegts → clean video-only HLS.
+    this._startHlsRemux();
+  }
+
+  // Spawn (or respawn) the standalone ffmpeg that re-muxes the loopback mpegts
+  // into video-only HLS. Restarts automatically if it dies while the channel is
+  // still running (e.g. brief UDP gap before CasparCG's consumer is up).
+  _startHlsRemux() {
+    if (!HLS_DIR) return;
+    try { mkdirSync(HLS_DIR, { recursive: true }); } catch (_) {}
+    this._stopHlsRemux();
+
+    const out = `${HLS_DIR}/index.m3u8`;
+    const args = [
+      '-hide_banner', '-loglevel', 'warning',
+      // Read the live mpegts loopback. genpts rebuilds timestamps; the analyze/
+      // probe sizes are kept small so playback starts promptly.
+      '-fflags', '+genpts',
+      '-analyzeduration', '2000000', '-probesize', '2000000',
+      '-i', `${PREVIEW_UDP_URL}?fifo_size=1000000&overrun_nonfatal=1`,
+      // Drop the (broken) audio entirely; copy the clean video bitstream.
+      '-an',
+      '-c:v', 'copy',
+      '-f', 'hls',
+      '-hls_time', '2',
+      '-hls_list_size', '8',
+      '-hls_flags', 'delete_segments+append_list+independent_segments',
+      '-hls_segment_filename', `${HLS_DIR}/index%d.ts`,
+      out,
+    ];
+    const proc = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] });
+    this._hlsProc = proc;
+    proc.stderr.on('data', (d) => {
+      const line = d.toString().trim();
+      if (line) console.warn(`[playout][hls-ffmpeg] ${line}`);
+    });
+    proc.on('exit', (code, signal) => {
+      console.warn(`[playout] HLS re-mux ffmpeg exited code=${code} signal=${signal}`);
+      if (this._hlsProc === proc) this._hlsProc = null;
+      // Auto-respawn while the channel is running (and we didn't kill it).
+      if (this.state.running && signal !== 'SIGTERM' && signal !== 'SIGKILL') {
+        this._hlsRestartTimer = setTimeout(() => {
+          this._hlsRestartTimer = null;
+          if (this.state.running) {
+            console.log('[playout] respawning HLS re-mux ffmpeg');
+            this._startHlsRemux();
+          }
+        }, 1000);
+      }
+    });
+    proc.on('error', (err) => {
+      console.warn(`[playout] HLS re-mux ffmpeg spawn error: ${err.message}`);
+    });
+    console.log(`[playout] HLS re-mux ffmpeg started: ${PREVIEW_UDP_URL} -> ${out}`);
+  }
+
+  _stopHlsRemux() {
+    if (this._hlsRestartTimer) {
+      clearTimeout(this._hlsRestartTimer);
+      this._hlsRestartTimer = null;
+    }
+    if (this._hlsProc) {
+      const proc = this._hlsProc;
+      this._hlsProc = null;
+      try { proc.kill('SIGTERM'); } catch (_) {}
+    }
   }
 
   async stopChannel() {
     this._clearAdvance();
+    this.state.running = false;   // set first so the ffmpeg exit handler won't respawn
+    this._stopHlsRemux();
     try { await this.amcp.send(`STOP ${CHANNEL}-${FG_LAYER}`); } catch (_) {}
     try { await this.amcp.send(`CLEAR ${CHANNEL}`); } catch (_) {}
     this.state.running = false;