dragonflight/docker-compose.worker.yml

# Wild Dragon MAM — Worker Node Stack
# ─────────────────────────────────────
# Deploy on any machine you want to join the cluster as a worker.
# The primary stack (mam-api, db, redis) continues running on TrueNAS.
#
# Required env vars (set in .env.worker or export before running):
#   MAM_API_URL   URL of the primary MAM API  e.g. http://10.0.0.25:47432
#   NODE_TOKEN    Bearer token from the primary's Tokens page
#   NODE_IP       Host LAN IP to report (set by onboard-node.sh)
#
# Optional hardware overrides (if Docker can't see /dev directly):
#   GPU_COUNT     Number of NVIDIA GPUs on this node  (default: auto-detect from /dev/nvidia*)
#   BMD_COUNT     Number of Blackmagic DeckLink cards (default: auto-detect from /dev/blackmagic/)
#   BMD_MODEL     Marketed card name (e.g. "DeckLink Duo 2") — drives the port-diagram UI
#
# Optional env vars (needed only if starting the worker or capture profiles):
#   REDIS_URL, DATABASE_URL, S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY
#   BMD_DEVICE_0  DeckLink device path (default: /dev/blackmagic/dv0)
#                 (DeckLink IO / Quad cards expose /dev/blackmagic/io* instead — set BMD_DEVICE_PREFIX=io)
#   BMD_DEVICE_1  DeckLink device path (default: /dev/blackmagic/dv1)
#   BMD_DEVICE_PREFIX  Naming prefix for synthesized BMD_COUNT-based devices (default: dv). Use 'io' for IO/Quad.
#   LIVE_DIR      Host path for HLS live segments (default: /mnt/NVME/MAM/wild-dragon-live)
#
# Profiles:
#   (default)           node-agent only — cluster visibility + hardware heartbeat
#   --profile worker    + CPU/GPU job worker (proxy generation, transcoding)
#   --profile capture   + SDI capture service (requires Blackmagic DeckLink card)
#
# To enable GPU transcoding, also apply docker-compose.gpu.yml:
#   docker compose -f docker-compose.worker.yml -f docker-compose.gpu.yml --profile worker up -d
#
# NOTE: The node-agent mounts /var/run/docker.sock to spawn on-demand SDI
# capture sidecars when the primary mam-api routes a recorder to this node.
# Build the capture image before first use:
#   docker compose -f docker-compose.worker.yml build capture

services:

  # node-agent runs in host network mode so it can see the real host
  # interfaces, GPU devices and DeckLink cards without bridging tricks.
  # The reported IP / hostname will be the host's, not the container's.
  node-agent:
    build: ./services/node-agent
    restart: unless-stopped
    network_mode: host
    pid: host
    environment:
      MAM_API_URL:   ${MAM_API_URL}
      NODE_TOKEN:    ${NODE_TOKEN:-}
      NODE_ROLE:     ${NODE_ROLE:-worker}
      # NODE_NAME pins the cluster identity (heartbeat key). Set it per-node so
      # cloned VMs that share /etc/hostname don't collide on the same
      # cluster_nodes row. Falls back to the OS hostname when unset.
      NODE_NAME:     ${NODE_NAME:-}
      NODE_IP:       ${NODE_IP:-}
      AGENT_PORT:    ${AGENT_PORT:-7436}
      HEARTBEAT_MS:  ${HEARTBEAT_MS:-30000}
      GPU_COUNT:           ${GPU_COUNT:--1}
      BMD_COUNT:           ${BMD_COUNT:--1}
      BMD_MODEL:           ${BMD_MODEL:-}
      BMD_DEVICE_PREFIX:   ${BMD_DEVICE_PREFIX:-dv}
      LIVE_DIR:      ${LIVE_DIR:-/mnt/NVME/MAM/wild-dragon-live}
      # REPO_DIR: host path to the checked-out repo. The agent passes this to the
      # one-shot driver-install container so install-driver.sh can read
      # sdk/<vendor>/ and run deploy/install-driver.sh. Must match the host path
      # bind-mounted below (onboard-node.sh clones to /opt/wild-dragon).
      REPO_DIR:      ${REPO_DIR:-/opt/wild-dragon}
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /dev:/dev:ro
      - /mnt/NVME/MAM/wild-dragon-live:/mnt/NVME/MAM/wild-dragon-live:ro
      # Capture-driver deployment ("Capture Drivers / SDKs" in the Cluster admin
      # screen): the agent itself does NOT run dkms/modprobe — it spawns a
      # separate privileged ubuntu container that bind-mounts these host paths.
      # The agent only needs to *see* the repo path so it can pass it through as
      # a bind to that container; no extra privileges are granted to the agent.
      #   /opt/wild-dragon  → repo (sdk/<vendor>/ + deploy/install-driver.sh)
      # The install container additionally mounts /lib/modules,/usr/src,/boot,
      # /dev and /opt from the host (handled in the agent, not here) so DKMS /
      # modprobe / ldconfig affect the host kernel.
      - ${REPO_DIR:-/opt/wild-dragon}:${REPO_DIR:-/opt/wild-dragon}:ro
    devices:
      - /dev/blackmagic:/dev/blackmagic

  worker:
    build: ./services/worker
    profiles: [worker]
    restart: unless-stopped
    privileged: true
    environment:
      REDIS_URL:      ${REDIS_URL}
      DATABASE_URL:   ${DATABASE_URL}
      S3_ENDPOINT:    ${S3_ENDPOINT}
      S3_BUCKET:      ${S3_BUCKET}
      S3_ACCESS_KEY:  ${S3_ACCESS_KEY}
      S3_SECRET_KEY:  ${S3_SECRET_KEY}
      S3_REGION:      ${S3_REGION:-us-east-1}
      NVENC_ENABLED:  ${NVENC_ENABLED:-false}
      GROWING_PATH:   /growing
    networks:
      - wild-dragon-worker

  # SDI capture service — only start on nodes with Blackmagic DeckLink cards
  # Set BMD_DEVICE_0 in .env.worker to the actual device path, e.g. /dev/blackmagic/dv0
  capture:
    build: ./services/capture
    profiles: [capture]
    restart: unless-stopped
    runtime: nvidia
    environment:
      REDIS_URL:     ${REDIS_URL}
      DATABASE_URL:  ${DATABASE_URL}
      S3_ENDPOINT:   ${S3_ENDPOINT}
      S3_BUCKET:     ${S3_BUCKET}
      S3_ACCESS_KEY: ${S3_ACCESS_KEY}
      S3_SECRET_KEY: ${S3_SECRET_KEY}
      CAPTURE_PORT:  3001
      NVIDIA_VISIBLE_DEVICES:     all
      NVIDIA_DRIVER_CAPABILITIES: video,compute,utility
    devices:
      - ${BMD_DEVICE_0:-/dev/blackmagic/dv0}:/dev/blackmagic/dv0
      - ${BMD_DEVICE_1:-/dev/blackmagic/dv1}:/dev/blackmagic/dv1
    ports:
      - "${CAPTURE_PORT:-7437}:3001"
    networks:
      - wild-dragon-worker

  # worker-l4: HEAVY tier (proxy/conform/trim) on the L4 (NVENC). Talks to
  # zampp1's Redis/Postgres/S3 over the LAN (.200). No promotion scanner here.
  worker-l4:
    profiles: [gpu]
    build:
      context: ./services/worker
      dockerfile: Dockerfile.gpu
    image: wild-dragon-worker-gpu:latest
    runtime: nvidia
    restart: unless-stopped
    environment:
      REDIS_URL:     ${REDIS_URL}
      DATABASE_URL:  ${DATABASE_URL}
      S3_ENDPOINT:   ${S3_ENDPOINT}
      S3_BUCKET:     ${S3_BUCKET}
      S3_ACCESS_KEY: ${S3_ACCESS_KEY}
      S3_SECRET_KEY: ${S3_SECRET_KEY}
      S3_REGION:     ${S3_REGION:-us-east-1}
      WORKER_QUEUES: proxy,conform,trim
      PROXY_CONCURRENCY: "3"
      NVIDIA_VISIBLE_DEVICES: GPU-13acf439-8bf4-a5e0-7804-c1071bca547a
      WORKER_LABEL: "zampp2 / L4"
      NVIDIA_DRIVER_CAPABILITIES: video,compute,utility
    networks:
      - wild-dragon-worker

  # Framecache — shared memory ring buffer for SDI + network ingest fan-out.
  # Runs on every worker node that has capture sources (Blackmagic, Deltacast).
  # IPC host mode lets all capture sidecars share /dev/shm with this container.
  # FC_SHM_SIZE can be tuned per node in .env.worker:
  #   Baratheon (251GB RAM): FC_SHM_SIZE=64424509440  (60GB)
  #   zampp1    (93GB RAM):  FC_SHM_SIZE=42949672960  (40GB)
  #   zampp2    (18GB RAM):  FC_SHM_SIZE=8589934592   (8GB — increase RAM first)
  framecache:
    build: ./services/framecache
    profiles: [capture]
    restart: unless-stopped
    ipc: host
    shm_size: '${FC_SHM_SIZE_GB:-40}gb'
    environment:
      FC_PORT: 7435
    ports:
      - "7435:7435"
    volumes:
      - /dev/shm:/dev/shm
    networks:
      - wild-dragon-worker
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:7435/health"]
      interval: 10s
      timeout: 3s
      retries: 3
      start_period: 5s

networks:
  wild-dragon-worker:
    driver: bridge