From cf1fe136d0be0bca3d2855c174d7b5320204bffb Mon Sep 17 00:00:00 2001
From: zgaetano <Zgaetano@wilddragon.net>
Date: Fri, 29 May 2026 19:51:27 -0400
Subject: [PATCH] fix(worker): route import queue to a consumer + refresh stale
 yt-dlp

YouTube imports were stuck in `ingesting` forever: after the capability-
routing split (fdec2e3), every worker's WORKER_QUEUES enumerated an explicit
list that omitted `import`, so index.js's `want('import')` was false on every
worker and nothing consumed the import queue. Add `import` to worker-p4
(heavy/primary worker; import is concurrency-1 and network-bound, one consumer
is enough).

Second defect that would have surfaced once routed: the baked yt-dlp came from
the distro package (Ubuntu 22.04 apt = 2022.04.08, ~4 years stale), which
current YouTube rejects. Pull the latest self-contained yt-dlp release binary
at image build instead (GPU image: yt-dlp_linux; alpine image: python zipapp).
Rebuild the worker image to refresh.

Verified on zampp1: import jobs drain ingesting -> processing -> ready,
failed=0, yt-dlp 2026.03.17.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 docker-compose.yml             |  6 +++++-
 services/worker/Dockerfile     |  8 +++++++-
 services/worker/Dockerfile.gpu | 13 +++++++++++--
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 5733471..fce5ffb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -116,7 +116,11 @@ services:
       S3_SECRET_KEY: ${S3_SECRET_KEY}
       S3_REGION: ${S3_REGION:-us-east-1}
       GROWING_PATH: /growing
-      WORKER_QUEUES: proxy,conform,trim
+      # Includes `import` (YouTube importer): the import queue had no consumer
+      # after the capability-routing split, so import jobs sat unprocessed and
+      # assets stayed `ingesting` forever. import is concurrency-1 + network-
+      # bound, so one consumer (this heavy/primary worker) is sufficient.
+      WORKER_QUEUES: proxy,conform,trim,import
       RUN_PROMOTION: "true"
       PROXY_CONCURRENCY: "2"
       NVIDIA_VISIBLE_DEVICES: GPU-79afca3e-2ab2-a6ea-1c44-706c1f0a26d6
diff --git a/services/worker/Dockerfile b/services/worker/Dockerfile
index b6c5733..f4f1ae8 100644
--- a/services/worker/Dockerfile
+++ b/services/worker/Dockerfile
@@ -1,6 +1,12 @@
 FROM node:20-alpine
 # yt-dlp powers the YouTube importer; python3 is its runtime dep.
-RUN apk add --no-cache ffmpeg yt-dlp python3
+# Not from apk: the packaged yt-dlp goes stale and YouTube breaks old versions.
+# Pull the latest python-zipapp release (runs on musl via system python3) so a
+# rebuild always refreshes it. /usr/local/bin precedes /usr/bin on PATH.
+RUN apk add --no-cache ffmpeg python3 curl \
+    && curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp \
+         -o /usr/local/bin/yt-dlp \
+    && chmod a+rx /usr/local/bin/yt-dlp
 WORKDIR /app
 COPY package*.json ./
 RUN npm install --omit=dev
diff --git a/services/worker/Dockerfile.gpu b/services/worker/Dockerfile.gpu
index 3dd8466..56d122b 100644
--- a/services/worker/Dockerfile.gpu
+++ b/services/worker/Dockerfile.gpu
@@ -9,11 +9,20 @@
 FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04
 
 # Install Node.js 20, ffmpeg (Ubuntu's ffmpeg includes h264_nvenc/hevc_nvenc),
-# and yt-dlp (+ python3 runtime) for the YouTube importer.
+# and yt-dlp for the YouTube importer.
+#
+# yt-dlp is NOT installed from apt: Ubuntu 22.04's package is pinned to a 2022
+# release, which YouTube has long since broken (extraction fails). yt-dlp must
+# track YouTube's frequent changes, so we pull the latest self-contained
+# release binary at build time. /usr/local/bin precedes /usr/bin on PATH, so
+# `yt-dlp` resolves to this one. Rebuild the worker image to refresh it.
 RUN apt-get update && apt-get install -y --no-install-recommends \
-      curl ca-certificates ffmpeg yt-dlp python3 \
+      curl ca-certificates ffmpeg python3 \
     && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
+    && curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux \
+         -o /usr/local/bin/yt-dlp \
+    && chmod a+rx /usr/local/bin/yt-dlp \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app