feat(assets): SMB tag + always-available S3 migrate for growing masters
- Growing-file masters (.mxf) are tagged smb on create (while live) and on pending-migration; the tag swaps to s3 once promoted. - Migrate-to-S3 (promote) now accepts assets stuck in live (sidecar post-stop call never landed) in addition to pending_migration, guarded to .mxf SMB masters only. - Promotion queue added to the Jobs tab QUEUES so SMB->S3 migrations are visible/trackable like other jobs. - Library: SMB badge shows alongside LIVE for growing masters; Move to S3 shown for any SMB-origin asset (live-stuck or pending_migration). Verified: stuck-live 5.4GB master migrated SMB->S3, job tracked in Jobs tab, tags swapped smb->s3.
This commit is contained in:
OpenCode
parent
105d04729a
commit
5c07b4e8b1
4 changed files with 63 additions and 17 deletions
|
|
@ -203,13 +203,18 @@ router.post('/', async (req, res, next) => {
|
|||
id = uuidv4();
|
||||
const mediaType = (sourceType === 'audio') ? 'audio' : 'video';
|
||||
const assetStatus = status || 'processing';
|
||||
// Growing-file masters land on the SMB share as .mxf — tag them 'smb' from
|
||||
// the moment they're created (while still 'live') so the library shows the
|
||||
// SMB origin and can always offer the S3 migrate action.
|
||||
const isGrowingSmb = !!(hiresKey && /\.mxf$/i.test(hiresKey));
|
||||
const initialTags = isGrowingSmb ? ['smb'] : [];
|
||||
const ins = await pool.query(
|
||||
`INSERT INTO assets (
|
||||
id, project_id, bin_id,
|
||||
filename, display_name,
|
||||
status, media_type,
|
||||
original_s3_key, proxy_s3_key,
|
||||
duration_ms,
|
||||
duration_ms, tags,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
|
|
@ -217,7 +222,7 @@ router.post('/', async (req, res, next) => {
|
|||
$4, $4,
|
||||
$10, $9,
|
||||
$5, $6,
|
||||
$7,
|
||||
$7, $11,
|
||||
COALESCE($8::timestamptz, NOW()), NOW()
|
||||
)
|
||||
RETURNING *`,
|
||||
|
|
@ -229,6 +234,7 @@ router.post('/', async (req, res, next) => {
|
|||
capturedAt || null,
|
||||
mediaType,
|
||||
assetStatus,
|
||||
initialTags,
|
||||
]
|
||||
);
|
||||
asset = ins.rows[0];
|
||||
|
|
@ -533,41 +539,57 @@ router.post('/:id/pending-migration', requireAssetEdit, async (req, res, next) =
|
|||
`UPDATE assets
|
||||
SET status = 'pending_migration',
|
||||
duration_ms = COALESCE($2, duration_ms),
|
||||
-- Tag the growing master as living on SMB (in addition to its live
|
||||
-- origin) so the library can show it + offer the S3 migrate action.
|
||||
tags = (
|
||||
SELECT ARRAY(SELECT DISTINCT unnest(COALESCE(tags, '{}'::text[]) || ARRAY['smb']))
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
RETURNING *`,
|
||||
[id, durationMs]
|
||||
);
|
||||
|
||||
console.log(`[assets] set pending-migration status for asset ${id}`);
|
||||
console.log(`[assets] set pending-migration status (+smb tag) for asset ${id}`);
|
||||
res.json(upd.rows[0]);
|
||||
} catch (err) { next(err); }
|
||||
});
|
||||
|
||||
// POST /:id/promote
|
||||
// Promotes an asset from 'pending_migration' (SMB) to S3.
|
||||
// Enqueues a 'promotion' job in BullMQ to handle the S3 upload and metadata updates.
|
||||
// Promotes a growing-file / SMB master to S3.
|
||||
// Normally an asset is 'pending_migration' (flipped by the sidecar on a clean
|
||||
// growing-file stop). But a growing recording can get STUCK in 'live' if the
|
||||
// sidecar's post-stop /pending-migration call never lands (crash, network).
|
||||
// Operators must always be able to migrate those too, so we accept BOTH
|
||||
// 'pending_migration' and 'live' here. Enqueues a 'promotion' BullMQ job (which
|
||||
// shows in the Jobs tab) to handle the SMB→S3 upload + metadata updates.
|
||||
router.post('/:id/promote', requireAssetEdit, async (req, res, next) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
|
||||
const check = await pool.query(`SELECT status FROM assets WHERE id = $1`, [id]);
|
||||
const check = await pool.query(`SELECT status, original_s3_key FROM assets WHERE id = $1`, [id]);
|
||||
if (check.rows.length === 0) return res.status(404).json({ error: 'Asset not found' });
|
||||
const { status } = check.rows[0];
|
||||
const { status, original_s3_key } = check.rows[0];
|
||||
|
||||
if (status !== 'pending_migration') {
|
||||
return res.status(400).json({ error: `Asset status is "${status}" — only "pending_migration" assets can be promoted` });
|
||||
const MIGRATABLE = new Set(['pending_migration', 'live']);
|
||||
if (!MIGRATABLE.has(status)) {
|
||||
return res.status(400).json({ error: `Asset status is "${status}" — only growing-file (SMB) assets in "pending_migration" or "live" can be migrated to S3` });
|
||||
}
|
||||
// Guard: only growing-file masters live on SMB. A non-growing 'live' asset
|
||||
// (still recording, or a normal upload) has no SMB master to migrate.
|
||||
if (status === 'live' && !(original_s3_key && /\.mxf$/i.test(original_s3_key))) {
|
||||
return res.status(400).json({ error: 'This live asset is not a finished growing-file master on SMB.' });
|
||||
}
|
||||
|
||||
// Update status to 'processing' so it is locked
|
||||
// Lock it: 'processing' while the promotion job runs.
|
||||
await pool.query(
|
||||
`UPDATE assets SET status = 'processing', updated_at = NOW() WHERE id = $1`,
|
||||
[id]
|
||||
);
|
||||
|
||||
// Queue the promotion job in BullMQ
|
||||
// Queue the promotion job in BullMQ — listed in the Jobs tab (type 'promotion').
|
||||
await promotionQueue.add('promote', { assetId: id });
|
||||
console.log(`[assets] queued promotion for asset ${id}`);
|
||||
console.log(`[assets] queued promotion (SMB→S3) for asset ${id} (was ${status})`);
|
||||
|
||||
res.json({ ok: true, status: 'processing' });
|
||||
} catch (err) { next(err); }
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ const conformQueue = new Queue('conform', { connection: redisConn })
|
|||
const importQueue = new Queue('import', { connection: redisConn });
|
||||
const trimQueue = new Queue('trim', { connection: redisConn });
|
||||
const playoutStageQueue = new Queue('playout-stage', { connection: redisConn });
|
||||
const promotionQueue = new Queue('promotion', { connection: redisConn });
|
||||
|
||||
const QUEUES = [
|
||||
{ queue: proxyQueue, type: 'proxy' },
|
||||
|
|
@ -38,6 +39,8 @@ const QUEUES = [
|
|||
{ queue: importQueue, type: 'import' },
|
||||
{ queue: trimQueue, type: 'trim' },
|
||||
{ queue: playoutStageQueue, type: 'playout-stage' },
|
||||
// SMB→S3 migration of growing-file masters. Shows the migrate action in Jobs.
|
||||
{ queue: promotionQueue, type: 'promotion' },
|
||||
];
|
||||
|
||||
// BullMQ state → API status mapping
|
||||
|
|
|
|||
|
|
@ -643,9 +643,18 @@ function AssetContextMenu({ asset, x, y, bins, onClose, onChanged, onOpen, onRen
|
|||
{asset.original_s3_key && onDownload && (
|
||||
<button onClick={function() { onDownload(asset); }}><Icon name="download" size={11} />Download original…</button>
|
||||
)}
|
||||
{asset.status === 'pending_migration' && (
|
||||
<button onClick={promoteToS3}><Icon name="upload" size={11} />Move to S3</button>
|
||||
)}
|
||||
{(function() {
|
||||
// A growing-file master lives on the SMB share. Offer "Move to S3" for
|
||||
// any such asset — both the normal 'pending_migration' state AND a
|
||||
// recording that got stuck in 'live' (its post-stop migrate never fired).
|
||||
const onSmb = (asset.tags || []).indexOf('smb') !== -1
|
||||
|| /\.mxf$/i.test(asset.original_s3_key || '');
|
||||
const migratable = asset.status === 'pending_migration'
|
||||
|| (asset.status === 'live' && onSmb);
|
||||
return migratable
|
||||
? <button onClick={promoteToS3}><Icon name="upload" size={11} />Move to S3</button>
|
||||
: null;
|
||||
})()}
|
||||
<div className="ctx-divider" />
|
||||
{(bins && bins.length > 0) ? (
|
||||
<>
|
||||
|
|
@ -758,7 +767,11 @@ function AssetCard({ asset, onOpen, onContextMenu, onDownload, onDragStart, drag
|
|||
{asset.status === 'live' && <span className="badge live">LIVE</span>}
|
||||
{asset.status === 'processing' && <span className="badge warning">Processing</span>}
|
||||
{asset.status === 'error' && <span className="badge danger">Error</span>}
|
||||
{asset.status === 'pending_migration' && <span className="badge warning" style={{ background: '#e8821c', color: '#fff' }}>SMB</span>}
|
||||
{/* SMB badge for any growing-file master on the share — shown ALONGSIDE
|
||||
LIVE while it records, and on its own while pending migration. */}
|
||||
{(((asset.tags || []).indexOf('smb') !== -1 || /\.mxf$/i.test(asset.original_s3_key || ''))
|
||||
&& (asset.status === 'live' || asset.status === 'pending_migration')) &&
|
||||
<span className="badge warning" style={{ background: '#e8821c', color: '#fff' }}>SMB</span>}
|
||||
</div>
|
||||
{/* Hi-res download trigger: only shown when the asset has an
|
||||
original_s3_key (everything queued through ingest / conform).
|
||||
|
|
|
|||
|
|
@ -99,12 +99,20 @@ export const promotionWorker = async (job) => {
|
|||
console.log(`[promotion] promoting asset ${assetId}: uploading ${localPath} (${st.size} bytes) -> s3://${S3_BUCKET}/${s3Key}`);
|
||||
await uploadStreamToS3(S3_BUCKET, s3Key, createReadStream(localPath));
|
||||
|
||||
// 4. Update asset status to ready (with correct S3 key and size)
|
||||
// 4. Update asset status to ready (with correct S3 key and size).
|
||||
// Swap the 'smb' origin tag for 's3' now the master lives in S3.
|
||||
await query(
|
||||
`UPDATE assets
|
||||
SET original_s3_key = $1,
|
||||
file_size = $2,
|
||||
status = 'ready',
|
||||
tags = (
|
||||
SELECT ARRAY(
|
||||
SELECT DISTINCT unnest(
|
||||
array_remove(COALESCE(tags, '{}'::text[]), 'smb') || ARRAY['s3']
|
||||
)
|
||||
)
|
||||
),
|
||||
updated_at = NOW()
|
||||
WHERE id = $3`,
|
||||
[s3Key, st.size, assetId]
|
||||
|
|
|
|||
Loading…
Reference in a new issue