diff --git a/services/mam-api/src/db/migrations/022-audio-metadata.sql b/services/mam-api/src/db/migrations/022-audio-metadata.sql new file mode 100644 index 0000000..b8f3cab --- /dev/null +++ b/services/mam-api/src/db/migrations/022-audio-metadata.sql @@ -0,0 +1,12 @@ +-- 022-audio-metadata.sql +-- Store per-track audio metadata extracted by ffprobe during proxy generation. +-- Shape: JSON array of objects, one per audio stream, e.g.: +-- [ +-- {"index":1,"codec":"pcm_s24le","channels":2,"channel_layout":"stereo", +-- "sample_rate":48000,"bit_depth":24,"bit_rate":2304000,"language":null}, +-- {"index":2,"codec":"aac","channels":2,"channel_layout":"stereo", +-- "sample_rate":48000,"bit_depth":null,"bit_rate":128000,"language":"en"} +-- ] +-- NULL means the asset has not been probed yet or has no audio streams. + +ALTER TABLE assets ADD COLUMN IF NOT EXISTS audio_metadata JSONB; diff --git a/services/mam-api/src/routes/assets.js b/services/mam-api/src/routes/assets.js index 27ccfba..a815245 100644 --- a/services/mam-api/src/routes/assets.js +++ b/services/mam-api/src/routes/assets.js @@ -840,4 +840,26 @@ router.get('/temp-segment-url/:clipInstanceId', async (req, res, next) => { } catch (err) { next(err); } }); +// GET /:id/audio — return audio metadata and a signed URL for audio extraction +router.get('/:id/audio', async (req, res, next) => { + try { + const { id } = req.params; + const r = await pool.query( + 'SELECT id, media_type, proxy_s3_key, original_s3_key, audio_metadata, duration_ms FROM assets WHERE id = $1', + [id] + ); + if (r.rows.length === 0) return res.status(404).json({ error: 'Asset not found' }); + const a = r.rows[0]; + const audioMeta = a.audio_metadata || null; + if (!audioMeta || !Array.isArray(audioMeta) || audioMeta.length === 0) { + return res.json({ tracks: [], hasAudio: false }); + } + res.json({ + tracks: audioMeta, + hasAudio: true, + durationMs: a.duration_ms || null, + }); + } catch (err) { next(err); } +}); + export default router; diff --git a/services/web-ui/public/screens-asset.jsx b/services/web-ui/public/screens-asset.jsx index 6e594af..b0d279b 100644 --- a/services/web-ui/public/screens-asset.jsx +++ b/services/web-ui/public/screens-asset.jsx @@ -927,16 +927,27 @@ function FilesTab({ asset, filmFrames, filmstripLoading, streamUrl, reprocessing } function MetadataTab({ asset }) { - const rows = [ - { k: "Filename", v: asset.name }, - { k: "Duration", v: asset.duration || '—' }, + var rows = [ + { k: "Filename", v: asset.name }, + { k: "Duration", v: asset.duration || '—' }, { k: "Resolution", v: asset.res || '—' }, - { k: "Codec", v: asset.codec || '—' }, - { k: "File size", v: asset.size || '—' }, - { k: "Status", v: asset.status || '—' }, - { k: "Updated", v: asset.updated || '—' }, - { k: "Project", v: asset.project || '—' }, + { k: "Codec", v: asset.codec || '—' }, + { k: "File size", v: asset.size || '—' }, + { k: "Status", v: asset.status || '—' }, + { k: "Updated", v: asset.updated || '—' }, + { k: "Project", v: asset.project || '—' }, ]; + var audioMeta = asset.audio_metadata; + if (audioMeta && Array.isArray(audioMeta) && audioMeta.length > 0) { + rows.push({ k: "Audio tracks", v: audioMeta.length }); + audioMeta.forEach(function(tr, i) { + var label = tr.title || ('Track ' + (tr.index != null ? tr.index : i + 1)); + var ch = tr.channel_layout || (tr.channels ? (tr.channels === 1 ? 'mono' : tr.channels === 2 ? 'stereo' : tr.channels + 'ch') : '—'); + var parts = [tr.codec || '—', ch, tr.sample_rate ? (tr.sample_rate / 1000).toFixed(1) + ' kHz' : '—', tr.bit_depth ? tr.bit_depth + '-bit' : '—', tr.bit_rate ? Math.round(tr.bit_rate / 1000) + ' kbps' : '—']; + if (tr.language) parts.push(tr.language); + rows.push({ k: " " + label, v: parts.join(' · ') }); + }); + } return (
{rows.map(function(r) { @@ -951,13 +962,226 @@ function MetadataTab({ asset }) { ); } +var _AUDIO_TRACK_COLORS = [ + 'var(--accent)', + 'var(--purple)', + 'var(--success)', + 'var(--warning)', + 'var(--danger)', + '#6B7280', + '#F472B6', + '#34D399', +]; + function AudioTab({ asset }) { - return ( -
-
- - + var assetId = asset && asset.id; + var [tracks, setTracks] = React.useState([]); + var [loading, setLoading] = React.useState(true); + var [masterVol, setMasterVol] = React.useState(100); + var [trackState, setTrackState] = React.useState({}); + var [levels, setLevels] = React.useState({}); + var analyserRef = React.useRef(null); + var rafRef = React.useRef(null); + var audioCtxRef = React.useRef(null); + var sourceRef = React.useRef(null); + + React.useEffect(function() { + if (!assetId) return; + var cancelled = false; + setLoading(true); + window.ZAMPP_API.fetch('/assets/' + assetId + '/audio') + .then(function(r) { + if (cancelled) return; + var t = (r && r.tracks) || []; + setTracks(t); + var initial = {}; + t.forEach(function(tr, i) { + initial[i] = { muted: false, solo: false, volume: 100 }; + }); + setTrackState(initial); + }) + .catch(function() { setTracks([]); }) + .finally(function() { if (!cancelled) setLoading(false); }); + return function() { cancelled = true; }; + }, [assetId]); + + var toggleMute = function(i) { + setTrackState(function(prev) { + var s = Object.assign({}, prev); + s[i] = Object.assign({}, s[i] || { muted: false, solo: false, volume: 100 }); + s[i].muted = !s[i].muted; + return s; + }); + }; + + var toggleSolo = function(i) { + setTrackState(function(prev) { + var s = Object.assign({}, prev); + s[i] = Object.assign({}, s[i] || { muted: false, solo: false, volume: 100 }); + s[i].solo = !s[i].solo; + return s; + }); + }; + + var setTrackVol = function(i, v) { + setTrackState(function(prev) { + var s = Object.assign({}, prev); + s[i] = Object.assign({}, s[i] || { muted: false, solo: false, volume: 100 }); + s[i].volume = v; + return s; + }); + }; + + var anySolo = tracks.some(function(_, i) { return trackState[i] && trackState[i].solo; }); + + React.useEffect(function() { + var video = document.querySelector('.player-canvas video'); + if (!video || !window.AudioContext) return; + var ctx, source, analyser, leftData, rightData; + try { + ctx = new (window.AudioContext || window.webkitAudioContext)(); + source = ctx.createMediaElementSource(video); + analyser = ctx.createAnalyser(); + analyser.fftSize = 256; + analyser.smoothingTimeConstant = 0.8; + var splitter = ctx.createChannelSplitter(2); + source.connect(analyser); + source.connect(ctx.destination); + analyser.connect(splitter); + var leftAnalyser = ctx.createAnalyser(); + leftAnalyser.fftSize = 256; + leftAnalyser.smoothingTimeConstant = 0.8; + var rightAnalyser = ctx.createAnalyser(); + rightAnalyser.fftSize = 256; + rightAnalyser.smoothingTimeConstant = 0.8; + splitter.connect(leftAnalyser, 0); + splitter.connect(rightAnalyser, 1); + audioCtxRef.current = ctx; + sourceRef.current = source; + analyserRef.current = { left: leftAnalyser, right: rightAnalyser }; + leftData = new Uint8Array(leftAnalyser.frequencyBinCount); + rightData = new Uint8Array(rightAnalyser.frequencyBinCount); + } catch (e) { + return; + } + var running = true; + var tick = function() { + if (!running) return; + if (analyserRef.current) { + analyserRef.current.left.getByteFrequencyData(leftData); + analyserRef.current.right.getByteFrequencyData(rightData); + var lAvg = 0, rAvg = 0; + for (var j = 0; j < leftData.length; j++) { lAvg += leftData[j]; rAvg += rightData[j]; } + lAvg = lAvg / leftData.length / 255; + rAvg = rAvg / rightData.length / 255; + setLevels({ L: lAvg, R: rAvg }); + } + rafRef.current = requestAnimationFrame(tick); + }; + tick(); + return function() { + running = false; + if (rafRef.current) cancelAnimationFrame(rafRef.current); + try { ctx.close(); } catch (_) {} + }; + }, [assetId, tracks.length]); + + if (loading) { + return
Loading audio…
; + } + + if (!tracks.length) { + return ( +
+ + No audio tracks found. Either this asset has no audio, or it has not been probed yet.
+ ); + } + + return ( +
+
+ {tracks.map(function(tr, i) { + var st = trackState[i] || { muted: false, solo: false, volume: 100 }; + var isAudible = st.muted ? false : (anySolo ? st.solo : true); + var color = _AUDIO_TRACK_COLORS[i % _AUDIO_TRACK_COLORS.length]; + var chLabel = tr.channel_layout || (tr.channels ? (tr.channels === 1 ? 'mono' : tr.channels === 2 ? 'stereo' : tr.channels + 'ch') : '—'); + var trackName = tr.title || ('Track ' + (tr.index != null ? tr.index : i + 1)); + var langTag = tr.language ? {tr.language} : null; + var codecLabel = tr.codec || '—'; + var srLabel = tr.sample_rate ? (tr.sample_rate / 1000).toFixed(1) + ' kHz' : '—'; + var bdLabel = tr.bit_depth ? tr.bit_depth + '-bit' : '—'; + var brLabel = tr.bit_rate ? Math.round(tr.bit_rate / 1000) + ' kbps' : '—'; + + return ( +
+
+ + {trackName} + {langTag} +
+
+ +
+
+ + +
+
+ {codecLabel} + {chLabel} + {srLabel} + {bdLabel} + {brLabel} +
+
+ + +
+ + {st.volume} +
+
+
+ ); + })} +
+
+
Master
+
+ + +
+
+ + {masterVol} +
+
+
+ ); +} + +function AudioLevelMeter({ level, label, tall }) { + var segs = tall ? 28 : 16; + var pct = Math.max(0, Math.min(1, level)); + return ( +
+
+ {Array.from({ length: segs }).map(function(_, i) { + var v = i / segs; + var on = v < pct; + var color = v < 0.6 ? 'var(--success)' : v < 0.85 ? 'var(--warning)' : 'var(--danger)'; + return
; + })} +
+ {label}
); } diff --git a/services/web-ui/public/styles-asset.css b/services/web-ui/public/styles-asset.css index 21daf8d..09f3dbe 100644 --- a/services/web-ui/public/styles-asset.css +++ b/services/web-ui/public/styles-asset.css @@ -388,3 +388,215 @@ } .meta-row .meta-k { color: var(--text-3); } .meta-row .meta-v { font-family: var(--font-mono); font-size: 12px; word-break: break-all; } + +/* ========== Audio tab ========== */ +.audio-tab { + display: flex; + flex-direction: column; + gap: 12px; + padding: 12px 0; +} + +.audio-tracks { + display: flex; + flex-direction: column; + gap: 6px; +} + +.audio-track { + display: grid; + grid-template-columns: 140px 1fr 64px auto auto; + align-items: center; + gap: 10px; + padding: 8px 12px; + background: var(--bg-2); + border: 1px solid var(--border); + border-radius: var(--r-md); + transition: opacity 120ms; +} + +.audio-track.muted { + opacity: 0.4; +} + +.audio-track-label { + display: flex; + align-items: center; + gap: 6px; + font-size: 12px; + min-width: 0; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} + +.audio-track-waveform { + min-width: 0; + overflow: hidden; + height: 32px; +} + +.audio-track-waveform .waveform { + width: 100%; + height: 32px; + display: block; +} + +.audio-track-meters { + display: flex; + gap: 4px; + justify-content: center; +} + +.audio-track-meta { + display: flex; + gap: 8px; + font-size: 10.5px; + color: var(--text-3); + white-space: nowrap; +} + +.audio-track-controls { + display: flex; + align-items: center; + gap: 6px; +} + +.audio-btn { + width: 22px; + height: 22px; + border-radius: 3px; + border: 1px solid var(--border); + background: var(--bg-3); + color: var(--text-3); + font-size: 10px; + font-weight: 700; + cursor: pointer; + display: grid; + place-items: center; + transition: background 80ms, color 80ms, border-color 80ms; +} + +.audio-btn:hover { + border-color: var(--border-stronger); + color: var(--text-1); +} + +.audio-btn.mute.active { + background: var(--danger); + color: white; + border-color: var(--danger); +} + +.audio-btn.solo.active { + background: var(--warning); + color: var(--bg-0); + border-color: var(--warning); +} + +.audio-fader { + display: flex; + align-items: center; + gap: 4px; +} + +.audio-fader input[type="range"] { + width: 60px; + height: 4px; + appearance: none; + background: var(--bg-3); + border-radius: 2px; + outline: none; + cursor: pointer; +} + +.audio-fader input[type="range"]::-webkit-slider-thumb { + appearance: none; + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--text-1); + border: 1px solid var(--border-strong); + cursor: pointer; +} + +.audio-fader input[type="range"]::-moz-range-thumb { + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--text-1); + border: 1px solid var(--border-strong); + cursor: pointer; +} + +.audio-fader .mono { + font-size: 10px; + color: var(--text-4); + min-width: 24px; + text-align: right; +} + +.audio-master { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 12px; + background: var(--bg-1); + border: 1px solid var(--border); + border-radius: var(--r-md); +} + +.audio-master-label { + font-size: 12.5px; + font-weight: 600; + color: var(--text-2); + min-width: 48px; +} + +.audio-master-meters { + display: flex; + gap: 6px; +} + +.master-fader input[type="range"] { + width: 100px; +} + +/* Audio level meters */ +.audio-level-meter { + display: flex; + flex-direction: column; + align-items: center; + gap: 2px; +} + +.audio-level-bar { + display: flex; + flex-direction: column-reverse; + gap: 1px; + width: 10px; +} + +.audio-level-seg { + width: 10px; + height: 3px; + border-radius: 1px; + transition: background 60ms; +} + +.audio-level-label { + font-size: 9px; + color: var(--text-4); + text-align: center; +} + +/* Responsive: stack on narrow screens */ +@media (max-width: 900px) { + .audio-track { + grid-template-columns: 1fr; + gap: 6px; + padding: 10px; + } + .audio-track-waveform { height: 24px; } + .audio-track-meters { justify-content: flex-start; } +} diff --git a/services/worker/src/ffmpeg/executor.js b/services/worker/src/ffmpeg/executor.js index ace835b..ca2dbd1 100644 --- a/services/worker/src/ffmpeg/executor.js +++ b/services/worker/src/ffmpeg/executor.js @@ -62,15 +62,37 @@ export const getMediaInfo = async (inputPath) => { if (den > 0) fps = Math.round((num / den) * 1000) / 1000; } - const hasAudio = (info.streams || []).some(s => s.codec_type === 'audio'); + const audioStreams = (info.streams || []).filter(s => s.codec_type === 'audio'); + const hasAudio = audioStreams.length > 0; + + const audioMetadata = audioStreams.map(s => { + const bitDepth = s.bits_per_raw_sample + ? parseInt(s.bits_per_raw_sample, 10) + : s.bit_depth + ? parseInt(s.bit_depth, 10) + : null; + return { + index: s.index ?? 0, + codec: s.codec_name || null, + channels: s.channels ? parseInt(s.channels, 10) : null, + channel_layout: s.channel_layout || null, + sample_rate: s.sample_rate ? parseInt(s.sample_rate, 10) : null, + bit_depth: bitDepth, + bit_rate: s.bit_rate ? parseInt(s.bit_rate, 10) : null, + language: s.tags?.language || null, + title: s.tags?.title || null, + disposition: s.disposition || {}, + }; + }); return { fps, - codec: videoStream?.codec_name || null, - resolution: videoStream ? `${videoStream.width}x${videoStream.height}` : null, - durationMs: fmt.duration ? Math.round(parseFloat(fmt.duration) * 1000) : null, - fileSizeBytes: fmt.size ? parseInt(fmt.size, 10) : null, + codec: videoStream?.codec_name || null, + resolution: videoStream ? `${videoStream.width}x${videoStream.height}` : null, + durationMs: fmt.duration ? Math.round(parseFloat(fmt.duration) * 1000) : null, + fileSizeBytes: fmt.size ? parseInt(fmt.size, 10) : null, hasAudio, + audioMetadata: audioMetadata.length > 0 ? audioMetadata : null, }; }; diff --git a/services/worker/src/workers/proxy.js b/services/worker/src/workers/proxy.js index d0d4d3f..6eb377a 100644 --- a/services/worker/src/workers/proxy.js +++ b/services/worker/src/workers/proxy.js @@ -200,26 +200,28 @@ export const proxyWorker = async (job) => { // Update asset record — store extracted metadata + proxy key // Use COALESCE so we never overwrite fields the capture service already set await job.updateProgress(90); - await query( - `UPDATE assets - SET proxy_s3_key = $1, - fps = COALESCE($2, fps), - codec = COALESCE($3, codec), - resolution = COALESCE($4, resolution), - duration_ms = COALESCE($5, duration_ms), - file_size = COALESCE($6, file_size), - updated_at = NOW() - WHERE id = $7`, - [ - outputKey, - mediaInfo.fps ?? null, - mediaInfo.codec ?? null, - mediaInfo.resolution ?? null, - mediaInfo.durationMs ?? null, - mediaInfo.fileSizeBytes ?? null, - assetId, - ] - ); + await query( + `UPDATE assets + SET proxy_s3_key = $1, + fps = COALESCE($2, fps), + codec = COALESCE($3, codec), + resolution = COALESCE($4, resolution), + duration_ms = COALESCE($5, duration_ms), + file_size = COALESCE($6, file_size), + audio_metadata = COALESCE($8, audio_metadata), + updated_at = NOW() + WHERE id = $7`, + [ + outputKey, + mediaInfo.fps ?? null, + mediaInfo.codec ?? null, + mediaInfo.resolution ?? null, + mediaInfo.durationMs ?? null, + mediaInfo.fileSizeBytes ?? null, + assetId, + mediaInfo.audioMetadata ? JSON.stringify(mediaInfo.audioMetadata) : null, + ] + ); // Now proxy exists in S3 — safe to queue thumbnail generation const thumbnailKey = `thumbnails/${assetId}.jpg`;