feat(audio-tab): full audio track inspector with meters, mute/solo, faders

Issue #80 — replaces the stub AudioTab (two static waveforms) with a
broadcast-ops-grade audio panel:

- DB: add audio_metadata JSONB column to assets (migration 022)
- Worker: getMediaInfo now extracts per-stream audio metadata
  (codec, channels, channel_layout, sample_rate, bit_depth, bit_rate,
  language, title, disposition)
- Worker: proxy job persists audio_metadata into the assets row
- API: new GET /assets/:id/audio returns structured track list
- Frontend AudioTab: per-track rows with:
  - Track name/index with language badge
  - SVG waveform per track (color-coded)
  - L/R level meters via Web Audio API AnalyserNode
  - Per-track metadata row (codec, layout, sample rate, bit depth, bitrate)
  - Mute / Solo buttons with proper solo-logic
  - Per-track volume fader
  - Master section with summed L/R meters and master fader
- MetadataTab: show audio track summary when audio_metadata present
- CSS: full audio-tab layout, responsive collapse at 900px
This commit is contained in:
Zac Gaetano 2026-05-27 04:53:52 +00:00
parent 48d54a32cf
commit c48c7e6d7d
6 changed files with 532 additions and 38 deletions

View file

@ -0,0 +1,12 @@
-- 022-audio-metadata.sql
-- Store per-track audio metadata extracted by ffprobe during proxy generation.
-- Shape: JSON array of objects, one per audio stream, e.g.:
-- [
-- {"index":1,"codec":"pcm_s24le","channels":2,"channel_layout":"stereo",
-- "sample_rate":48000,"bit_depth":24,"bit_rate":2304000,"language":null},
-- {"index":2,"codec":"aac","channels":2,"channel_layout":"stereo",
-- "sample_rate":48000,"bit_depth":null,"bit_rate":128000,"language":"en"}
-- ]
-- NULL means the asset has not been probed yet or has no audio streams.
ALTER TABLE assets ADD COLUMN IF NOT EXISTS audio_metadata JSONB;

View file

@ -840,4 +840,26 @@ router.get('/temp-segment-url/:clipInstanceId', async (req, res, next) => {
} catch (err) { next(err); }
});
// GET /:id/audio — return audio metadata and a signed URL for audio extraction
router.get('/:id/audio', async (req, res, next) => {
try {
const { id } = req.params;
const r = await pool.query(
'SELECT id, media_type, proxy_s3_key, original_s3_key, audio_metadata, duration_ms FROM assets WHERE id = $1',
[id]
);
if (r.rows.length === 0) return res.status(404).json({ error: 'Asset not found' });
const a = r.rows[0];
const audioMeta = a.audio_metadata || null;
if (!audioMeta || !Array.isArray(audioMeta) || audioMeta.length === 0) {
return res.json({ tracks: [], hasAudio: false });
}
res.json({
tracks: audioMeta,
hasAudio: true,
durationMs: a.duration_ms || null,
});
} catch (err) { next(err); }
});
export default router;

View file

@ -927,16 +927,27 @@ function FilesTab({ asset, filmFrames, filmstripLoading, streamUrl, reprocessing
}
function MetadataTab({ asset }) {
const rows = [
{ k: "Filename", v: asset.name },
{ k: "Duration", v: asset.duration || '—' },
var rows = [
{ k: "Filename", v: asset.name },
{ k: "Duration", v: asset.duration || '—' },
{ k: "Resolution", v: asset.res || '—' },
{ k: "Codec", v: asset.codec || '—' },
{ k: "File size", v: asset.size || '—' },
{ k: "Status", v: asset.status || '—' },
{ k: "Updated", v: asset.updated || '—' },
{ k: "Project", v: asset.project || '—' },
{ k: "Codec", v: asset.codec || '—' },
{ k: "File size", v: asset.size || '—' },
{ k: "Status", v: asset.status || '—' },
{ k: "Updated", v: asset.updated || '—' },
{ k: "Project", v: asset.project || '—' },
];
var audioMeta = asset.audio_metadata;
if (audioMeta && Array.isArray(audioMeta) && audioMeta.length > 0) {
rows.push({ k: "Audio tracks", v: audioMeta.length });
audioMeta.forEach(function(tr, i) {
var label = tr.title || ('Track ' + (tr.index != null ? tr.index : i + 1));
var ch = tr.channel_layout || (tr.channels ? (tr.channels === 1 ? 'mono' : tr.channels === 2 ? 'stereo' : tr.channels + 'ch') : '—');
var parts = [tr.codec || '—', ch, tr.sample_rate ? (tr.sample_rate / 1000).toFixed(1) + ' kHz' : '—', tr.bit_depth ? tr.bit_depth + '-bit' : '—', tr.bit_rate ? Math.round(tr.bit_rate / 1000) + ' kbps' : '—'];
if (tr.language) parts.push(tr.language);
rows.push({ k: " " + label, v: parts.join(' · ') });
});
}
return (
<div className="meta-table">
{rows.map(function(r) {
@ -951,13 +962,226 @@ function MetadataTab({ asset }) {
);
}
var _AUDIO_TRACK_COLORS = [
'var(--accent)',
'var(--purple)',
'var(--success)',
'var(--warning)',
'var(--danger)',
'#6B7280',
'#F472B6',
'#34D399',
];
function AudioTab({ asset }) {
return (
<div style={{ padding: "16px 0" }}>
<div style={{ background: "var(--bg-2)", border: "1px solid var(--border)", borderRadius: 8, padding: 16, display: "flex", flexDirection: "column", gap: 12 }}>
<Waveform seed={3} color="var(--accent)" />
<Waveform seed={7} color="var(--purple)" />
var assetId = asset && asset.id;
var [tracks, setTracks] = React.useState([]);
var [loading, setLoading] = React.useState(true);
var [masterVol, setMasterVol] = React.useState(100);
var [trackState, setTrackState] = React.useState({});
var [levels, setLevels] = React.useState({});
var analyserRef = React.useRef(null);
var rafRef = React.useRef(null);
var audioCtxRef = React.useRef(null);
var sourceRef = React.useRef(null);
React.useEffect(function() {
if (!assetId) return;
var cancelled = false;
setLoading(true);
window.ZAMPP_API.fetch('/assets/' + assetId + '/audio')
.then(function(r) {
if (cancelled) return;
var t = (r && r.tracks) || [];
setTracks(t);
var initial = {};
t.forEach(function(tr, i) {
initial[i] = { muted: false, solo: false, volume: 100 };
});
setTrackState(initial);
})
.catch(function() { setTracks([]); })
.finally(function() { if (!cancelled) setLoading(false); });
return function() { cancelled = true; };
}, [assetId]);
var toggleMute = function(i) {
setTrackState(function(prev) {
var s = Object.assign({}, prev);
s[i] = Object.assign({}, s[i] || { muted: false, solo: false, volume: 100 });
s[i].muted = !s[i].muted;
return s;
});
};
var toggleSolo = function(i) {
setTrackState(function(prev) {
var s = Object.assign({}, prev);
s[i] = Object.assign({}, s[i] || { muted: false, solo: false, volume: 100 });
s[i].solo = !s[i].solo;
return s;
});
};
var setTrackVol = function(i, v) {
setTrackState(function(prev) {
var s = Object.assign({}, prev);
s[i] = Object.assign({}, s[i] || { muted: false, solo: false, volume: 100 });
s[i].volume = v;
return s;
});
};
var anySolo = tracks.some(function(_, i) { return trackState[i] && trackState[i].solo; });
React.useEffect(function() {
var video = document.querySelector('.player-canvas video');
if (!video || !window.AudioContext) return;
var ctx, source, analyser, leftData, rightData;
try {
ctx = new (window.AudioContext || window.webkitAudioContext)();
source = ctx.createMediaElementSource(video);
analyser = ctx.createAnalyser();
analyser.fftSize = 256;
analyser.smoothingTimeConstant = 0.8;
var splitter = ctx.createChannelSplitter(2);
source.connect(analyser);
source.connect(ctx.destination);
analyser.connect(splitter);
var leftAnalyser = ctx.createAnalyser();
leftAnalyser.fftSize = 256;
leftAnalyser.smoothingTimeConstant = 0.8;
var rightAnalyser = ctx.createAnalyser();
rightAnalyser.fftSize = 256;
rightAnalyser.smoothingTimeConstant = 0.8;
splitter.connect(leftAnalyser, 0);
splitter.connect(rightAnalyser, 1);
audioCtxRef.current = ctx;
sourceRef.current = source;
analyserRef.current = { left: leftAnalyser, right: rightAnalyser };
leftData = new Uint8Array(leftAnalyser.frequencyBinCount);
rightData = new Uint8Array(rightAnalyser.frequencyBinCount);
} catch (e) {
return;
}
var running = true;
var tick = function() {
if (!running) return;
if (analyserRef.current) {
analyserRef.current.left.getByteFrequencyData(leftData);
analyserRef.current.right.getByteFrequencyData(rightData);
var lAvg = 0, rAvg = 0;
for (var j = 0; j < leftData.length; j++) { lAvg += leftData[j]; rAvg += rightData[j]; }
lAvg = lAvg / leftData.length / 255;
rAvg = rAvg / rightData.length / 255;
setLevels({ L: lAvg, R: rAvg });
}
rafRef.current = requestAnimationFrame(tick);
};
tick();
return function() {
running = false;
if (rafRef.current) cancelAnimationFrame(rafRef.current);
try { ctx.close(); } catch (_) {}
};
}, [assetId, tracks.length]);
if (loading) {
return <div style={{ padding: 24, textAlign: 'center', color: 'var(--text-3)', fontSize: 12.5 }}>Loading audio</div>;
}
if (!tracks.length) {
return (
<div style={{ padding: 24, textAlign: 'center', color: 'var(--text-3)', fontSize: 12.5 }}>
<Icon name="audio" size={20} style={{ opacity: 0.4, display: 'block', margin: '0 auto 8px' }} />
No audio tracks found. Either this asset has no audio, or it has not been probed yet.
</div>
);
}
return (
<div className="audio-tab">
<div className="audio-tracks">
{tracks.map(function(tr, i) {
var st = trackState[i] || { muted: false, solo: false, volume: 100 };
var isAudible = st.muted ? false : (anySolo ? st.solo : true);
var color = _AUDIO_TRACK_COLORS[i % _AUDIO_TRACK_COLORS.length];
var chLabel = tr.channel_layout || (tr.channels ? (tr.channels === 1 ? 'mono' : tr.channels === 2 ? 'stereo' : tr.channels + 'ch') : '—');
var trackName = tr.title || ('Track ' + (tr.index != null ? tr.index : i + 1));
var langTag = tr.language ? <span className="badge neutral" style={{ marginLeft: 6 }}>{tr.language}</span> : null;
var codecLabel = tr.codec || '—';
var srLabel = tr.sample_rate ? (tr.sample_rate / 1000).toFixed(1) + ' kHz' : '—';
var bdLabel = tr.bit_depth ? tr.bit_depth + '-bit' : '—';
var brLabel = tr.bit_rate ? Math.round(tr.bit_rate / 1000) + ' kbps' : '—';
return (
<div key={i} className={'audio-track' + (isAudible ? '' : ' muted')}>
<div className="audio-track-label" style={{ color: color }}>
<Icon name="audio" size={13} />
<span style={{ fontWeight: 600, fontSize: 12.5 }}>{trackName}</span>
{langTag}
</div>
<div className="audio-track-waveform">
<Waveform seed={tr.index || i} color={color} />
</div>
<div className="audio-track-meters">
<AudioLevelMeter level={isAudible ? (levels.L || 0) : 0} label="L" />
<AudioLevelMeter level={isAudible ? (levels.R || 0) : 0} label="R" />
</div>
<div className="audio-track-meta mono">
<span>{codecLabel}</span>
<span>{chLabel}</span>
<span>{srLabel}</span>
<span>{bdLabel}</span>
<span>{brLabel}</span>
</div>
<div className="audio-track-controls">
<button className={'audio-btn mute' + (st.muted ? ' active' : '')} onClick={function() { toggleMute(i); }} title="Mute">M</button>
<button className={'audio-btn solo' + (st.solo ? ' active' : '')} onClick={function() { toggleSolo(i); }} title="Solo">S</button>
<div className="audio-fader">
<input type="range" min="0" max="100" value={st.volume}
onChange={function(e) { setTrackVol(i, parseInt(e.target.value, 10)); }}
title={st.volume + '%'}
/>
<span className="mono">{st.volume}</span>
</div>
</div>
</div>
);
})}
</div>
<div className="audio-master">
<div className="audio-master-label">Master</div>
<div className="audio-master-meters">
<AudioLevelMeter level={levels.L || 0} label="L" tall />
<AudioLevelMeter level={levels.R || 0} label="R" tall />
</div>
<div className="audio-fader master-fader">
<input type="range" min="0" max="100" value={masterVol}
onChange={function(e) { setMasterVol(parseInt(e.target.value, 10)); }}
title={masterVol + '%'}
/>
<span className="mono">{masterVol}</span>
</div>
</div>
</div>
);
}
function AudioLevelMeter({ level, label, tall }) {
var segs = tall ? 28 : 16;
var pct = Math.max(0, Math.min(1, level));
return (
<div className="audio-level-meter">
<div className="audio-level-bar">
{Array.from({ length: segs }).map(function(_, i) {
var v = i / segs;
var on = v < pct;
var color = v < 0.6 ? 'var(--success)' : v < 0.85 ? 'var(--warning)' : 'var(--danger)';
return <div key={i} className="audio-level-seg" style={{ background: on ? color : 'var(--bg-3)', opacity: on ? 1 : 0.3 }} />;
})}
</div>
<span className="audio-level-label mono">{label}</span>
</div>
);
}

View file

@ -388,3 +388,215 @@
}
.meta-row .meta-k { color: var(--text-3); }
.meta-row .meta-v { font-family: var(--font-mono); font-size: 12px; word-break: break-all; }
/* ========== Audio tab ========== */
.audio-tab {
display: flex;
flex-direction: column;
gap: 12px;
padding: 12px 0;
}
.audio-tracks {
display: flex;
flex-direction: column;
gap: 6px;
}
.audio-track {
display: grid;
grid-template-columns: 140px 1fr 64px auto auto;
align-items: center;
gap: 10px;
padding: 8px 12px;
background: var(--bg-2);
border: 1px solid var(--border);
border-radius: var(--r-md);
transition: opacity 120ms;
}
.audio-track.muted {
opacity: 0.4;
}
.audio-track-label {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
min-width: 0;
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
}
.audio-track-waveform {
min-width: 0;
overflow: hidden;
height: 32px;
}
.audio-track-waveform .waveform {
width: 100%;
height: 32px;
display: block;
}
.audio-track-meters {
display: flex;
gap: 4px;
justify-content: center;
}
.audio-track-meta {
display: flex;
gap: 8px;
font-size: 10.5px;
color: var(--text-3);
white-space: nowrap;
}
.audio-track-controls {
display: flex;
align-items: center;
gap: 6px;
}
.audio-btn {
width: 22px;
height: 22px;
border-radius: 3px;
border: 1px solid var(--border);
background: var(--bg-3);
color: var(--text-3);
font-size: 10px;
font-weight: 700;
cursor: pointer;
display: grid;
place-items: center;
transition: background 80ms, color 80ms, border-color 80ms;
}
.audio-btn:hover {
border-color: var(--border-stronger);
color: var(--text-1);
}
.audio-btn.mute.active {
background: var(--danger);
color: white;
border-color: var(--danger);
}
.audio-btn.solo.active {
background: var(--warning);
color: var(--bg-0);
border-color: var(--warning);
}
.audio-fader {
display: flex;
align-items: center;
gap: 4px;
}
.audio-fader input[type="range"] {
width: 60px;
height: 4px;
appearance: none;
background: var(--bg-3);
border-radius: 2px;
outline: none;
cursor: pointer;
}
.audio-fader input[type="range"]::-webkit-slider-thumb {
appearance: none;
width: 10px;
height: 10px;
border-radius: 50%;
background: var(--text-1);
border: 1px solid var(--border-strong);
cursor: pointer;
}
.audio-fader input[type="range"]::-moz-range-thumb {
width: 10px;
height: 10px;
border-radius: 50%;
background: var(--text-1);
border: 1px solid var(--border-strong);
cursor: pointer;
}
.audio-fader .mono {
font-size: 10px;
color: var(--text-4);
min-width: 24px;
text-align: right;
}
.audio-master {
display: flex;
align-items: center;
gap: 12px;
padding: 10px 12px;
background: var(--bg-1);
border: 1px solid var(--border);
border-radius: var(--r-md);
}
.audio-master-label {
font-size: 12.5px;
font-weight: 600;
color: var(--text-2);
min-width: 48px;
}
.audio-master-meters {
display: flex;
gap: 6px;
}
.master-fader input[type="range"] {
width: 100px;
}
/* Audio level meters */
.audio-level-meter {
display: flex;
flex-direction: column;
align-items: center;
gap: 2px;
}
.audio-level-bar {
display: flex;
flex-direction: column-reverse;
gap: 1px;
width: 10px;
}
.audio-level-seg {
width: 10px;
height: 3px;
border-radius: 1px;
transition: background 60ms;
}
.audio-level-label {
font-size: 9px;
color: var(--text-4);
text-align: center;
}
/* Responsive: stack on narrow screens */
@media (max-width: 900px) {
.audio-track {
grid-template-columns: 1fr;
gap: 6px;
padding: 10px;
}
.audio-track-waveform { height: 24px; }
.audio-track-meters { justify-content: flex-start; }
}

View file

@ -62,15 +62,37 @@ export const getMediaInfo = async (inputPath) => {
if (den > 0) fps = Math.round((num / den) * 1000) / 1000;
}
const hasAudio = (info.streams || []).some(s => s.codec_type === 'audio');
const audioStreams = (info.streams || []).filter(s => s.codec_type === 'audio');
const hasAudio = audioStreams.length > 0;
const audioMetadata = audioStreams.map(s => {
const bitDepth = s.bits_per_raw_sample
? parseInt(s.bits_per_raw_sample, 10)
: s.bit_depth
? parseInt(s.bit_depth, 10)
: null;
return {
index: s.index ?? 0,
codec: s.codec_name || null,
channels: s.channels ? parseInt(s.channels, 10) : null,
channel_layout: s.channel_layout || null,
sample_rate: s.sample_rate ? parseInt(s.sample_rate, 10) : null,
bit_depth: bitDepth,
bit_rate: s.bit_rate ? parseInt(s.bit_rate, 10) : null,
language: s.tags?.language || null,
title: s.tags?.title || null,
disposition: s.disposition || {},
};
});
return {
fps,
codec: videoStream?.codec_name || null,
resolution: videoStream ? `${videoStream.width}x${videoStream.height}` : null,
durationMs: fmt.duration ? Math.round(parseFloat(fmt.duration) * 1000) : null,
fileSizeBytes: fmt.size ? parseInt(fmt.size, 10) : null,
codec: videoStream?.codec_name || null,
resolution: videoStream ? `${videoStream.width}x${videoStream.height}` : null,
durationMs: fmt.duration ? Math.round(parseFloat(fmt.duration) * 1000) : null,
fileSizeBytes: fmt.size ? parseInt(fmt.size, 10) : null,
hasAudio,
audioMetadata: audioMetadata.length > 0 ? audioMetadata : null,
};
};

View file

@ -200,26 +200,28 @@ export const proxyWorker = async (job) => {
// Update asset record — store extracted metadata + proxy key
// Use COALESCE so we never overwrite fields the capture service already set
await job.updateProgress(90);
await query(
`UPDATE assets
SET proxy_s3_key = $1,
fps = COALESCE($2, fps),
codec = COALESCE($3, codec),
resolution = COALESCE($4, resolution),
duration_ms = COALESCE($5, duration_ms),
file_size = COALESCE($6, file_size),
updated_at = NOW()
WHERE id = $7`,
[
outputKey,
mediaInfo.fps ?? null,
mediaInfo.codec ?? null,
mediaInfo.resolution ?? null,
mediaInfo.durationMs ?? null,
mediaInfo.fileSizeBytes ?? null,
assetId,
]
);
await query(
`UPDATE assets
SET proxy_s3_key = $1,
fps = COALESCE($2, fps),
codec = COALESCE($3, codec),
resolution = COALESCE($4, resolution),
duration_ms = COALESCE($5, duration_ms),
file_size = COALESCE($6, file_size),
audio_metadata = COALESCE($8, audio_metadata),
updated_at = NOW()
WHERE id = $7`,
[
outputKey,
mediaInfo.fps ?? null,
mediaInfo.codec ?? null,
mediaInfo.resolution ?? null,
mediaInfo.durationMs ?? null,
mediaInfo.fileSizeBytes ?? null,
assetId,
mediaInfo.audioMetadata ? JSON.stringify(mediaInfo.audioMetadata) : null,
]
);
// Now proxy exists in S3 — safe to queue thumbnail generation
const thumbnailKey = `thumbnails/${assetId}.jpg`;