WildDragonLLC/dragonflight
1
0
Fork 0
Code Issues 32 Pull requests 7 Projects Releases Packages Wiki Activity Actions

fix(mam-api): remove stitchedS3Stream workaround — RustFS range bug fixed in beta.6 (#143)

Browse source
This commit is contained in:
Zac Gaetano 2026-06-03 16:05:47 +00:00
parent eeaa1c1b58
commit d654f7c8a1
1 changed files with 6 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
services/mam-api/src/routes/assets.js
View file

@ -858,65 +858,9 @@ router.get('/:id/live-path', async (req, res, next) => {
// - ETag + Last-Modified for conditional requests (304 on repeat visits)
// - Cache-Control: private, max-age=3600 so the browser caches segments
// and doesn't re-fetch them on every seek within a session
// Issue #143 — RustFS returns empty bodies for ranged GETs whose start offset
// is past ~5.9 MB on single-file proxy MP4s. Confirmed via direct S3 probe:
// HEAD reports correct size, full GET (`bytes=0-`) works perfectly, but
// `bytes=8179166-` returns 206 + the right Content-Range header and a zero-
// byte body. A streaming GET from 0 reads cleanly *through* the broken zone.
//
// Workaround until the proxy worker emits HLS (planned v1.2.1): stream the
// proxy from offset 0, skip bytes the client didn't ask for, stop after the
// requested end. Browser sees a normal 206 + Content-Range. Mem stays flat;
// extra RustFS-to-mam-api bandwidth = (end+1 - actual-range) per seek.
//
// Small head-of-file ranges below RUSTFS_RANGE_SAFE_START are handled by a
// direct ranged GET — saves the streaming-from-0 cost on the common case of
// initial moov + first-segment fetch.
async function* stitchedS3Stream(key, startByte, endByte) {
// Yields buffers covering exactly [startByte, endByte] inclusive.
//
// RustFS only mis-serves a ranged GET when the *start* offset of the
// request is past ~5.8 MB. So we pull the object in 4 MB windows whose
// START offsets always stay below the broken threshold:
// - We anchor every chunk's start at a multiple of RUSTFS_SAFE_CHUNK
// (0, 4 MB, 8 MB, …).
// - Wait — that puts later starts past the threshold.
// Instead: skip directly to the chunk containing `startByte`, but request
// it as `bytes=anchorStart-end` where anchorStart < threshold. Since the
// bug only bites when the *request start* offset is large, we never issue
// a single GET whose Range start is past the broken zone — we instead
// exploit that a low-offset GET that *continues past* the threshold reads
// cleanly (confirmed by the bytes=0- full-GET probe).
//
// Practically: one GET from 0 that streams up through endByte, dropping
// the bytes below startByte as they arrive. Memory stays flat; we pay
// (endByte+1) bytes of RustFS-to-mam-api bandwidth per request.
const res = await s3Client.send(new GetObjectCommand({
Bucket: getS3Bucket(),
Key: key,
Range: `bytes=0-${endByte}`,
}));
let consumed = 0; // bytes seen so far from S3
let totalEmitted = 0;
for await (const buf of res.Body) {
const bufStart = consumed; // file offset of buf[0]
const bufEnd = consumed + buf.length - 1;
consumed += buf.length;
if (bufEnd < startByte) continue; // entirely before window
const sliceFrom = Math.max(0, startByte - bufStart);
const sliceTo = Math.min(buf.length, endByte - bufStart + 1);
if (sliceTo > sliceFrom) {
yield buf.subarray(sliceFrom, sliceTo);
totalEmitted += sliceTo - sliceFrom;
}
if (bufEnd >= endByte) break;
}
if (totalEmitted === 0) {
throw new Error(`RustFS returned empty body for ${key} bytes=0-${endByte}`);
}
}
// RustFS issue #143 (empty body on ranged GETs past ~5.9 MB) was fixed in
// RustFS 1.0.0-alpha.94 (PR #2493). Standard ranged GETs used throughout.
router.get('/:id/video', async (req, res, next) => {
try {
@ -997,39 +941,11 @@ router.get('/:id/video', async (req, res, next) => {
if (etag) headers['ETag'] = etag;
if (lastModified) headers['Last-Modified'] = lastModified.toUTCString();
// For small head-of-file ranges (entirely below the broken threshold)
// a direct ranged GET works and saves the streaming-from-0 cost.
const RUSTFS_RANGE_SAFE_START = parseInt(process.env.RUSTFS_RANGE_SAFE_START || String(5_500_000), 10);
if (start < RUSTFS_RANGE_SAFE_START && end < RUSTFS_RANGE_SAFE_START) {
const s3Res = await s3Client.send(new GetObjectCommand({
Bucket: getS3Bucket(), Key: key, Range: `bytes=${start}-${end}`,
}));
res.writeHead(206, headers);
s3Res.Body.pipe(res);
return;
}
// Otherwise: stream from offset 0, drop bytes below `start`, stop at
// `end`. Browser sees a normal 206; mam-api stays memory-flat.
const s3Res = await s3Client.send(new GetObjectCommand({
Bucket: getS3Bucket(), Key: key, Range: `bytes=${start}-${end}`,
}));
res.writeHead(206, headers);
try {
for await (const buf of stitchedS3Stream(key, start, end)) {
// res.write returns false when backpressure builds — pause and wait.
if (!res.write(buf)) {
await new Promise(r => res.once('drain', r));
}
if (res.destroyed) return;
}
res.end();
} catch (err) {
console.error(`[video] stitch failed for ${key}:`, err.message);
if (!res.headersSent) {
res.writeHead(500, { 'Content-Type': 'text/plain', 'Cache-Control': 'no-store' });
res.end('Upstream storage error');
} else {
res.destroy(err);
}
}
s3Res.Body.pipe(res);
} catch (err) { next(err); }
});