From d654f7c8a1ff1b108e096df42f1500f55ac0d9f1 Mon Sep 17 00:00:00 2001 From: ZGaetano Date: Wed, 3 Jun 2026 16:05:47 +0000 Subject: [PATCH] =?UTF-8?q?fix(mam-api):=20remove=20stitchedS3Stream=20wor?= =?UTF-8?q?karound=20=E2=80=94=20RustFS=20range=20bug=20fixed=20in=20beta.?= =?UTF-8?q?6=20(#143)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/mam-api/src/routes/assets.js | 96 ++------------------------- 1 file changed, 6 insertions(+), 90 deletions(-) diff --git a/services/mam-api/src/routes/assets.js b/services/mam-api/src/routes/assets.js index af83b44..167c7bb 100644 --- a/services/mam-api/src/routes/assets.js +++ b/services/mam-api/src/routes/assets.js @@ -858,65 +858,9 @@ router.get('/:id/live-path', async (req, res, next) => { // - ETag + Last-Modified for conditional requests (304 on repeat visits) // - Cache-Control: private, max-age=3600 so the browser caches segments // and doesn't re-fetch them on every seek within a session -// Issue #143 — RustFS returns empty bodies for ranged GETs whose start offset -// is past ~5.9 MB on single-file proxy MP4s. Confirmed via direct S3 probe: -// HEAD reports correct size, full GET (`bytes=0-`) works perfectly, but -// `bytes=8179166-` returns 206 + the right Content-Range header and a zero- -// byte body. A streaming GET from 0 reads cleanly *through* the broken zone. // -// Workaround until the proxy worker emits HLS (planned v1.2.1): stream the -// proxy from offset 0, skip bytes the client didn't ask for, stop after the -// requested end. Browser sees a normal 206 + Content-Range. Mem stays flat; -// extra RustFS-to-mam-api bandwidth = (end+1 - actual-range) per seek. -// -// Small head-of-file ranges below RUSTFS_RANGE_SAFE_START are handled by a -// direct ranged GET — saves the streaming-from-0 cost on the common case of -// initial moov + first-segment fetch. - -async function* stitchedS3Stream(key, startByte, endByte) { - // Yields buffers covering exactly [startByte, endByte] inclusive. - // - // RustFS only mis-serves a ranged GET when the *start* offset of the - // request is past ~5.8 MB. So we pull the object in 4 MB windows whose - // START offsets always stay below the broken threshold: - // - We anchor every chunk's start at a multiple of RUSTFS_SAFE_CHUNK - // (0, 4 MB, 8 MB, …). - // - Wait — that puts later starts past the threshold. - // Instead: skip directly to the chunk containing `startByte`, but request - // it as `bytes=anchorStart-end` where anchorStart < threshold. Since the - // bug only bites when the *request start* offset is large, we never issue - // a single GET whose Range start is past the broken zone — we instead - // exploit that a low-offset GET that *continues past* the threshold reads - // cleanly (confirmed by the bytes=0- full-GET probe). - // - // Practically: one GET from 0 that streams up through endByte, dropping - // the bytes below startByte as they arrive. Memory stays flat; we pay - // (endByte+1) bytes of RustFS-to-mam-api bandwidth per request. - const res = await s3Client.send(new GetObjectCommand({ - Bucket: getS3Bucket(), - Key: key, - Range: `bytes=0-${endByte}`, - })); - - let consumed = 0; // bytes seen so far from S3 - let totalEmitted = 0; - for await (const buf of res.Body) { - const bufStart = consumed; // file offset of buf[0] - const bufEnd = consumed + buf.length - 1; - consumed += buf.length; - if (bufEnd < startByte) continue; // entirely before window - const sliceFrom = Math.max(0, startByte - bufStart); - const sliceTo = Math.min(buf.length, endByte - bufStart + 1); - if (sliceTo > sliceFrom) { - yield buf.subarray(sliceFrom, sliceTo); - totalEmitted += sliceTo - sliceFrom; - } - if (bufEnd >= endByte) break; - } - if (totalEmitted === 0) { - throw new Error(`RustFS returned empty body for ${key} bytes=0-${endByte}`); - } -} +// RustFS issue #143 (empty body on ranged GETs past ~5.9 MB) was fixed in +// RustFS 1.0.0-alpha.94 (PR #2493). Standard ranged GETs used throughout. router.get('/:id/video', async (req, res, next) => { try { @@ -997,39 +941,11 @@ router.get('/:id/video', async (req, res, next) => { if (etag) headers['ETag'] = etag; if (lastModified) headers['Last-Modified'] = lastModified.toUTCString(); - // For small head-of-file ranges (entirely below the broken threshold) - // a direct ranged GET works and saves the streaming-from-0 cost. - const RUSTFS_RANGE_SAFE_START = parseInt(process.env.RUSTFS_RANGE_SAFE_START || String(5_500_000), 10); - if (start < RUSTFS_RANGE_SAFE_START && end < RUSTFS_RANGE_SAFE_START) { - const s3Res = await s3Client.send(new GetObjectCommand({ - Bucket: getS3Bucket(), Key: key, Range: `bytes=${start}-${end}`, - })); - res.writeHead(206, headers); - s3Res.Body.pipe(res); - return; - } - - // Otherwise: stream from offset 0, drop bytes below `start`, stop at - // `end`. Browser sees a normal 206; mam-api stays memory-flat. + const s3Res = await s3Client.send(new GetObjectCommand({ + Bucket: getS3Bucket(), Key: key, Range: `bytes=${start}-${end}`, + })); res.writeHead(206, headers); - try { - for await (const buf of stitchedS3Stream(key, start, end)) { - // res.write returns false when backpressure builds — pause and wait. - if (!res.write(buf)) { - await new Promise(r => res.once('drain', r)); - } - if (res.destroyed) return; - } - res.end(); - } catch (err) { - console.error(`[video] stitch failed for ${key}:`, err.message); - if (!res.headersSent) { - res.writeHead(500, { 'Content-Type': 'text/plain', 'Cache-Control': 'no-store' }); - res.end('Upstream storage error'); - } else { - res.destroy(err); - } - } + s3Res.Body.pipe(res); } catch (err) { next(err); } });