feat: presigned multipart upload for large files (browser-direct)

Large files (>32MB) now use presigned S3 multipart URLs via
/api/desktop/multipart/init, letting the browser PUT 32MB chunks
directly to S3 in 6 parallel streams. No data passes through Node.
Small files still use single presigned PUT. Updated HTTP mode
description to "HTTP multi-part upload".
This commit is contained in:
Zac Gaetano 2026-04-09 21:43:22 -04:00
parent 6947e25230
commit f7d8ba14cf

View file

@ -468,7 +468,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
<div style="position:relative;display:inline-block"><button class="mode-btn" id="btn-udp" onclick="toggleDlMenu(event)">🖥️ Electron App &#9660;</button><div id="dl-menu" style="display:none;position:absolute;top:110%;left:0;background:var(--bg-card);border:1px solid var(--border-bright);border-radius:8px;padding:6px;z-index:100;min-width:210px;box-shadow:0 8px 24px rgba(0,0,0,.4)"><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind.Setup.0.1.0.exe">&#x1FA9F; Windows <span>.exe</span></a><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind-0.1.0-arm64.dmg">&#x1F34E; macOS <span>.dmg</span></a></div></div> <div style="position:relative;display:inline-block"><button class="mode-btn" id="btn-udp" onclick="toggleDlMenu(event)">🖥️ Electron App &#9660;</button><div id="dl-menu" style="display:none;position:absolute;top:110%;left:0;background:var(--bg-card);border:1px solid var(--border-bright);border-radius:8px;padding:6px;z-index:100;min-width:210px;box-shadow:0 8px 24px rgba(0,0,0,.4)"><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind.Setup.0.1.0.exe">&#x1FA9F; Windows <span>.exe</span></a><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind-0.1.0-arm64.dmg">&#x1F34E; macOS <span>.dmg</span></a></div></div>
</div> </div>
<div class="mode-desc" id="mode-desc" style="margin-bottom:0"> <div class="mode-desc" id="mode-desc" style="margin-bottom:0">
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server.</span> <strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">HTTP multi-part upload</span>
</div> </div>
</div> </div>
@ -946,7 +946,7 @@ function setMode(mode) {
if (mode === 'http') { if (mode === 'http') {
btn.className = 'btn-upload'; btn.className = 'btn-upload';
if (label) label.textContent = 'HTTP Mode:'; if (label) label.textContent = 'HTTP Mode:';
if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server.'; if (detail) detail.textContent = 'HTTP multi-part upload';
if (hint) hint.style.display = 'none'; if (hint) hint.style.display = 'none';
if (btnHttp) { btnHttp.className = 'mode-btn active-http'; } if (btnHttp) { btnHttp.className = 'mode-btn active-http'; }
if (btnUdp) { btnUdp.className = 'mode-btn'; } if (btnUdp) { btnUdp.className = 'mode-btn'; }
@ -1127,21 +1127,41 @@ async function startUpload() {
} }
// ============================================================ // ============================================================
// PRESIGNED DIRECT-TO-S3 UPLOAD // HTTP MULTI-PART UPLOAD
// Browser uploads directly to RustFS/S3 via presigned PUT URLs. // Small files (<= 32 MB): single presigned PUT direct to S3.
// Server only generates signed URLs — file data never touches Node. // Large files (> 32 MB): S3 multipart with presigned part URLs —
// Falls back to server-proxied upload if presigned fails. // browser PUTs each 32 MB chunk directly to S3, 6 in parallel.
// Node only signs URLs; file data never touches the server.
// Falls back to server-proxied chunked upload if presigned fails.
// ============================================================ // ============================================================
const UPLOAD_CONCURRENCY = 6; // concurrent file uploads const UPLOAD_CONCURRENCY = 6; // concurrent file uploads
const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per chunk const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per chunk
const CHUNKS_PARALLEL = 6; // concurrent chunks per file const CHUNKS_PARALLEL = 6; // concurrent chunks per file
// Helper: PUT a blob to a presigned URL, return ETag from response
function putChunkPresigned(url, blob, onProgress) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
if (onProgress) xhr.upload.addEventListener('progress', onProgress);
xhr.addEventListener('load', () => {
if (xhr.status >= 200 && xhr.status < 300) {
const etag = xhr.getResponseHeader('ETag');
resolve(etag);
} else { reject(new Error('S3 part upload returned ' + xhr.status)); }
});
xhr.addEventListener('error', () => reject(new Error('Part upload network error')));
xhr.addEventListener('abort', () => reject(new Error('Part upload aborted')));
xhr.open('PUT', url);
xhr.send(blob);
});
}
async function uploadFilePresigned(item, idx) { async function uploadFilePresigned(item, idx) {
const pb = document.getElementById(`progbar-${idx}`); const pb = document.getElementById(`progbar-${idx}`);
const mime = item.file.type || 'application/octet-stream'; const mime = item.file.type || 'application/octet-stream';
const totalParts = Math.max(1, Math.ceil(item.file.size / CHUNK_SIZE)); const totalParts = Math.max(1, Math.ceil(item.file.size / CHUNK_SIZE));
// Small files (<= 32 MB): use direct presigned PUT (fastest path) // Small files (<= 32 MB): single presigned PUT
if (totalParts === 1) { if (totalParts === 1) {
setFileStatus(idx, 'uploading', 'Getting URL\u2026'); setFileStatus(idx, 'uploading', 'Getting URL\u2026');
const pre = await api('POST', '/api/presigned', { const pre = await api('POST', '/api/presigned', {
@ -1173,39 +1193,43 @@ async function uploadFilePresigned(item, idx) {
}); });
} }
// Large files (> 32 MB): chunked multipart upload // Large files (> 32 MB): presigned multipart \u2014 browser \u2192 S3 direct
setFileStatus(idx, 'uploading', 'Initiating\u2026'); setFileStatus(idx, 'uploading', 'Initiating\u2026');
const init = await api('POST', '/api/upload/initiate', { const init = await api('POST', '/api/desktop/multipart/init', {
filename: item.name, prefix: selectedPrefix, contentType: mime, totalParts, filename: item.name, prefix: selectedPrefix, size: item.file.size, totalParts,
}); });
if (!init.success) throw new Error(init.error || 'Failed to initiate upload'); if (!init.uploadId || !init.presignedParts) throw new Error('Failed to initiate multipart upload');
const { uploadId, key } = init; const { uploadId, key, bucket, presignedParts } = init;
let uploaded = 0; let uploaded = 0;
const completedParts = [];
const chunkQueue = []; const chunkQueue = [];
for (let i = 1; i <= totalParts; i++) chunkQueue.push(i); for (let i = 0; i < totalParts; i++) chunkQueue.push(i);
let chunkError = null; let chunkError = null;
async function chunkWorker() { async function chunkWorker() {
while (chunkQueue.length > 0 && !chunkError) { while (chunkQueue.length > 0 && !chunkError) {
const partNum = chunkQueue.shift(); const i = chunkQueue.shift();
if (!partNum) break; if (i === undefined) break;
const start = (partNum - 1) * CHUNK_SIZE; const start = i * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, item.file.size); const end = Math.min(start + CHUNK_SIZE, item.file.size);
const blob = item.file.slice(start, end); const blob = item.file.slice(start, end);
const fd = new FormData(); const chunkSize = end - start;
fd.append('uploadId', uploadId);
fd.append('partNumber', String(partNum));
fd.append('chunk', blob);
try { try {
const resp = await fetch('/api/upload/chunk', { const etag = await putChunkPresigned(presignedParts[i], blob, (e) => {
method: 'POST', headers: { 'x-auth-token': authToken }, body: fd, if (e.lengthComputable) {
}).then(r => r.json()); const chunkPct = e.loaded / e.total;
if (!resp.success) throw new Error(resp.error || 'Chunk ' + partNum + ' failed'); const totalDone = uploaded + chunkPct * chunkSize;
uploaded += (end - start); const pct = Math.round(totalDone / item.file.size * 100);
if (pb) pb.style.width = Math.min(pct, 99) + '%';
setFileStatus(idx, 'uploading', Math.min(pct, 99) + '%');
}
});
completedParts.push({ PartNumber: i + 1, ETag: etag });
uploaded += chunkSize;
const pct = Math.round(uploaded / item.file.size * 100); const pct = Math.round(uploaded / item.file.size * 100);
if (pb) pb.style.width = pct + '%'; if (pb) pb.style.width = Math.min(pct, 99) + '%';
setFileStatus(idx, 'uploading', pct + '%'); setFileStatus(idx, 'uploading', Math.min(pct, 99) + '%');
} catch (e) { chunkError = e; } } catch (e) { chunkError = e; }
} }
} }
@ -1215,13 +1239,20 @@ async function uploadFilePresigned(item, idx) {
await Promise.all(workers); await Promise.all(workers);
if (chunkError) { if (chunkError) {
try { await api('POST', '/api/upload/abort', { uploadId }); } catch(_) {} try { await api('POST', '/api/desktop/multipart/abort', { uploadId, key, bucket }); } catch(_) {}
throw chunkError; throw chunkError;
} }
setFileStatus(idx, 'uploading', 'Finalizing\u2026'); setFileStatus(idx, 'uploading', 'Finalizing\u2026');
const complete = await api('POST', '/api/upload/complete', { uploadId }); const complete = await api('POST', '/api/desktop/multipart/complete', {
if (!complete.success) throw new Error(complete.error || 'Failed to complete upload'); uploadId, key, bucket, parts: completedParts,
});
if (!complete.success) throw new Error(complete.error || 'Failed to complete multipart upload');
if (pb) pb.style.width = '100%';
setFileStatus(idx, 'uploading', '100%');
// Notify server for quota tracking
try { await api('POST', '/api/presigned/complete', { key, size: item.file.size }); } catch(_) {}
return { key }; return { key };
} }