zgaetano/DragonWind
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: presigned multipart upload for large files (browser-direct)

Browse source 
Large files (>32MB) now use presigned S3 multipart URLs via
/api/desktop/multipart/init, letting the browser PUT 32MB chunks
directly to S3 in 6 parallel streams. No data passes through Node.
Small files still use single presigned PUT. Updated HTTP mode
description to "HTTP multi-part upload".
This commit is contained in:
Zac Gaetano 2026-04-09 21:43:22 -04:00
parent 6947e25230
commit f7d8ba14cf
1 changed files with 61 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
public/index.html
View file

@ -468,7 +468,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
<div style="position:relative;display:inline-block"><button class="mode-btn" id="btn-udp" onclick="toggleDlMenu(event)">🖥️ Electron App &#9660;</button><div id="dl-menu" style="display:none;position:absolute;top:110%;left:0;background:var(--bg-card);border:1px solid var(--border-bright);border-radius:8px;padding:6px;z-index:100;min-width:210px;box-shadow:0 8px 24px rgba(0,0,0,.4)"><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind.Setup.0.1.0.exe">&#x1FA9F; Windows <span>.exe</span></a><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind-0.1.0-arm64.dmg">&#x1F34E; macOS <span>.dmg</span></a></div></div>
</div>
<div class="mode-desc" id="mode-desc" style="margin-bottom:0">
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server.</span>
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">HTTP multi-part upload</span>
</div>
</div>
@ -946,7 +946,7 @@ function setMode(mode) {
if (mode === 'http') {
btn.className = 'btn-upload';
if (label) label.textContent = 'HTTP Mode:';
if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server.';
if (detail) detail.textContent = 'HTTP multi-part upload';
if (hint) hint.style.display = 'none';
if (btnHttp) { btnHttp.className = 'mode-btn active-http'; }
if (btnUdp) { btnUdp.className = 'mode-btn'; }
@ -1127,21 +1127,41 @@ async function startUpload() {
}
// ============================================================
// PRESIGNED DIRECT-TO-S3 UPLOAD
// Browser uploads directly to RustFS/S3 via presigned PUT URLs.
// Server only generates signed URLs — file data never touches Node.
// Falls back to server-proxied upload if presigned fails.
// HTTP MULTI-PART UPLOAD
// Small files (<= 32 MB): single presigned PUT direct to S3.
// Large files (> 32 MB): S3 multipart with presigned part URLs —
// browser PUTs each 32 MB chunk directly to S3, 6 in parallel.
// Node only signs URLs; file data never touches the server.
// Falls back to server-proxied chunked upload if presigned fails.
// ============================================================
const UPLOAD_CONCURRENCY = 6; // concurrent file uploads
const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per chunk
const CHUNKS_PARALLEL = 6; // concurrent chunks per file
// Helper: PUT a blob to a presigned URL, return ETag from response
function putChunkPresigned(url, blob, onProgress) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
if (onProgress) xhr.upload.addEventListener('progress', onProgress);
xhr.addEventListener('load', () => {
if (xhr.status >= 200 && xhr.status < 300) {
const etag = xhr.getResponseHeader('ETag');
resolve(etag);
} else { reject(new Error('S3 part upload returned ' + xhr.status)); }
});
xhr.addEventListener('error', () => reject(new Error('Part upload network error')));
xhr.addEventListener('abort', () => reject(new Error('Part upload aborted')));
xhr.open('PUT', url);
xhr.send(blob);
});
}
async function uploadFilePresigned(item, idx) {
const pb = document.getElementById(`progbar-${idx}`);
const mime = item.file.type || 'application/octet-stream';
const totalParts = Math.max(1, Math.ceil(item.file.size / CHUNK_SIZE));
// Small files (<= 32 MB): use direct presigned PUT (fastest path)
// Small files (<= 32 MB): single presigned PUT
if (totalParts === 1) {
setFileStatus(idx, 'uploading', 'Getting URL\u2026');
const pre = await api('POST', '/api/presigned', {
@ -1173,39 +1193,43 @@ async function uploadFilePresigned(item, idx) {
});
}
// Large files (> 32 MB): chunked multipart upload
// Large files (> 32 MB): presigned multipart \u2014 browser \u2192 S3 direct
setFileStatus(idx, 'uploading', 'Initiating\u2026');
const init = await api('POST', '/api/upload/initiate', {
filename: item.name, prefix: selectedPrefix, contentType: mime, totalParts,
const init = await api('POST', '/api/desktop/multipart/init', {
filename: item.name, prefix: selectedPrefix, size: item.file.size, totalParts,
});
if (!init.success) throw new Error(init.error || 'Failed to initiate upload');
const { uploadId, key } = init;
if (!init.uploadId || !init.presignedParts) throw new Error('Failed to initiate multipart upload');
const { uploadId, key, bucket, presignedParts } = init;
let uploaded = 0;
const completedParts = [];
const chunkQueue = [];
for (let i = 1; i <= totalParts; i++) chunkQueue.push(i);
for (let i = 0; i < totalParts; i++) chunkQueue.push(i);
let chunkError = null;
async function chunkWorker() {
while (chunkQueue.length > 0 && !chunkError) {
const partNum = chunkQueue.shift();
if (!partNum) break;
const start = (partNum - 1) * CHUNK_SIZE;
const i = chunkQueue.shift();
if (i === undefined) break;
const start = i * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, item.file.size);
const blob = item.file.slice(start, end);
const fd = new FormData();
fd.append('uploadId', uploadId);
fd.append('partNumber', String(partNum));
fd.append('chunk', blob);
const chunkSize = end - start;
try {
const resp = await fetch('/api/upload/chunk', {
method: 'POST', headers: { 'x-auth-token': authToken }, body: fd,
}).then(r => r.json());
if (!resp.success) throw new Error(resp.error || 'Chunk ' + partNum + ' failed');
uploaded += (end - start);
const etag = await putChunkPresigned(presignedParts[i], blob, (e) => {
if (e.lengthComputable) {
const chunkPct = e.loaded / e.total;
const totalDone = uploaded + chunkPct * chunkSize;
const pct = Math.round(totalDone / item.file.size * 100);
if (pb) pb.style.width = Math.min(pct, 99) + '%';
setFileStatus(idx, 'uploading', Math.min(pct, 99) + '%');
}
});
completedParts.push({ PartNumber: i + 1, ETag: etag });
uploaded += chunkSize;
const pct = Math.round(uploaded / item.file.size * 100);
if (pb) pb.style.width = pct + '%';
setFileStatus(idx, 'uploading', pct + '%');
if (pb) pb.style.width = Math.min(pct, 99) + '%';
setFileStatus(idx, 'uploading', Math.min(pct, 99) + '%');
} catch (e) { chunkError = e; }
}
}
@ -1215,13 +1239,20 @@ async function uploadFilePresigned(item, idx) {
await Promise.all(workers);
if (chunkError) {
try { await api('POST', '/api/upload/abort', { uploadId }); } catch(_) {}
try { await api('POST', '/api/desktop/multipart/abort', { uploadId, key, bucket }); } catch(_) {}
throw chunkError;
}
setFileStatus(idx, 'uploading', 'Finalizing\u2026');
const complete = await api('POST', '/api/upload/complete', { uploadId });
if (!complete.success) throw new Error(complete.error || 'Failed to complete upload');
const complete = await api('POST', '/api/desktop/multipart/complete', {
uploadId, key, bucket, parts: completedParts,
});
if (!complete.success) throw new Error(complete.error || 'Failed to complete multipart upload');
if (pb) pb.style.width = '100%';
setFileStatus(idx, 'uploading', '100%');
// Notify server for quota tracking
try { await api('POST', '/api/presigned/complete', { key, size: item.file.size }); } catch(_) {}
return { key };
}