feat: presigned multipart upload for large files (browser-direct)
Large files (>32MB) now use presigned S3 multipart URLs via /api/desktop/multipart/init, letting the browser PUT 32MB chunks directly to S3 in 6 parallel streams. No data passes through Node. Small files still use single presigned PUT. Updated HTTP mode description to "HTTP multi-part upload".
This commit is contained in:
parent
6947e25230
commit
f7d8ba14cf
1 changed files with 61 additions and 30 deletions
|
|
@ -468,7 +468,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
||||||
<div style="position:relative;display:inline-block"><button class="mode-btn" id="btn-udp" onclick="toggleDlMenu(event)">🖥️ Electron App ▼</button><div id="dl-menu" style="display:none;position:absolute;top:110%;left:0;background:var(--bg-card);border:1px solid var(--border-bright);border-radius:8px;padding:6px;z-index:100;min-width:210px;box-shadow:0 8px 24px rgba(0,0,0,.4)"><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind.Setup.0.1.0.exe">🪟 Windows <span>.exe</span></a><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind-0.1.0-arm64.dmg">🍎 macOS <span>.dmg</span></a></div></div>
|
<div style="position:relative;display:inline-block"><button class="mode-btn" id="btn-udp" onclick="toggleDlMenu(event)">🖥️ Electron App ▼</button><div id="dl-menu" style="display:none;position:absolute;top:110%;left:0;background:var(--bg-card);border:1px solid var(--border-bright);border-radius:8px;padding:6px;z-index:100;min-width:210px;box-shadow:0 8px 24px rgba(0,0,0,.4)"><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind.Setup.0.1.0.exe">🪟 Windows <span>.exe</span></a><a class="dl-item" href="https://github.com/Acsriot94/dragon-wind-desktop/releases/download/v0.1.0/Dragon.Wind-0.1.0-arm64.dmg">🍎 macOS <span>.dmg</span></a></div></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="mode-desc" id="mode-desc" style="margin-bottom:0">
|
<div class="mode-desc" id="mode-desc" style="margin-bottom:0">
|
||||||
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server.</span>
|
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">HTTP multi-part upload</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
@ -946,7 +946,7 @@ function setMode(mode) {
|
||||||
if (mode === 'http') {
|
if (mode === 'http') {
|
||||||
btn.className = 'btn-upload';
|
btn.className = 'btn-upload';
|
||||||
if (label) label.textContent = 'HTTP Mode:';
|
if (label) label.textContent = 'HTTP Mode:';
|
||||||
if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server.';
|
if (detail) detail.textContent = 'HTTP multi-part upload';
|
||||||
if (hint) hint.style.display = 'none';
|
if (hint) hint.style.display = 'none';
|
||||||
if (btnHttp) { btnHttp.className = 'mode-btn active-http'; }
|
if (btnHttp) { btnHttp.className = 'mode-btn active-http'; }
|
||||||
if (btnUdp) { btnUdp.className = 'mode-btn'; }
|
if (btnUdp) { btnUdp.className = 'mode-btn'; }
|
||||||
|
|
@ -1127,21 +1127,41 @@ async function startUpload() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// PRESIGNED DIRECT-TO-S3 UPLOAD
|
// HTTP MULTI-PART UPLOAD
|
||||||
// Browser uploads directly to RustFS/S3 via presigned PUT URLs.
|
// Small files (<= 32 MB): single presigned PUT direct to S3.
|
||||||
// Server only generates signed URLs — file data never touches Node.
|
// Large files (> 32 MB): S3 multipart with presigned part URLs —
|
||||||
// Falls back to server-proxied upload if presigned fails.
|
// browser PUTs each 32 MB chunk directly to S3, 6 in parallel.
|
||||||
|
// Node only signs URLs; file data never touches the server.
|
||||||
|
// Falls back to server-proxied chunked upload if presigned fails.
|
||||||
// ============================================================
|
// ============================================================
|
||||||
const UPLOAD_CONCURRENCY = 6; // concurrent file uploads
|
const UPLOAD_CONCURRENCY = 6; // concurrent file uploads
|
||||||
const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per chunk
|
const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per chunk
|
||||||
const CHUNKS_PARALLEL = 6; // concurrent chunks per file
|
const CHUNKS_PARALLEL = 6; // concurrent chunks per file
|
||||||
|
|
||||||
|
// Helper: PUT a blob to a presigned URL, return ETag from response
|
||||||
|
function putChunkPresigned(url, blob, onProgress) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const xhr = new XMLHttpRequest();
|
||||||
|
if (onProgress) xhr.upload.addEventListener('progress', onProgress);
|
||||||
|
xhr.addEventListener('load', () => {
|
||||||
|
if (xhr.status >= 200 && xhr.status < 300) {
|
||||||
|
const etag = xhr.getResponseHeader('ETag');
|
||||||
|
resolve(etag);
|
||||||
|
} else { reject(new Error('S3 part upload returned ' + xhr.status)); }
|
||||||
|
});
|
||||||
|
xhr.addEventListener('error', () => reject(new Error('Part upload network error')));
|
||||||
|
xhr.addEventListener('abort', () => reject(new Error('Part upload aborted')));
|
||||||
|
xhr.open('PUT', url);
|
||||||
|
xhr.send(blob);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function uploadFilePresigned(item, idx) {
|
async function uploadFilePresigned(item, idx) {
|
||||||
const pb = document.getElementById(`progbar-${idx}`);
|
const pb = document.getElementById(`progbar-${idx}`);
|
||||||
const mime = item.file.type || 'application/octet-stream';
|
const mime = item.file.type || 'application/octet-stream';
|
||||||
const totalParts = Math.max(1, Math.ceil(item.file.size / CHUNK_SIZE));
|
const totalParts = Math.max(1, Math.ceil(item.file.size / CHUNK_SIZE));
|
||||||
|
|
||||||
// Small files (<= 32 MB): use direct presigned PUT (fastest path)
|
// Small files (<= 32 MB): single presigned PUT
|
||||||
if (totalParts === 1) {
|
if (totalParts === 1) {
|
||||||
setFileStatus(idx, 'uploading', 'Getting URL\u2026');
|
setFileStatus(idx, 'uploading', 'Getting URL\u2026');
|
||||||
const pre = await api('POST', '/api/presigned', {
|
const pre = await api('POST', '/api/presigned', {
|
||||||
|
|
@ -1173,39 +1193,43 @@ async function uploadFilePresigned(item, idx) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Large files (> 32 MB): chunked multipart upload
|
// Large files (> 32 MB): presigned multipart \u2014 browser \u2192 S3 direct
|
||||||
setFileStatus(idx, 'uploading', 'Initiating\u2026');
|
setFileStatus(idx, 'uploading', 'Initiating\u2026');
|
||||||
const init = await api('POST', '/api/upload/initiate', {
|
const init = await api('POST', '/api/desktop/multipart/init', {
|
||||||
filename: item.name, prefix: selectedPrefix, contentType: mime, totalParts,
|
filename: item.name, prefix: selectedPrefix, size: item.file.size, totalParts,
|
||||||
});
|
});
|
||||||
if (!init.success) throw new Error(init.error || 'Failed to initiate upload');
|
if (!init.uploadId || !init.presignedParts) throw new Error('Failed to initiate multipart upload');
|
||||||
const { uploadId, key } = init;
|
const { uploadId, key, bucket, presignedParts } = init;
|
||||||
|
|
||||||
let uploaded = 0;
|
let uploaded = 0;
|
||||||
|
const completedParts = [];
|
||||||
const chunkQueue = [];
|
const chunkQueue = [];
|
||||||
for (let i = 1; i <= totalParts; i++) chunkQueue.push(i);
|
for (let i = 0; i < totalParts; i++) chunkQueue.push(i);
|
||||||
let chunkError = null;
|
let chunkError = null;
|
||||||
|
|
||||||
async function chunkWorker() {
|
async function chunkWorker() {
|
||||||
while (chunkQueue.length > 0 && !chunkError) {
|
while (chunkQueue.length > 0 && !chunkError) {
|
||||||
const partNum = chunkQueue.shift();
|
const i = chunkQueue.shift();
|
||||||
if (!partNum) break;
|
if (i === undefined) break;
|
||||||
const start = (partNum - 1) * CHUNK_SIZE;
|
const start = i * CHUNK_SIZE;
|
||||||
const end = Math.min(start + CHUNK_SIZE, item.file.size);
|
const end = Math.min(start + CHUNK_SIZE, item.file.size);
|
||||||
const blob = item.file.slice(start, end);
|
const blob = item.file.slice(start, end);
|
||||||
const fd = new FormData();
|
const chunkSize = end - start;
|
||||||
fd.append('uploadId', uploadId);
|
|
||||||
fd.append('partNumber', String(partNum));
|
|
||||||
fd.append('chunk', blob);
|
|
||||||
try {
|
try {
|
||||||
const resp = await fetch('/api/upload/chunk', {
|
const etag = await putChunkPresigned(presignedParts[i], blob, (e) => {
|
||||||
method: 'POST', headers: { 'x-auth-token': authToken }, body: fd,
|
if (e.lengthComputable) {
|
||||||
}).then(r => r.json());
|
const chunkPct = e.loaded / e.total;
|
||||||
if (!resp.success) throw new Error(resp.error || 'Chunk ' + partNum + ' failed');
|
const totalDone = uploaded + chunkPct * chunkSize;
|
||||||
uploaded += (end - start);
|
const pct = Math.round(totalDone / item.file.size * 100);
|
||||||
|
if (pb) pb.style.width = Math.min(pct, 99) + '%';
|
||||||
|
setFileStatus(idx, 'uploading', Math.min(pct, 99) + '%');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
completedParts.push({ PartNumber: i + 1, ETag: etag });
|
||||||
|
uploaded += chunkSize;
|
||||||
const pct = Math.round(uploaded / item.file.size * 100);
|
const pct = Math.round(uploaded / item.file.size * 100);
|
||||||
if (pb) pb.style.width = pct + '%';
|
if (pb) pb.style.width = Math.min(pct, 99) + '%';
|
||||||
setFileStatus(idx, 'uploading', pct + '%');
|
setFileStatus(idx, 'uploading', Math.min(pct, 99) + '%');
|
||||||
} catch (e) { chunkError = e; }
|
} catch (e) { chunkError = e; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1215,13 +1239,20 @@ async function uploadFilePresigned(item, idx) {
|
||||||
await Promise.all(workers);
|
await Promise.all(workers);
|
||||||
|
|
||||||
if (chunkError) {
|
if (chunkError) {
|
||||||
try { await api('POST', '/api/upload/abort', { uploadId }); } catch(_) {}
|
try { await api('POST', '/api/desktop/multipart/abort', { uploadId, key, bucket }); } catch(_) {}
|
||||||
throw chunkError;
|
throw chunkError;
|
||||||
}
|
}
|
||||||
|
|
||||||
setFileStatus(idx, 'uploading', 'Finalizing\u2026');
|
setFileStatus(idx, 'uploading', 'Finalizing\u2026');
|
||||||
const complete = await api('POST', '/api/upload/complete', { uploadId });
|
const complete = await api('POST', '/api/desktop/multipart/complete', {
|
||||||
if (!complete.success) throw new Error(complete.error || 'Failed to complete upload');
|
uploadId, key, bucket, parts: completedParts,
|
||||||
|
});
|
||||||
|
if (!complete.success) throw new Error(complete.error || 'Failed to complete multipart upload');
|
||||||
|
if (pb) pb.style.width = '100%';
|
||||||
|
setFileStatus(idx, 'uploading', '100%');
|
||||||
|
|
||||||
|
// Notify server for quota tracking
|
||||||
|
try { await api('POST', '/api/presigned/complete', { key, size: item.file.size }); } catch(_) {}
|
||||||
return { key };
|
return { key };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue