Presigned direct-to-S3 uploads — bypass Node server entirely
Browser now uploads files directly to RustFS/S3 via presigned PUT URLs. The Node server only generates signed URLs and tracks quota — file data never touches the server. 6 concurrent file uploads. Falls back to server-proxied PutObjectCommand upload if presigned fails. Server changes: - /api/presigned now checks folder permissions, quota, and blocked files - /api/presigned/complete endpoint for post-upload quota tracking Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0f81cac6ec
commit
ecdfe0f7cd
2 changed files with 109 additions and 29 deletions
|
|
@ -467,7 +467,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
<button class="mode-btn" id="btn-udp" disabled style="opacity:.4;cursor:not-allowed;pointer-events:none">🖥️ Electron App</button>
|
||||
</div>
|
||||
<div class="mode-desc" id="mode-desc" style="margin-bottom:0">
|
||||
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct HTTP upload to S3-compatible storage (up to 4 concurrent files). Compatible with RustFS, MinIO, and AWS S3.</span>
|
||||
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct-to-S3 upload via presigned URLs (up to 6 concurrent files). Browser uploads straight to storage, bypassing the server.</span>
|
||||
</div>
|
||||
<div style="text-align:right;margin-top:.3rem;font-size:.7rem;color:var(--text-dim);opacity:.6">Electron App — Aspera-speed desktop application (coming soon)</div>
|
||||
</div>
|
||||
|
|
@ -946,7 +946,7 @@ function setMode(mode) {
|
|||
if (mode === 'http') {
|
||||
btn.className = 'btn-upload';
|
||||
if (label) label.textContent = 'HTTP Mode:';
|
||||
if (detail) detail.textContent = 'Direct HTTP upload to S3-compatible storage (up to 4 concurrent files). Compatible with RustFS, MinIO, and AWS S3.';
|
||||
if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs (up to 6 concurrent files). Browser uploads straight to storage, bypassing the server.';
|
||||
if (hint) hint.style.display = 'none';
|
||||
if (btnHttp) { btnHttp.className = 'mode-btn active-http'; }
|
||||
if (btnUdp) { btnUdp.className = 'mode-btn'; }
|
||||
|
|
@ -1121,12 +1121,60 @@ async function startUpload() {
|
|||
}
|
||||
|
||||
// ============================================================
|
||||
// SIMPLE UPLOAD — uses /api/upload with PutObjectCommand on server
|
||||
// Compatible with RustFS / MinIO / generic S3 endpoints.
|
||||
// XMLHttpRequest used for upload progress tracking.
|
||||
// PRESIGNED DIRECT-TO-S3 UPLOAD
|
||||
// Browser uploads directly to RustFS/S3 via presigned PUT URLs.
|
||||
// Server only generates signed URLs — file data never touches Node.
|
||||
// Falls back to server-proxied upload if presigned fails.
|
||||
// ============================================================
|
||||
const UPLOAD_CONCURRENCY = 6; // concurrent file uploads
|
||||
|
||||
async function uploadFileDirect(item, idx) {
|
||||
async function uploadFilePresigned(item, idx) {
|
||||
const pb = document.getElementById(`progbar-${idx}`);
|
||||
const mime = item.file.type || 'application/octet-stream';
|
||||
|
||||
// 1. Get presigned PUT URL from server
|
||||
setFileStatus(idx, 'uploading', 'Getting URL…');
|
||||
const pre = await api('POST', '/api/presigned', {
|
||||
filename: item.name,
|
||||
prefix: selectedPrefix,
|
||||
contentType: mime,
|
||||
size: item.file.size,
|
||||
});
|
||||
if (!pre.success) throw new Error(pre.error || 'Failed to get presigned URL');
|
||||
|
||||
// 2. PUT directly to S3 via presigned URL
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
|
||||
xhr.upload.addEventListener('progress', (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const pct = Math.round(e.loaded / e.total * 100);
|
||||
if (pb) pb.style.width = pct + '%';
|
||||
setFileStatus(idx, 'uploading', pct + '%');
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('load', async () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
// 3. Notify server for quota tracking
|
||||
try { await api('POST', '/api/presigned/complete', { key: pre.key, size: item.file.size }); } catch(_) {}
|
||||
resolve({ key: pre.key });
|
||||
} else {
|
||||
reject(new Error(`S3 returned ${xhr.status}: ${xhr.statusText}`));
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('error', () => reject(new Error('Direct upload failed — network error')));
|
||||
xhr.addEventListener('abort', () => reject(new Error('Upload aborted')));
|
||||
|
||||
xhr.open('PUT', pre.url);
|
||||
xhr.setRequestHeader('Content-Type', mime);
|
||||
xhr.send(item.file);
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback: upload through Node server (for when presigned fails)
|
||||
async function uploadFileViaServer(item, idx) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const fd = new FormData();
|
||||
fd.append('prefix', selectedPrefix);
|
||||
|
|
@ -1139,21 +1187,16 @@ async function uploadFileDirect(item, idx) {
|
|||
if (e.lengthComputable) {
|
||||
const pct = Math.round(e.loaded / e.total * 100);
|
||||
if (pb) pb.style.width = pct + '%';
|
||||
setFileStatus(idx, 'uploading', pct + '%');
|
||||
setFileStatus(idx, 'uploading', `${pct}% (server)`);
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('load', () => {
|
||||
try {
|
||||
const data = JSON.parse(xhr.responseText);
|
||||
if (xhr.status >= 200 && xhr.status < 300 && data.success) {
|
||||
resolve(data);
|
||||
} else {
|
||||
reject(new Error(data.error || `Server returned ${xhr.status}`));
|
||||
}
|
||||
} catch (e) {
|
||||
reject(new Error(`Bad response: ${xhr.status}`));
|
||||
}
|
||||
if (xhr.status >= 200 && xhr.status < 300 && data.success) resolve(data);
|
||||
else reject(new Error(data.error || `Server returned ${xhr.status}`));
|
||||
} catch (e) { reject(new Error(`Bad response: ${xhr.status}`)); }
|
||||
});
|
||||
|
||||
xhr.addEventListener('error', () => reject(new Error('Network error')));
|
||||
|
|
@ -1166,8 +1209,6 @@ async function uploadFileDirect(item, idx) {
|
|||
}
|
||||
|
||||
async function uploadHTTP(files) {
|
||||
// Upload files with up to 4 concurrent file uploads
|
||||
const FILE_CONCURRENCY = 4;
|
||||
const fileQueue = [...files];
|
||||
|
||||
async function fileWorker() {
|
||||
|
|
@ -1175,22 +1216,34 @@ async function uploadHTTP(files) {
|
|||
const item = fileQueue.shift();
|
||||
if (!item) break;
|
||||
const idx = selectedFiles.indexOf(item);
|
||||
setFileStatus(idx,'uploading','Uploading…');
|
||||
document.getElementById(`prog-${idx}`).style.display='block';
|
||||
setFileStatus(idx, 'uploading', 'Starting…');
|
||||
document.getElementById(`prog-${idx}`).style.display = 'block';
|
||||
try {
|
||||
await uploadFileDirect(item, idx);
|
||||
document.getElementById(`progbar-${idx}`).style.width='100%';
|
||||
setFileStatus(idx,'done','✓ Done'); item.status='done';
|
||||
showToast(`Uploaded: ${item.name}`,'success');
|
||||
} catch(e) {
|
||||
setFileStatus(idx,'error','✗ Error'); item.status='error';
|
||||
showToast(`Failed: ${item.name} — ${e.message}`,'error');
|
||||
// Try presigned direct-to-S3 first
|
||||
await uploadFilePresigned(item, idx);
|
||||
document.getElementById(`progbar-${idx}`).style.width = '100%';
|
||||
setFileStatus(idx, 'done', '✓ Done'); item.status = 'done';
|
||||
showToast(`Uploaded: ${item.name}`, 'success');
|
||||
} catch(presignedErr) {
|
||||
// Fallback to server-proxied upload
|
||||
console.warn(`[upload] Presigned failed for ${item.name}: ${presignedErr.message}, falling back to server upload`);
|
||||
try {
|
||||
setFileStatus(idx, 'uploading', 'Retrying via server…');
|
||||
if (document.getElementById(`progbar-${idx}`)) document.getElementById(`progbar-${idx}`).style.width = '0%';
|
||||
await uploadFileViaServer(item, idx);
|
||||
document.getElementById(`progbar-${idx}`).style.width = '100%';
|
||||
setFileStatus(idx, 'done', '✓ Done (server)'); item.status = 'done';
|
||||
showToast(`Uploaded: ${item.name}`, 'success');
|
||||
} catch(serverErr) {
|
||||
setFileStatus(idx, 'error', '✗ Error'); item.status = 'error';
|
||||
showToast(`Failed: ${item.name} — ${serverErr.message}`, 'error');
|
||||
}
|
||||
}
|
||||
updateStats();
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({length: Math.min(FILE_CONCURRENCY, files.length)}, fileWorker));
|
||||
await Promise.all(Array.from({length: Math.min(UPLOAD_CONCURRENCY, files.length)}, fileWorker));
|
||||
}
|
||||
|
||||
async function uploadUDP(files) {
|
||||
|
|
|
|||
31
server.js
31
server.js
|
|
@ -558,11 +558,26 @@ app.post("/api/upload", requireAuth, upload.array("files", 50), async (req, res)
|
|||
}
|
||||
});
|
||||
|
||||
// ==================== PRESIGNED URL (for Chrome extension HTTP mode) ====================
|
||||
// ==================== PRESIGNED URL (direct-to-S3 upload) ====================
|
||||
// Client uploads directly to RustFS/S3 using presigned PUT URLs, bypassing the Node server.
|
||||
// Server only generates the signed URL and tracks quota/permissions.
|
||||
app.post("/api/presigned", requireAuth, async (req, res) => {
|
||||
if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" });
|
||||
const { filename, prefix, contentType } = req.body;
|
||||
const { filename, prefix, contentType, size } = req.body;
|
||||
if (!filename) return res.status(400).json({ success: false, error: "filename required" });
|
||||
if (isBlockedFile(filename)) return res.status(400).json({ success: false, error: `Blocked file type: ${filename}` });
|
||||
|
||||
const user = db.users.find(u => u.username === req.sessionData.user);
|
||||
// Folder permission check
|
||||
if (user && !isFolderAllowed(user, prefix || "")) {
|
||||
return res.status(403).json({ success: false, error: "You do not have permission to upload to this folder" });
|
||||
}
|
||||
// Quota check
|
||||
if (size && user && isQuotaExceeded(user, size)) {
|
||||
const usedMB = ((user.uploadedBytes || 0) / 1024 / 1024).toFixed(1);
|
||||
return res.status(403).json({ success: false, error: `Upload quota exceeded (${usedMB} MB of ${user.quotaMB} MB used)` });
|
||||
}
|
||||
|
||||
const bucket = db.s3Config?.bucket || "";
|
||||
const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename;
|
||||
const mime = contentType || getMimeType(filename, "application/octet-stream");
|
||||
|
|
@ -575,6 +590,18 @@ app.post("/api/presigned", requireAuth, async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
// Called by client after a successful direct-to-S3 upload to update quota tracking
|
||||
app.post("/api/presigned/complete", requireAuth, (req, res) => {
|
||||
const { key, size } = req.body;
|
||||
const user = db.users.find(u => u.username === req.sessionData.user);
|
||||
if (user && size) {
|
||||
user.uploadedBytes = (user.uploadedBytes || 0) + size;
|
||||
saveData(db);
|
||||
}
|
||||
console.log(`[presigned] Completed: ${key} (${size} bytes) by ${req.sessionData.user}`);
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
||||
// ==================== UDP UPLOAD SESSION ====================
|
||||
// Sessions stored in memory; relay handles actual transfer
|
||||
const udpSessions = new Map();
|
||||
|
|
|
|||
Loading…
Reference in a new issue