merge: keep desktop multipart API routes alongside parallel chunk upload routes
This commit is contained in:
commit
c4158f0cfd
4 changed files with 470 additions and 128 deletions
|
|
@ -1,10 +1,7 @@
|
|||
version: "3.9"
|
||||
|
||||
# =============================================================
|
||||
# Dragon Wind — Full Stack
|
||||
# Services:
|
||||
# dragon-wind — Main upload web app (port 3000)
|
||||
# udp-relay — UDP relay server (TCP 3001 + UDP 5000)
|
||||
# Dragon Wind — Upload Portal
|
||||
# =============================================================
|
||||
|
||||
services:
|
||||
|
|
@ -30,30 +27,9 @@ services:
|
|||
- S3_BUCKET=${S3_BUCKET:-}
|
||||
- S3_ACCESS_KEY=${S3_ACCESS_KEY:-}
|
||||
- S3_SECRET_KEY=${S3_SECRET_KEY:-}
|
||||
# Relay URL for UDP mode
|
||||
- RELAY_URL=${RELAY_URL:-http://udp-relay:3001}
|
||||
- UDP_PORT=${UDP_PORT:-5000}
|
||||
# AMPP (optional)
|
||||
- AMPP_BASE_URL=${AMPP_BASE_URL:-https://us-east-1.gvampp.com}
|
||||
- AMPP_API_KEY=${AMPP_API_KEY:-}
|
||||
depends_on:
|
||||
- udp-relay
|
||||
networks:
|
||||
- dragon-wind-net
|
||||
|
||||
udp-relay:
|
||||
build:
|
||||
context: ./udp-relay
|
||||
dockerfile: Dockerfile
|
||||
container_name: dragon-wind-relay
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${RELAY_TCP_PORT:-3001}:3001" # Control API (TCP)
|
||||
- "${RELAY_UDP_PORT:-5000}:5000/udp" # Data transfer (UDP)
|
||||
environment:
|
||||
- PORT=3001
|
||||
- UDP_PORT=5000
|
||||
- MAX_SESSIONS=${MAX_RELAY_SESSIONS:-50}
|
||||
networks:
|
||||
- dragon-wind-net
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ class UploadManager {
|
|||
createSession({ filename, size, mode = "http", prefix = "" }) {
|
||||
if (!["http", "udp"].includes(mode)) throw new Error("mode must be 'http' or 'udp'");
|
||||
const sessionId = crypto.randomBytes(16).toString("hex");
|
||||
const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename;
|
||||
// Normalize prefix: UI uses "/" for nested folders, FLX expects "--" as delimiter
|
||||
const normalized = prefix ? prefix.replace(/\//g, "--").replace(/[-]+$/, "") : "";
|
||||
const key = normalized ? `${normalized}--${filename}` : filename;
|
||||
const session = {
|
||||
sessionId, filename, size, mode, key, prefix,
|
||||
status: "pending",
|
||||
|
|
|
|||
|
|
@ -464,17 +464,18 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
<div style="margin-top:1.5rem;margin-bottom:1.25rem">
|
||||
<div style="display:flex;gap:.5rem;margin-bottom:.75rem">
|
||||
<button class="mode-btn active-http" id="btn-http" onclick="setMode('http')">🌐 HTTP Mode</button>
|
||||
<button class="mode-btn" id="btn-udp" onclick="setMode('udp')">⚡ UDP Mode</button>
|
||||
<button class="mode-btn" id="btn-udp" disabled style="opacity:.4;cursor:not-allowed;pointer-events:none">🖥️ Electron App</button>
|
||||
</div>
|
||||
<div class="mode-desc" id="mode-desc" style="margin-bottom:0">
|
||||
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct S3 presigned upload. Best for LAN and stable connections.</span>
|
||||
<span id="udp-ext-hint" style="display:none;margin-left:.5rem;color:var(--dragon-bright);font-size:.75rem">⚡ UDP extension detected</span>
|
||||
<strong id="mode-label">HTTP Mode:</strong> <span id="mode-detail">Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server. Files are processed 6 at a time.</span>
|
||||
</div>
|
||||
<div style="text-align:right;margin-top:.3rem;font-size:.7rem;color:var(--text-dim);opacity:.6">Electron App — Aspera-speed desktop application (coming soon)</div>
|
||||
</div>
|
||||
|
||||
<div style="margin-bottom:1.5rem">
|
||||
<div class="section-title">Destination Folder</div>
|
||||
<div class="folder-tree-box" id="folder-tree-box"></div>
|
||||
<input class="form-input" id="folder-search" placeholder="🔍 Search folders…" oninput="renderFolderTree()" style="margin-bottom:.4rem;font-size:.78rem;padding:.4rem .65rem"/>
|
||||
<div class="folder-tree-box" id="folder-tree-box" style="max-height:320px;overflow-y:auto"></div>
|
||||
<div class="add-row" style="margin-top:.55rem" id="add-folder-row">
|
||||
<input class="add-input" id="new-folder-input" placeholder="New folder name…" onkeydown="if(event.key==='Enter')addFolder()"/>
|
||||
<button class="btn-small" onclick="addFolder()">+ Add</button>
|
||||
|
|
@ -521,11 +522,9 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
<div class="admin-tabs">
|
||||
<div class="admin-tab active" data-tab="s3" onclick="switchAdminTab('s3')">S3 Storage</div>
|
||||
<div class="admin-tab" data-tab="ampp" onclick="switchAdminTab('ampp')">AMPP</div>
|
||||
<div class="admin-tab" data-tab="extension" onclick="switchAdminTab('extension')">🧩 Extension</div>
|
||||
<div class="admin-tab" data-tab="users" onclick="switchAdminTab('users')">Users</div>
|
||||
<div class="admin-tab" data-tab="sharelinks" onclick="switchAdminTab('sharelinks')">🔗 Share Links</div>
|
||||
<div class="admin-tab" data-tab="folders" onclick="switchAdminTab('folders')">Folders</div>
|
||||
<div class="admin-tab" data-tab="relay" onclick="switchAdminTab('relay')">⚡ UDP Relay</div>
|
||||
</div>
|
||||
|
||||
<!-- S3 -->
|
||||
|
|
@ -554,8 +553,12 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
</div>
|
||||
|
||||
<!-- Relay -->
|
||||
<div class="admin-panel" id="admin-relay">
|
||||
<div class="admin-panel" id="admin-relay" style="display:none!important">
|
||||
<div class="section-title">UDP Relay Configuration</div>
|
||||
<div style="background:linear-gradient(135deg,rgba(255,180,0,.08),rgba(255,120,0,.08));border:1px solid rgba(255,180,0,.3);border-radius:10px;padding:1.2rem 1.4rem;margin-bottom:1.5rem">
|
||||
<div style="font-size:1rem;font-weight:700;color:var(--dragon-bright);margin-bottom:.4rem">⚡ Coming Soon — Electron Desktop Uploader</div>
|
||||
<p style="color:var(--text-secondary);margin:0;line-height:1.6;font-size:.88rem">UDP relay mode is being replaced by a native <strong style="color:var(--text-primary)">Electron desktop app</strong> that delivers Aspera-class transfer speeds over a direct connection — no browser limitations, no port forwarding, and significantly higher throughput. Configuration below is reserved for the upcoming release.</p>
|
||||
</div>
|
||||
<div class="relay-status-indicator"><div class="relay-dot grey" id="relay-dot"></div><span id="relay-status-text">Not checked</span></div>
|
||||
<div class="form-group"><label class="form-label">Internal Relay URL</label><input class="form-input" id="relay-url" type="url" placeholder="http://dragon-wind-relay:3001"/><div class="form-hint">Internal URL the server uses to reach the relay container (Docker service name or localhost)</div></div>
|
||||
<div class="form-group"><label class="form-label">Public Relay URL <span style="font-weight:400;text-transform:none;letter-spacing:0;color:var(--text-dim)">(what browsers connect to)</span></label><input class="form-input" id="relay-public-url" type="url" placeholder="http://vpm.broadcastmgmt.cloud:3001"/><div class="form-hint">The externally reachable URL for the relay — sent to uploaders' browsers. Must be reachable on port 3001 from the internet.</div></div>
|
||||
|
|
@ -587,8 +590,8 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
<div class="status-msg" id="ampp-cfg-status"></div>
|
||||
</div>
|
||||
|
||||
<!-- Extension -->
|
||||
<div class="admin-panel" id="admin-extension">
|
||||
<!-- Extension (hidden) -->
|
||||
<div class="admin-panel" id="admin-extension" style="display:none!important">
|
||||
<div class="section-title">Chrome Extension</div>
|
||||
<p style="color:var(--text-secondary);margin-bottom:1.5rem;line-height:1.6">The Chrome extension is required to use fast UDP uploads. Install it once in Chrome — no other setup needed on your end.</p>
|
||||
|
||||
|
|
@ -669,6 +672,17 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
<div class="status-msg" id="perm-status"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- User Audit modal -->
|
||||
<div id="audit-modal" style="display:none;position:fixed;inset:0;background:rgba(0,0,0,.7);z-index:500;align-items:center;justify-content:center">
|
||||
<div style="background:var(--bg-card);border:1px solid var(--border);border-radius:16px;padding:2rem;width:520px;max-width:95vw;max-height:90vh;overflow-y:auto">
|
||||
<div class="section-title" id="audit-modal-title">User Audit</div>
|
||||
<div id="audit-content" style="margin-top:1rem"></div>
|
||||
<div class="btn-row" style="margin-top:1.5rem">
|
||||
<button class="btn-secondary" onclick="document.getElementById('audit-modal').style.display='none'">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Share Links -->
|
||||
|
|
@ -724,7 +738,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
|
|||
<!-- FOOTER -->
|
||||
<footer class="app-footer">
|
||||
<div class="app-footer-text">
|
||||
Built by <strong>Zac Gaetano</strong> & <strong>Wild Dragon LLC</strong>
|
||||
Built by <strong>Zac Gaetano</strong>
|
||||
<span class="app-footer-divider">·</span>
|
||||
In partnership with <strong>Broadcast Management Group</strong>
|
||||
</div>
|
||||
|
|
@ -932,7 +946,7 @@ function setMode(mode) {
|
|||
if (mode === 'http') {
|
||||
btn.className = 'btn-upload';
|
||||
if (label) label.textContent = 'HTTP Mode:';
|
||||
if (detail) detail.textContent = 'Direct S3 presigned upload. Best for LAN and stable connections.';
|
||||
if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server. Files are processed 6 at a time.';
|
||||
if (hint) hint.style.display = 'none';
|
||||
if (btnHttp) { btnHttp.className = 'mode-btn active-http'; }
|
||||
if (btnUdp) { btnUdp.className = 'mode-btn'; }
|
||||
|
|
@ -960,6 +974,11 @@ async function loadFolders() {
|
|||
try {
|
||||
const d = await api('GET','/api/folders');
|
||||
folderTree = d.tree || [];
|
||||
// Auto-select VPM as default folder if nothing is selected yet
|
||||
if (!selectedPrefix && folderTree.some(n => n.name === 'VPM')) {
|
||||
selectedPrefix = 'VPM';
|
||||
updatePrefixDisplay();
|
||||
}
|
||||
renderFolderTree();
|
||||
} catch(e) { console.error('loadFolders:',e); }
|
||||
}
|
||||
|
|
@ -968,14 +987,30 @@ function renderFolderTree() {
|
|||
const box = document.getElementById('folder-tree-box');
|
||||
if (!box) return;
|
||||
box.innerHTML = '';
|
||||
// Root row
|
||||
const rootRow = document.createElement('div');
|
||||
rootRow.className = 'folder-tree-row' + (selectedPrefix==='' ? ' active' : '');
|
||||
rootRow.innerHTML = `<span class="ftr-icon">🏠</span><span class="ftr-name" style="font-style:${selectedPrefix===''?'normal':'italic'};color:${selectedPrefix===''?'':'var(--text-dim)'}">Root (no folder)</span>`;
|
||||
rootRow.onclick = () => { selectedPrefix=''; updatePrefixDisplay(); renderFolderTree(); };
|
||||
box.appendChild(rootRow);
|
||||
const searchEl = document.getElementById('folder-search');
|
||||
const filter = (searchEl ? searchEl.value.trim().toLowerCase() : '');
|
||||
|
||||
// Helper: does a node (or any descendant) match the filter?
|
||||
function matchesFilter(node, pathArr) {
|
||||
const key = [...pathArr, node.name].join('/');
|
||||
if (key.toLowerCase().includes(filter)) return true;
|
||||
if (node.children) return node.children.some(c => matchesFilter(c, [...pathArr, node.name]));
|
||||
return false;
|
||||
}
|
||||
|
||||
// Root row (always shown unless filtering)
|
||||
if (!filter) {
|
||||
const rootRow = document.createElement('div');
|
||||
rootRow.className = 'folder-tree-row' + (selectedPrefix==='' ? ' active' : '');
|
||||
rootRow.innerHTML = `<span class="ftr-icon">🏠</span><span class="ftr-name" style="font-style:${selectedPrefix===''?'normal':'italic'};color:${selectedPrefix===''?'':'var(--text-dim)'}">Root (no folder)</span>`;
|
||||
rootRow.onclick = () => { selectedPrefix=''; updatePrefixDisplay(); renderFolderTree(); };
|
||||
box.appendChild(rootRow);
|
||||
}
|
||||
|
||||
function addRows(nodes, pathArr, container) {
|
||||
nodes.forEach(n => {
|
||||
const sorted = [...nodes].sort((a, b) => a.name.localeCompare(b.name, undefined, {sensitivity: 'base'}));
|
||||
sorted.forEach(n => {
|
||||
if (filter && !matchesFilter(n, pathArr)) return;
|
||||
const fullPath = [...pathArr, n.name];
|
||||
const key = fullPath.join('/');
|
||||
const indent = pathArr.length;
|
||||
|
|
@ -997,6 +1032,10 @@ function renderFolderTree() {
|
|||
});
|
||||
}
|
||||
addRows(folderTree, [], box);
|
||||
|
||||
if (!box.children.length) {
|
||||
box.innerHTML = '<div style="color:var(--text-dim);font-size:.8rem;padding:.75rem;text-align:center">No folders match your search</div>';
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy aliases so other code still works
|
||||
|
|
@ -1022,7 +1061,7 @@ async function deleteFolder(pathArr) {
|
|||
try {
|
||||
await api('POST','/api/folders/delete',{path:pathArr});
|
||||
if (selectedPrefix.startsWith(pathArr.join('/'))) { selectedPrefix=''; updatePrefixDisplay(); }
|
||||
await loadFolders(); showToast('Folder deleted','success');
|
||||
await loadFolders(); await loadAdminFolders(); showToast('Folder deleted','success');
|
||||
} catch(e) { showToast(e.message,'error'); }
|
||||
}
|
||||
|
||||
|
|
@ -1087,31 +1126,130 @@ async function startUpload() {
|
|||
updateUploadBtn(); updateStats();
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// PRESIGNED DIRECT-TO-S3 UPLOAD
|
||||
// Browser uploads directly to RustFS/S3 via presigned PUT URLs.
|
||||
// Server only generates signed URLs — file data never touches Node.
|
||||
// Falls back to server-proxied upload if presigned fails.
|
||||
// ============================================================
|
||||
const UPLOAD_CONCURRENCY = 6; // concurrent file uploads
|
||||
|
||||
async function uploadFilePresigned(item, idx) {
|
||||
const pb = document.getElementById(`progbar-${idx}`);
|
||||
const mime = item.file.type || 'application/octet-stream';
|
||||
|
||||
// 1. Get presigned PUT URL from server
|
||||
setFileStatus(idx, 'uploading', 'Getting URL…');
|
||||
const pre = await api('POST', '/api/presigned', {
|
||||
filename: item.name,
|
||||
prefix: selectedPrefix,
|
||||
contentType: mime,
|
||||
size: item.file.size,
|
||||
});
|
||||
if (!pre.success) throw new Error(pre.error || 'Failed to get presigned URL');
|
||||
|
||||
// 2. PUT directly to S3 via presigned URL
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
|
||||
xhr.upload.addEventListener('progress', (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const pct = Math.round(e.loaded / e.total * 100);
|
||||
if (pb) pb.style.width = pct + '%';
|
||||
setFileStatus(idx, 'uploading', pct + '%');
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('load', async () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
// 3. Notify server for quota tracking
|
||||
try { await api('POST', '/api/presigned/complete', { key: pre.key, size: item.file.size }); } catch(_) {}
|
||||
resolve({ key: pre.key });
|
||||
} else {
|
||||
reject(new Error(`S3 returned ${xhr.status}: ${xhr.statusText}`));
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('error', () => reject(new Error('Direct upload failed — network error')));
|
||||
xhr.addEventListener('abort', () => reject(new Error('Upload aborted')));
|
||||
|
||||
xhr.open('PUT', pre.url);
|
||||
xhr.setRequestHeader('Content-Type', mime);
|
||||
xhr.send(item.file);
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback: upload through Node server (for when presigned fails)
|
||||
async function uploadFileViaServer(item, idx) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const fd = new FormData();
|
||||
fd.append('prefix', selectedPrefix);
|
||||
fd.append('files', item.file, item.name);
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
const pb = document.getElementById(`progbar-${idx}`);
|
||||
|
||||
xhr.upload.addEventListener('progress', (e) => {
|
||||
if (e.lengthComputable) {
|
||||
const pct = Math.round(e.loaded / e.total * 100);
|
||||
if (pb) pb.style.width = pct + '%';
|
||||
setFileStatus(idx, 'uploading', `${pct}% (server)`);
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('load', () => {
|
||||
try {
|
||||
const data = JSON.parse(xhr.responseText);
|
||||
if (xhr.status >= 200 && xhr.status < 300 && data.success) resolve(data);
|
||||
else reject(new Error(data.error || `Server returned ${xhr.status}`));
|
||||
} catch (e) { reject(new Error(`Bad response: ${xhr.status}`)); }
|
||||
});
|
||||
|
||||
xhr.addEventListener('error', () => reject(new Error('Network error')));
|
||||
xhr.addEventListener('abort', () => reject(new Error('Upload aborted')));
|
||||
|
||||
xhr.open('POST', '/api/upload');
|
||||
xhr.setRequestHeader('x-auth-token', authToken);
|
||||
xhr.send(fd);
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadHTTP(files) {
|
||||
for (const item of files) {
|
||||
const idx = selectedFiles.indexOf(item);
|
||||
setFileStatus(idx,'uploading','Uploading…');
|
||||
document.getElementById(`prog-${idx}`).style.display='block';
|
||||
try {
|
||||
const presigned = await api('POST','/api/presigned',{filename:item.name,prefix:selectedPrefix,contentType:item.file.type||'application/octet-stream'});
|
||||
if (!presigned.success) throw new Error(presigned.error||'Failed to get presigned URL');
|
||||
// Use the content type the server signed — browser file.type may differ for broadcast formats
|
||||
const signedType = presigned.contentType || item.file.type || 'application/octet-stream';
|
||||
await new Promise((resolve,reject) => {
|
||||
const xhr=new XMLHttpRequest();
|
||||
xhr.open('PUT',presigned.url);
|
||||
xhr.setRequestHeader('Content-Type',signedType);
|
||||
xhr.upload.onprogress=e=>{ if(e.lengthComputable){ const p=Math.round(e.loaded/e.total*100); document.getElementById(`progbar-${idx}`).style.width=p+'%'; setFileStatus(idx,'uploading',p+'%'); } };
|
||||
xhr.onload=()=>xhr.status<300?resolve():reject(new Error(`S3 error ${xhr.status}`));
|
||||
xhr.onerror=()=>reject(new Error('Network error'));
|
||||
xhr.send(item.file);
|
||||
});
|
||||
document.getElementById(`progbar-${idx}`).style.width='100%';
|
||||
setFileStatus(idx,'done','✓ Done'); item.status='done';
|
||||
showToast(`Uploaded: ${item.name}`,'success');
|
||||
} catch(e) { setFileStatus(idx,'error','✗ Error'); item.status='error'; showToast(`Failed: ${item.name} — ${e.message}`,'error'); }
|
||||
updateStats();
|
||||
const fileQueue = [...files];
|
||||
|
||||
async function fileWorker() {
|
||||
while (fileQueue.length) {
|
||||
const item = fileQueue.shift();
|
||||
if (!item) break;
|
||||
const idx = selectedFiles.indexOf(item);
|
||||
setFileStatus(idx, 'uploading', 'Starting…');
|
||||
document.getElementById(`prog-${idx}`).style.display = 'block';
|
||||
try {
|
||||
// Try presigned direct-to-S3 first
|
||||
await uploadFilePresigned(item, idx);
|
||||
document.getElementById(`progbar-${idx}`).style.width = '100%';
|
||||
setFileStatus(idx, 'done', '✓ Done'); item.status = 'done';
|
||||
showToast(`Uploaded: ${item.name}`, 'success');
|
||||
} catch(presignedErr) {
|
||||
// Fallback to server-proxied upload
|
||||
console.warn(`[upload] Presigned failed for ${item.name}: ${presignedErr.message}, falling back to server upload`);
|
||||
try {
|
||||
setFileStatus(idx, 'uploading', 'Retrying via server…');
|
||||
if (document.getElementById(`progbar-${idx}`)) document.getElementById(`progbar-${idx}`).style.width = '0%';
|
||||
await uploadFileViaServer(item, idx);
|
||||
document.getElementById(`progbar-${idx}`).style.width = '100%';
|
||||
setFileStatus(idx, 'done', '✓ Done (server)'); item.status = 'done';
|
||||
showToast(`Uploaded: ${item.name}`, 'success');
|
||||
} catch(serverErr) {
|
||||
setFileStatus(idx, 'error', '✗ Error'); item.status = 'error';
|
||||
showToast(`Failed: ${item.name} — ${serverErr.message}`, 'error');
|
||||
}
|
||||
}
|
||||
updateStats();
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({length: Math.min(UPLOAD_CONCURRENCY, files.length)}, fileWorker));
|
||||
}
|
||||
|
||||
async function uploadUDP(files) {
|
||||
|
|
@ -1170,12 +1308,19 @@ async function loadAmppJobs() {
|
|||
list.innerHTML='<div style="color:var(--text-dim);text-align:center;padding:2rem;font-size:.85rem">No jobs in queue</div>';
|
||||
return;
|
||||
}
|
||||
// Log first job's keys for debugging field names
|
||||
if (jobs.length) console.log('[AMPP] Sample job keys:', Object.keys(jobs[0]), 'Full:', JSON.stringify(jobs[0]).substring(0, 500));
|
||||
jobs.forEach(job => {
|
||||
const el=document.createElement('div'); el.className='job-item';
|
||||
const st=(job.status||job.state||job.jobStatus||'unknown').toLowerCase();
|
||||
const cls=st.includes('run')||st.includes('active')?'running':st.includes('complet')||st.includes('success')||st.includes('done')?'completed':st.includes('fail')||st.includes('error')?'failed':st.includes('queue')||st.includes('wait')||st.includes('pend')?'queued':'unknown';
|
||||
const name=job.name||job.displayName||job.id||'Job';
|
||||
const meta=[job.created?new Date(job.created).toLocaleString():'', job.type||job.jobType||''].filter(Boolean).join(' · ');
|
||||
// AMPP API uses colon-namespaced keys e.g. "state:jobState", "name:text", "job:id"
|
||||
const st=(job['state:jobState']||job.status||job.state||job.jobStatus||'unknown').toLowerCase();
|
||||
const cls=st.includes('run')||st.includes('active')?'running':st.includes('complet')||st.includes('success')||st.includes('done')?'completed':st.includes('fail')||st.includes('error')||st.includes('abort')?'failed':st.includes('queue')||st.includes('wait')||st.includes('pend')?'queued':'unknown';
|
||||
// Try many possible asset name fields — AMPP responses vary by job type
|
||||
const name=job['name:text']||job['assetName:text']||job['source:text']||job['sourceFile:text']||job['inputFile:text']||job['input:text']||job.name||job.displayName||job.assetName||job.sourceName||job.sourceFile||job.inputFile||job['job:id']||job.id||'Job';
|
||||
const jobType=(job['type:jobType']||job['subtype:jobSubtype']||job.type||job.jobType||'').replace(/([A-Z])/g,' $1').trim();
|
||||
const creator=job['creator:id']||job.creator||'';
|
||||
const created=job['created:dateTime']||job.created||'';
|
||||
const meta=[created?new Date(created).toLocaleString():'', jobType, creator].filter(Boolean).join(' · ');
|
||||
el.innerHTML=`<div class="job-dot ${cls}"></div><div class="job-info"><div class="job-name">${esc(name)}</div><div class="job-meta">${esc(meta)}</div></div><span class="job-status ${cls}">${cls.charAt(0).toUpperCase()+cls.slice(1)}</span>`;
|
||||
list.appendChild(el);
|
||||
});
|
||||
|
|
@ -1300,6 +1445,7 @@ async function loadUsers() {
|
|||
<td style="color:var(--text-dim);font-size:.73rem;font-family:'JetBrains Mono',monospace">${u.created?new Date(u.created).toLocaleDateString():''}</td>
|
||||
<td style="display:flex;gap:.35rem;flex-wrap:wrap">
|
||||
<button class="btn-secondary" style="padding:.22rem .55rem;font-size:.68rem" onclick="openPermissions('${esc(u.username)}')">⚙ Perms</button>
|
||||
<button class="btn-secondary" style="padding:.22rem .55rem;font-size:.68rem" onclick="openUserAudit('${esc(u.username)}')">👁 Audit</button>
|
||||
${u.username!==currentUser?`<button class="btn-danger" style="padding:.22rem .55rem;font-size:.68rem" onclick="deleteUser('${esc(u.username)}')">Delete</button>`:'<span style="color:var(--text-dim);font-size:.73rem">(you)</span>'}
|
||||
</td>`;
|
||||
tbody.appendChild(tr);
|
||||
|
|
@ -1400,6 +1546,83 @@ function fmtBytes(b) {
|
|||
return (b/1073741824).toFixed(2) + ' GB';
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// USER AUDIT
|
||||
// ============================================================
|
||||
async function openUserAudit(username) {
|
||||
const modal = document.getElementById('audit-modal');
|
||||
modal.style.display = 'flex';
|
||||
document.getElementById('audit-modal-title').textContent = `Audit — ${username}`;
|
||||
const content = document.getElementById('audit-content');
|
||||
content.innerHTML = '<div style="color:var(--text-dim);font-size:.82rem">Loading…</div>';
|
||||
try {
|
||||
const [pd, fd] = await Promise.all([
|
||||
api('GET', `/api/users/${encodeURIComponent(username)}/permissions`),
|
||||
api('GET', '/api/folders')
|
||||
]);
|
||||
if (!pd.success) throw new Error(pd.error);
|
||||
const allFolders = flattenFolders(fd.tree || []);
|
||||
const allowed = pd.allowedFolders || [];
|
||||
const hasRestrictions = allowed.length > 0;
|
||||
const visibleFolders = hasRestrictions ? allFolders.filter(f => allowed.some(a => f === a || f.startsWith(a + '--'))) : allFolders;
|
||||
|
||||
let html = '';
|
||||
|
||||
// Role & Quota
|
||||
html += `<div style="display:grid;grid-template-columns:1fr 1fr;gap:.75rem;margin-bottom:1.25rem">`;
|
||||
html += `<div style="background:var(--bg-secondary);border:1px solid var(--border);border-radius:8px;padding:.75rem">
|
||||
<div style="font-size:.68rem;text-transform:uppercase;letter-spacing:.08em;color:var(--text-dim);margin-bottom:.25rem">Role</div>
|
||||
<div style="font-size:.88rem;font-weight:600"><span class="role-badge ${pd.role||'user'}">${pd.role||'user'}</span></div>
|
||||
</div>`;
|
||||
html += `<div style="background:var(--bg-secondary);border:1px solid var(--border);border-radius:8px;padding:.75rem">
|
||||
<div style="font-size:.68rem;text-transform:uppercase;letter-spacing:.08em;color:var(--text-dim);margin-bottom:.25rem">Upload Quota</div>
|
||||
<div style="font-size:.88rem;font-weight:600">${pd.quotaMB ? `${fmtBytes(pd.uploadedBytes||0)} / ${pd.quotaMB} MB` : 'Unlimited'}</div>
|
||||
</div>`;
|
||||
html += `</div>`;
|
||||
|
||||
// Visible pages
|
||||
const pages = ['Upload'];
|
||||
if (pd.role === 'admin') pages.push('AMPP Monitor', 'Admin Panel');
|
||||
else pages.push('AMPP Monitor');
|
||||
html += `<div style="margin-bottom:1.25rem">
|
||||
<div style="font-size:.72rem;text-transform:uppercase;letter-spacing:.08em;color:var(--text-dim);margin-bottom:.5rem">Visible Pages</div>
|
||||
<div style="display:flex;gap:.4rem;flex-wrap:wrap">${pages.map(p => `<span style="background:var(--accent-glow);color:var(--blue-bright);padding:.2rem .6rem;border-radius:6px;font-size:.75rem;font-weight:500">${p}</span>`).join('')}</div>
|
||||
</div>`;
|
||||
|
||||
// Admin features
|
||||
if (pd.role === 'admin') {
|
||||
html += `<div style="margin-bottom:1.25rem">
|
||||
<div style="font-size:.72rem;text-transform:uppercase;letter-spacing:.08em;color:var(--text-dim);margin-bottom:.5rem">Admin Capabilities</div>
|
||||
<div style="font-size:.8rem;color:var(--text-secondary);line-height:1.6">
|
||||
S3 Storage settings, AMPP config, User management, Share Links, Folder management, Add/delete folders on upload page
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// Folder access
|
||||
html += `<div>
|
||||
<div style="font-size:.72rem;text-transform:uppercase;letter-spacing:.08em;color:var(--text-dim);margin-bottom:.5rem">Folder Access ${hasRestrictions ? '<span style="color:var(--warning)">(restricted)</span>' : '<span style="color:var(--success)">(all folders)</span>'}</div>
|
||||
<div style="max-height:200px;overflow-y:auto;background:var(--bg-secondary);border:1px solid var(--border);border-radius:8px;padding:.5rem">`;
|
||||
if (!visibleFolders.length) {
|
||||
html += '<div style="color:var(--text-dim);font-size:.78rem;padding:.25rem">No folders configured</div>';
|
||||
} else {
|
||||
visibleFolders.forEach(f => {
|
||||
const depth = (f.match(/--/g) || []).length;
|
||||
const name = f.includes('--') ? f.split('--').pop() : f;
|
||||
const isAllowed = !hasRestrictions || allowed.includes(f);
|
||||
html += `<div style="padding:.2rem .4rem .2rem ${.4 + depth * 1}rem;font-family:'JetBrains Mono',monospace;font-size:.75rem;color:${isAllowed ? 'var(--text-secondary)' : 'var(--text-dim)'};display:flex;align-items:center;gap:.4rem">
|
||||
<span>${isAllowed ? '✅' : '🚫'}</span> ${esc(name)}
|
||||
</div>`;
|
||||
});
|
||||
}
|
||||
html += `</div></div>`;
|
||||
|
||||
content.innerHTML = html;
|
||||
} catch(e) {
|
||||
content.innerHTML = `<div style="color:var(--error);font-size:.82rem">Error: ${e.message}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SHARE LINKS
|
||||
// ============================================================
|
||||
|
|
@ -1485,10 +1708,18 @@ async function loadAdminFolders() {
|
|||
function renderAdminFolderTree(nodes,container,pathArr) {
|
||||
container.innerHTML='';
|
||||
if(!nodes.length){container.innerHTML='<div style="color:var(--text-dim);font-size:.8rem;padding:.5rem">No folders. Add one below.</div>';return;}
|
||||
nodes.forEach(n=>{
|
||||
const sorted=[...nodes].sort((a,b)=>a.name.localeCompare(b.name,undefined,{sensitivity:'base'}));
|
||||
sorted.forEach(n=>{
|
||||
const fp=[...pathArr,n.name];
|
||||
const div=document.createElement('div');div.style.paddingLeft=pathArr.length*20+'px';div.style.marginBottom='.3rem';
|
||||
div.innerHTML=`<div style="display:flex;align-items:center;gap:.5rem;padding:.3rem .5rem;border:1px solid var(--border);border-radius:7px;background:var(--bg-card)"><span style="font-size:.82rem">${n.children.length?'📁':'📄'}</span><span style="font-family:'JetBrains Mono',monospace;font-size:.78rem;flex:1">${esc(n.name)}</span><button class="btn-danger" style="padding:.18rem .48rem;font-size:.66rem" onclick="deleteFolder(${JSON.stringify(fp)})">Delete</button></div>`;
|
||||
const row=document.createElement('div');
|
||||
row.style.cssText='display:flex;align-items:center;gap:.5rem;padding:.3rem .5rem;border:1px solid var(--border);border-radius:7px;background:var(--bg-card)';
|
||||
row.innerHTML=`<span style="font-size:.82rem">${n.children.length?'📁':'📄'}</span><span style="font-family:'JetBrains Mono',monospace;font-size:.78rem;flex:1">${esc(n.name)}</span>`;
|
||||
const delBtn=document.createElement('button');
|
||||
delBtn.className='btn-danger';delBtn.style.cssText='padding:.18rem .48rem;font-size:.66rem';delBtn.textContent='Delete';
|
||||
delBtn.onclick=()=>{ deleteFolder(fp).then(()=>{loadAdminFolders();}); };
|
||||
row.appendChild(delBtn);
|
||||
div.appendChild(row);
|
||||
container.appendChild(div);
|
||||
if(n.children.length){const sub=document.createElement('div');renderAdminFolderTree(n.children,sub,fp);container.appendChild(sub);}
|
||||
});
|
||||
|
|
|
|||
241
server.js
241
server.js
|
|
@ -3,7 +3,7 @@ const express = require("express");
|
|||
const multer = require("multer");
|
||||
const path = require("path");
|
||||
const crypto = require("crypto");
|
||||
const { S3Client, PutObjectCommand, HeadBucketCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3");
|
||||
const { S3Client, PutObjectCommand, HeadBucketCommand, DeleteObjectCommand, ListObjectsV2Command, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require("@aws-sdk/client-s3");
|
||||
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
|
||||
const { Upload } = require("@aws-sdk/lib-storage");
|
||||
const { NodeHttpHandler } = require("@smithy/node-http-handler");
|
||||
|
|
@ -186,6 +186,15 @@ function cleanName(s) {
|
|||
return (s || "").trim().replace(/[^a-zA-Z0-9'\-.,&()! ]/g, "");
|
||||
}
|
||||
|
||||
// Build S3 object key from a folder prefix and filename.
|
||||
// The prefix may use "/" to separate nested folders (from the UI tree),
|
||||
// but FLX expects "--" as the folder delimiter in object keys.
|
||||
function buildS3Key(prefix, filename) {
|
||||
if (!prefix) return filename;
|
||||
const normalized = prefix.replace(/\//g, "--").replace(/[-]+$/, "");
|
||||
return `${normalized}--${filename}`;
|
||||
}
|
||||
|
||||
function findNode(pathArr) {
|
||||
let current = getTree();
|
||||
for (const segment of pathArr) {
|
||||
|
|
@ -426,56 +435,48 @@ app.put("/api/s3/config", requireAdmin, (req, res) => {
|
|||
// endpoint is optional for AWS S3 (leave blank to use AWS default)
|
||||
if (!region || !bucket || !accessKeyId)
|
||||
return res.status(400).json({ success: false, error: "region, bucket, and accessKeyId are required" });
|
||||
// First-time setup requires a secret key
|
||||
const existingSecret = db.s3Config?.secretAccessKey;
|
||||
if (!secretAccessKey && !existingSecret)
|
||||
return res.status(400).json({ success: false, error: "Secret Access Key is required (no existing secret on file)" });
|
||||
if (!db.s3Config) db.s3Config = {};
|
||||
db.s3Config.endpoint = endpoint.trim();
|
||||
db.s3Config.endpoint = (endpoint || "").trim();
|
||||
db.s3Config.region = region.trim();
|
||||
db.s3Config.bucket = bucket.trim();
|
||||
db.s3Config.accessKeyId = accessKeyId.trim();
|
||||
if (secretAccessKey) db.s3Config.secretAccessKey = secretAccessKey;
|
||||
if (secretAccessKey) db.s3Config.secretAccessKey = secretAccessKey.trim();
|
||||
saveData(db);
|
||||
initS3();
|
||||
res.json({ success: true, message: "S3 configuration saved" });
|
||||
});
|
||||
|
||||
app.post("/api/s3/test", requireAdmin, async (req, res) => {
|
||||
// Support testing with submitted credentials (may not be saved yet)
|
||||
const { endpoint, region, bucket, accessKeyId, secretAccessKey } = req.body;
|
||||
const saved = db.s3Config || {};
|
||||
const testCfg = {
|
||||
endpoint: endpoint || db.s3Config?.endpoint || "",
|
||||
region: region || db.s3Config?.region || "us-east-1",
|
||||
bucket: bucket || db.s3Config?.bucket || "",
|
||||
accessKeyId: accessKeyId || db.s3Config?.accessKeyId || "",
|
||||
secretAccessKey: secretAccessKey || db.s3Config?.secretAccessKey || "",
|
||||
endpoint: (req.body.endpoint || saved.endpoint || "").trim(),
|
||||
region: (req.body.region || saved.region || "us-east-1").trim(),
|
||||
bucket: (req.body.bucket || saved.bucket || "").trim(),
|
||||
accessKeyId: (req.body.accessKeyId || saved.accessKeyId || "").trim(),
|
||||
secretAccessKey: (req.body.secretAccessKey || saved.secretAccessKey || "").trim(),
|
||||
};
|
||||
console.log("[S3 Test] Config:", { endpoint: testCfg.endpoint, region: testCfg.region, bucket: testCfg.bucket, accessKeyId: testCfg.accessKeyId, hasSecret: !!testCfg.secretAccessKey });
|
||||
if (!testCfg.bucket || !testCfg.accessKeyId || !testCfg.secretAccessKey)
|
||||
return res.status(400).json({ success: false, error: "Bucket, Access Key ID, and Secret Key are required to test" });
|
||||
|
||||
const testClient = buildS3Client(testCfg);
|
||||
const testKey = `_dragonwind_test_${Date.now()}.txt`;
|
||||
|
||||
try {
|
||||
// Upload a tiny test file
|
||||
await testClient.send(new PutObjectCommand({
|
||||
Bucket: testCfg.bucket,
|
||||
Key: testKey,
|
||||
Body: Buffer.from("Dragon Wind S3 connection test"),
|
||||
ContentType: "text/plain",
|
||||
}));
|
||||
// Delete it immediately
|
||||
try {
|
||||
await testClient.send(new DeleteObjectCommand({ Bucket: testCfg.bucket, Key: testKey }));
|
||||
} catch (_) { /* delete failure is non-fatal */ }
|
||||
|
||||
res.json({ success: true, message: "S3 connection confirmed! Test file uploaded and deleted successfully." });
|
||||
const result = await testClient.send(new ListObjectsV2Command({ Bucket: testCfg.bucket, MaxKeys: 1 }));
|
||||
console.log("[S3 Test] ListObjects OK, KeyCount:", result.KeyCount);
|
||||
res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." });
|
||||
} catch (err) {
|
||||
let friendly = err.message;
|
||||
if (err.name === "NoSuchBucket") friendly = `Bucket "${testCfg.bucket}" does not exist`;
|
||||
else if (err.name === "InvalidAccessKeyId") friendly = "Invalid Access Key ID";
|
||||
else if (err.name === "SignatureDoesNotMatch") friendly = "Invalid Secret Access Key";
|
||||
else if (err.code === "ECONNREFUSED" || err.message?.includes("ECONNREFUSED")) friendly = "Connection refused — check endpoint URL and port";
|
||||
else if (err.code === "ENOTFOUND" || err.message?.includes("ENOTFOUND")) friendly = "Endpoint not found — check the URL";
|
||||
else if (err.message?.includes("TLS") || err.message?.includes("certificate")) friendly = "TLS/SSL error — endpoint certificate issue";
|
||||
console.error("[S3 Test] Error:", err.message);
|
||||
console.error("[S3 Test] Full error:", JSON.stringify({ name: err.name, message: err.message, code: err.Code || err.code, status: err.$metadata?.httpStatusCode }));
|
||||
let friendly = err.message || "Unknown error";
|
||||
if (err.name === "NoSuchBucket" || err.Code === "NoSuchBucket") friendly = `Bucket "${testCfg.bucket}" does not exist`;
|
||||
else if (err.name === "InvalidAccessKeyId" || err.Code === "InvalidAccessKeyId") friendly = "Invalid Access Key ID";
|
||||
else if (err.name === "SignatureDoesNotMatch" || err.Code === "SignatureDoesNotMatch") friendly = "Invalid Secret Access Key";
|
||||
else if (err.name === "AccessDenied" || err.Code === "AccessDenied") friendly = "Access denied — credentials don't have permission for this bucket";
|
||||
else if (err.message?.includes("ECONNREFUSED")) friendly = "Connection refused — check endpoint URL";
|
||||
else if (err.message?.includes("ENOTFOUND")) friendly = "Endpoint not found — check the URL";
|
||||
res.status(400).json({ success: false, error: friendly });
|
||||
}
|
||||
});
|
||||
|
|
@ -540,17 +541,17 @@ app.post("/api/upload", requireAuth, upload.array("files", 50), async (req, res)
|
|||
}
|
||||
const bucket = db.s3Config?.bucket || "";
|
||||
for (const file of req.files) {
|
||||
const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${file.originalname}` : file.originalname;
|
||||
const key = buildS3Key(prefix, file.originalname);
|
||||
const contentType = getMimeType(file.originalname, file.mimetype);
|
||||
console.log(`Uploading: ${key} (${file.size} bytes, ${contentType})`);
|
||||
const uploadPromise = new Upload({
|
||||
client: s3Client,
|
||||
params: { Bucket: bucket, Key: key, Body: fs.createReadStream(file.path), ContentType: contentType },
|
||||
queueSize: 4,
|
||||
partSize: 10 * 1024 * 1024,
|
||||
leavePartsOnError: false,
|
||||
}).done();
|
||||
const result = await withTimeout(uploadPromise, UPLOAD_TIMEOUT_MS, key);
|
||||
// Use PutObjectCommand (single PUT) — compatible with RustFS/MinIO/generic S3.
|
||||
// The @aws-sdk/lib-storage Upload class uses CreateMultipartUpload internally
|
||||
// which returns non-standard XML from RustFS and causes UnknownError.
|
||||
const fileBuffer = fs.readFileSync(file.path);
|
||||
const putPromise = s3Client.send(new PutObjectCommand({
|
||||
Bucket: bucket, Key: key, Body: fileBuffer, ContentType: contentType,
|
||||
}));
|
||||
const result = await withTimeout(putPromise, UPLOAD_TIMEOUT_MS, key);
|
||||
if (result?.assumed) console.log(`Assumed success (timeout): ${key}`);
|
||||
else console.log(`Confirmed success: ${key}`);
|
||||
try { fs.unlinkSync(file.path); } catch (_) {}
|
||||
|
|
@ -566,13 +567,28 @@ app.post("/api/upload", requireAuth, upload.array("files", 50), async (req, res)
|
|||
}
|
||||
});
|
||||
|
||||
// ==================== PRESIGNED URL (for Chrome extension HTTP mode) ====================
|
||||
// ==================== PRESIGNED URL (direct-to-S3 upload) ====================
|
||||
// Client uploads directly to RustFS/S3 using presigned PUT URLs, bypassing the Node server.
|
||||
// Server only generates the signed URL and tracks quota/permissions.
|
||||
app.post("/api/presigned", requireAuth, async (req, res) => {
|
||||
if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" });
|
||||
const { filename, prefix, contentType } = req.body;
|
||||
const { filename, prefix, contentType, size } = req.body;
|
||||
if (!filename) return res.status(400).json({ success: false, error: "filename required" });
|
||||
if (isBlockedFile(filename)) return res.status(400).json({ success: false, error: `Blocked file type: ${filename}` });
|
||||
|
||||
const user = db.users.find(u => u.username === req.sessionData.user);
|
||||
// Folder permission check
|
||||
if (user && !isFolderAllowed(user, prefix || "")) {
|
||||
return res.status(403).json({ success: false, error: "You do not have permission to upload to this folder" });
|
||||
}
|
||||
// Quota check
|
||||
if (size && user && isQuotaExceeded(user, size)) {
|
||||
const usedMB = ((user.uploadedBytes || 0) / 1024 / 1024).toFixed(1);
|
||||
return res.status(403).json({ success: false, error: `Upload quota exceeded (${usedMB} MB of ${user.quotaMB} MB used)` });
|
||||
}
|
||||
|
||||
const bucket = db.s3Config?.bucket || "";
|
||||
const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename;
|
||||
const key = buildS3Key(prefix, filename);
|
||||
const mime = contentType || getMimeType(filename, "application/octet-stream");
|
||||
try {
|
||||
const url = await getSignedUrl(s3Client, new PutObjectCommand({ Bucket: bucket, Key: key, ContentType: mime }), { expiresIn: 3600 });
|
||||
|
|
@ -583,6 +599,18 @@ app.post("/api/presigned", requireAuth, async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
// Called by client after a successful direct-to-S3 upload to update quota tracking
|
||||
app.post("/api/presigned/complete", requireAuth, (req, res) => {
|
||||
const { key, size } = req.body;
|
||||
const user = db.users.find(u => u.username === req.sessionData.user);
|
||||
if (user && size) {
|
||||
user.uploadedBytes = (user.uploadedBytes || 0) + size;
|
||||
saveData(db);
|
||||
}
|
||||
console.log(`[presigned] Completed: ${key} (${size} bytes) by ${req.sessionData.user}`);
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
||||
// ==================== UDP UPLOAD SESSION ====================
|
||||
// Sessions stored in memory; relay handles actual transfer
|
||||
const udpSessions = new Map();
|
||||
|
|
@ -595,7 +623,7 @@ app.post("/api/udp/session", requireAuth, async (req, res) => {
|
|||
if (!relayUrl) return res.status(503).json({ success: false, error: "UDP relay not configured. Go to Admin → Relay Settings." });
|
||||
const publicRelayUrl = (db.relayConfig?.publicRelayUrl || "").trim() || relayUrl;
|
||||
const sessionId = crypto.randomBytes(16).toString("hex");
|
||||
const key = (prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename);
|
||||
const key = buildS3Key(prefix, filename);
|
||||
const s3Cfg = db.s3Config || {};
|
||||
|
||||
// Register session on the relay server so it can accept chunks and upload to S3
|
||||
|
|
@ -756,7 +784,14 @@ app.get("/api/ampp/jobs", requireAuth, async (req, res) => {
|
|||
return res.json({ success: true, jobs: await r2.json() });
|
||||
}
|
||||
if (!r.ok) return res.status(r.status).json({ success: false, error: `AMPP ${r.status}` });
|
||||
res.json({ success: true, jobs: await r.json() });
|
||||
const jobData = await r.json();
|
||||
// Debug: log the first job's full structure to identify field names
|
||||
const items = Array.isArray(jobData) ? jobData : (jobData?.items || jobData?.results || []);
|
||||
if (items.length > 0) {
|
||||
console.log("[AMPP] Sample job keys:", Object.keys(items[0]));
|
||||
console.log("[AMPP] Sample job data:", JSON.stringify(items[0]).substring(0, 1000));
|
||||
}
|
||||
res.json({ success: true, jobs: jobData });
|
||||
} catch (err) {
|
||||
if (err.name === "AbortError") return res.status(504).json({ success: false, error: "AMPP timeout" });
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
|
|
@ -847,14 +882,13 @@ app.post("/api/sharelinks/:token/upload", upload.array("files", 50), async (req,
|
|||
const bucket = db.s3Config?.bucket || "";
|
||||
const results = [];
|
||||
for (const file of req.files) {
|
||||
const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${file.originalname}` : file.originalname;
|
||||
const key = buildS3Key(prefix, file.originalname);
|
||||
const contentType = getMimeType(file.originalname, file.mimetype);
|
||||
const uploadPromise = new Upload({
|
||||
client: s3Client,
|
||||
params: { Bucket: bucket, Key: key, Body: fs.createReadStream(file.path), ContentType: contentType },
|
||||
queueSize: 4, partSize: 10 * 1024 * 1024, leavePartsOnError: false,
|
||||
}).done();
|
||||
await withTimeout(uploadPromise, UPLOAD_TIMEOUT_MS, key);
|
||||
const fileBuffer = fs.readFileSync(file.path);
|
||||
const putPromise = s3Client.send(new PutObjectCommand({
|
||||
Bucket: bucket, Key: key, Body: fileBuffer, ContentType: contentType,
|
||||
}));
|
||||
await withTimeout(putPromise, UPLOAD_TIMEOUT_MS, key);
|
||||
try { fs.unlinkSync(file.path); } catch (_) {}
|
||||
results.push({ originalName: file.originalname, key, size: file.size });
|
||||
}
|
||||
|
|
@ -951,5 +985,104 @@ app.post("/api/desktop/multipart/abort", requireAuth, async (req, res) => {
|
|||
const { uploadId, key, bucket } = req.body;
|
||||
try { if (uploadId && key && bucket && s3Client) await s3Client.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId })); } catch (_) {}
|
||||
desktopSessions.delete(uploadId);
|
||||
|
||||
// ==================== PARALLEL CHUNK UPLOAD (Option 4 — HTTP parallelism) ====================
|
||||
// Client splits files into 32 MB chunks and POSTs 6 concurrently.
|
||||
// Server proxies each chunk to S3 as a multipart upload part.
|
||||
// Achieves Aspera-class throughput over plain HTTP — no UDP, no port forwarding needed.
|
||||
// Based on the same approach used by MASV.
|
||||
|
||||
// In-memory multipart session map
|
||||
// uploadId → { key, bucket, parts: [{PartNumber, ETag}], partCount }
|
||||
const chunkSessions = new Map();
|
||||
|
||||
// 1. Initiate — creates S3 multipart upload, returns uploadId to client
|
||||
app.post("/api/upload/initiate", requireAuth, async (req, res) => {
|
||||
if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" });
|
||||
const { filename, prefix, contentType, totalParts } = req.body;
|
||||
if (!filename || !totalParts) return res.status(400).json({ success: false, error: "filename and totalParts required" });
|
||||
if (isBlockedFile(filename)) return res.status(400).json({ success: false, error: `Blocked file type: ${filename}` });
|
||||
|
||||
const bucket = db.s3Config?.bucket || "";
|
||||
if (!bucket) return res.status(503).json({ success: false, error: "S3 bucket not configured" });
|
||||
|
||||
const key = buildS3Key(prefix, filename);
|
||||
const mime = contentType || getMimeType(filename, "application/octet-stream");
|
||||
|
||||
try {
|
||||
const resp = await s3Client.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ContentType: mime }));
|
||||
const uploadId = resp.UploadId;
|
||||
chunkSessions.set(uploadId, { key, bucket, parts: [], partCount: parseInt(totalParts) });
|
||||
setTimeout(() => chunkSessions.delete(uploadId), 4 * 60 * 60 * 1000); // 4-hour TTL
|
||||
console.log(`[chunk] Initiated: ${key} — ${totalParts} parts, uploadId=${uploadId}`);
|
||||
res.json({ success: true, uploadId, key });
|
||||
} catch (err) {
|
||||
console.error("[chunk] Initiate error:", err.message);
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 2. Chunk — receive one chunk (multipart/form-data), proxy to S3
|
||||
const chunkMulter = multer({ storage: multer.memoryStorage(), limits: { fileSize: 200 * 1024 * 1024 } });
|
||||
app.post("/api/upload/chunk", requireAuth, chunkMulter.single("chunk"), async (req, res) => {
|
||||
if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" });
|
||||
const { uploadId, partNumber } = req.body;
|
||||
const partNum = parseInt(partNumber);
|
||||
if (!uploadId || !partNum) return res.status(400).json({ success: false, error: "uploadId and partNumber required" });
|
||||
|
||||
const session = chunkSessions.get(uploadId);
|
||||
if (!session) return res.status(404).json({ success: false, error: "Session not found or expired" });
|
||||
|
||||
const body = req.file?.buffer;
|
||||
if (!body || body.length === 0) return res.status(400).json({ success: false, error: "No chunk data" });
|
||||
|
||||
try {
|
||||
const resp = await s3Client.send(new UploadPartCommand({
|
||||
Bucket: session.bucket, Key: session.key,
|
||||
UploadId: uploadId, PartNumber: partNum, Body: body,
|
||||
}));
|
||||
session.parts.push({ PartNumber: partNum, ETag: resp.ETag });
|
||||
console.log(`[chunk] Part ${partNum}/${session.partCount} OK for ${session.key}`);
|
||||
res.json({ success: true, partNumber: partNum, etag: resp.ETag });
|
||||
} catch (err) {
|
||||
console.error(`[chunk] Part ${partNum} error:`, err.message);
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 3. Complete — assemble all parts into final S3 object
|
||||
app.post("/api/upload/complete", requireAuth, async (req, res) => {
|
||||
if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" });
|
||||
const { uploadId } = req.body;
|
||||
if (!uploadId) return res.status(400).json({ success: false, error: "uploadId required" });
|
||||
|
||||
const session = chunkSessions.get(uploadId);
|
||||
if (!session) return res.status(404).json({ success: false, error: "Session not found or expired" });
|
||||
|
||||
const parts = session.parts.slice().sort((a, b) => a.PartNumber - b.PartNumber);
|
||||
try {
|
||||
await s3Client.send(new CompleteMultipartUploadCommand({
|
||||
Bucket: session.bucket, Key: session.key, UploadId: uploadId,
|
||||
MultipartUpload: { Parts: parts },
|
||||
}));
|
||||
chunkSessions.delete(uploadId);
|
||||
console.log(`[chunk] Completed: ${session.key} (${parts.length} parts)`);
|
||||
res.json({ success: true, key: session.key });
|
||||
} catch (err) {
|
||||
console.error("[chunk] Complete error:", err.message);
|
||||
try { await s3Client.send(new AbortMultipartUploadCommand({ Bucket: session.bucket, Key: session.key, UploadId: uploadId })); } catch (_) {}
|
||||
chunkSessions.delete(uploadId);
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 4. Abort — cancel a multipart upload (called on client-side error)
|
||||
app.post("/api/upload/abort", requireAuth, async (req, res) => {
|
||||
const { uploadId } = req.body;
|
||||
const session = chunkSessions.get(uploadId);
|
||||
if (session && s3Client) {
|
||||
try { await s3Client.send(new AbortMultipartUploadCommand({ Bucket: session.bucket, Key: session.key, UploadId: uploadId })); } catch (_) {}
|
||||
}
|
||||
if (uploadId) chunkSessions.delete(uploadId);
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue