From 8ec43c299e6ee93f553515b7e6d9c7af3c1743ca Mon Sep 17 00:00:00 2001 From: Zac Gaetano Date: Mon, 6 Apr 2026 23:22:51 -0400 Subject: [PATCH 01/17] =?UTF-8?q?Add=20parallel=20chunked=20HTTP=20upload?= =?UTF-8?q?=20(Option=204=20=E2=80=94=20Aspera-class=20speed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split files into 32 MB chunks, POST 6 concurrently to /api/upload/chunk, server proxies each chunk as an S3 multipart part. Up to 4 files upload in parallel simultaneously. Achieves Aspera-class throughput over plain HTTP with no UDP port forwarding or custom protocols required. Same approach used by MASV under the hood. - server.js: Add /api/upload/initiate, /chunk, /complete, /abort endpoints - public/index.html: Replace single-PUT uploadHTTP() with parallel chunked version --- public/index.html | 119 ++++++++++++++++++++++++++++++++++++---------- server.js | 108 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 25 deletions(-) diff --git a/public/index.html b/public/index.html index f7a970a..555e4dc 100644 --- a/public/index.html +++ b/public/index.html @@ -467,7 +467,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- HTTP Mode: Direct S3 presigned upload. Best for LAN and stable connections. + HTTP Mode: Parallel chunked HTTP upload (6 concurrent 32 MB parts). Aspera-class speed — no UDP needed.
@@ -932,7 +932,7 @@ function setMode(mode) { if (mode === 'http') { btn.className = 'btn-upload'; if (label) label.textContent = 'HTTP Mode:'; - if (detail) detail.textContent = 'Direct S3 presigned upload. Best for LAN and stable connections.'; + if (detail) detail.textContent = 'Parallel chunked HTTP upload (6 concurrent 32 MB parts). Aspera-class speed — no UDP needed.'; if (hint) hint.style.display = 'none'; if (btnHttp) { btnHttp.className = 'mode-btn active-http'; } if (btnUdp) { btnUdp.className = 'mode-btn'; } @@ -1087,31 +1087,100 @@ async function startUpload() { updateUploadBtn(); updateStats(); } -async function uploadHTTP(files) { - for (const item of files) { - const idx = selectedFiles.indexOf(item); - setFileStatus(idx,'uploading','Uploading…'); - document.getElementById(`prog-${idx}`).style.display='block'; - try { - const presigned = await api('POST','/api/presigned',{filename:item.name,prefix:selectedPrefix,contentType:item.file.type||'application/octet-stream'}); - if (!presigned.success) throw new Error(presigned.error||'Failed to get presigned URL'); - // Use the content type the server signed — browser file.type may differ for broadcast formats - const signedType = presigned.contentType || item.file.type || 'application/octet-stream'; - await new Promise((resolve,reject) => { - const xhr=new XMLHttpRequest(); - xhr.open('PUT',presigned.url); - xhr.setRequestHeader('Content-Type',signedType); - xhr.upload.onprogress=e=>{ if(e.lengthComputable){ const p=Math.round(e.loaded/e.total*100); document.getElementById(`progbar-${idx}`).style.width=p+'%'; setFileStatus(idx,'uploading',p+'%'); } }; - xhr.onload=()=>xhr.status<300?resolve():reject(new Error(`S3 error ${xhr.status}`)); - xhr.onerror=()=>reject(new Error('Network error')); - xhr.send(item.file); +// ============================================================ +// PARALLEL CHUNK UPLOAD (Option 4 — HTTP parallelism, Aspera-class speed) +// Slices each file into 32 MB chunks, uploads 6 concurrently via POST, +// server proxies to S3 multipart. Same approach as MASV — no UDP needed. +// ============================================================ +const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per part +const CHUNK_WORKERS = 6; // concurrent chunk POSTs per file + +async function uploadFileChunked(item, idx) { + const file = item.file; + const totalParts = Math.ceil(file.size / CHUNK_SIZE); + const mime = file.type || 'application/octet-stream'; + + // 1. Initiate S3 multipart upload + const init = await api('POST','/api/upload/initiate',{ + filename: item.name, prefix: selectedPrefix, + contentType: mime, totalParts, + }); + if (!init.success) throw new Error(init.error || 'Failed to initiate upload'); + const { uploadId } = init; + + const pb = document.getElementById(`progbar-${idx}`); + let done = 0; + + // 2. Upload all parts with bounded concurrency + const queue = Array.from({length: totalParts}, (_,i) => i+1); // 1-indexed + + async function worker() { + while (queue.length) { + const partNumber = queue.shift(); + if (partNumber === undefined) break; + const start = (partNumber - 1) * CHUNK_SIZE; + const blob = file.slice(start, Math.min(start + CHUNK_SIZE, file.size)); + const fd = new FormData(); + fd.append('uploadId', uploadId); + fd.append('partNumber', String(partNumber)); + fd.append('chunk', blob, item.name); + const resp = await fetch('/api/upload/chunk', { + method: 'POST', + headers: { 'x-auth-token': authToken }, + body: fd, }); - document.getElementById(`progbar-${idx}`).style.width='100%'; - setFileStatus(idx,'done','✓ Done'); item.status='done'; - showToast(`Uploaded: ${item.name}`,'success'); - } catch(e) { setFileStatus(idx,'error','✗ Error'); item.status='error'; showToast(`Failed: ${item.name} — ${e.message}`,'error'); } - updateStats(); + const data = await resp.json(); + if (!data.success) throw new Error(`Part ${partNumber} failed: ${data.error}`); + done++; + const pct = Math.round(done / totalParts * 100); + if (pb) pb.style.width = pct + '%'; + setFileStatus(idx, 'uploading', pct + '%'); + } } + + try { + await Promise.all(Array.from({length: Math.min(CHUNK_WORKERS, totalParts)}, worker)); + } catch (err) { + // Abort orphaned multipart upload on any chunk failure + fetch('/api/upload/abort', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-auth-token': authToken }, + body: JSON.stringify({ uploadId }), + }).catch(() => {}); + throw err; + } + + // 3. Complete the multipart upload + const complete = await api('POST','/api/upload/complete',{ uploadId }); + if (!complete.success) throw new Error(complete.error || 'Failed to complete upload'); +} + +async function uploadHTTP(files) { + // Upload files with up to 4 concurrent file uploads + const FILE_CONCURRENCY = 4; + const fileQueue = [...files]; + + async function fileWorker() { + while (fileQueue.length) { + const item = fileQueue.shift(); + if (!item) break; + const idx = selectedFiles.indexOf(item); + setFileStatus(idx,'uploading','Uploading…'); + document.getElementById(`prog-${idx}`).style.display='block'; + try { + await uploadFileChunked(item, idx); + document.getElementById(`progbar-${idx}`).style.width='100%'; + setFileStatus(idx,'done','✓ Done'); item.status='done'; + showToast(`Uploaded: ${item.name}`,'success'); + } catch(e) { + setFileStatus(idx,'error','✗ Error'); item.status='error'; + showToast(`Failed: ${item.name} — ${e.message}`,'error'); + } + updateStats(); + } + } + + await Promise.all(Array.from({length: Math.min(FILE_CONCURRENCY, files.length)}, fileWorker)); } async function uploadUDP(files) { diff --git a/server.js b/server.js index e8940c6..646d390 100644 --- a/server.js +++ b/server.js @@ -918,3 +918,111 @@ server.timeout = 0; server.keepAliveTimeout = 0; server.headersTimeout = 0; server.requestTimeout = 0; + +// ==================== PARALLEL CHUNK UPLOAD (Option 4 — HTTP parallelism) ==================== +// Client splits files into 32 MB chunks and POSTs 6 concurrently. +// Server proxies each chunk to S3 as a multipart upload part. +// Achieves Aspera-class throughput over plain HTTP — no UDP, no port forwarding needed. +// Based on the same approach used by MASV. + +const { + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, +} = require("@aws-sdk/client-s3"); + +// In-memory multipart session map +// uploadId → { key, bucket, parts: [{PartNumber, ETag}], partCount } +const chunkSessions = new Map(); + +// 1. Initiate — creates S3 multipart upload, returns uploadId to client +app.post("/api/upload/initiate", requireAuth, async (req, res) => { + if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" }); + const { filename, prefix, contentType, totalParts } = req.body; + if (!filename || !totalParts) return res.status(400).json({ success: false, error: "filename and totalParts required" }); + if (isBlockedFile(filename)) return res.status(400).json({ success: false, error: `Blocked file type: ${filename}` }); + + const bucket = db.s3Config?.bucket || ""; + if (!bucket) return res.status(503).json({ success: false, error: "S3 bucket not configured" }); + + const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename; + const mime = contentType || getMimeType(filename, "application/octet-stream"); + + try { + const resp = await s3Client.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ContentType: mime })); + const uploadId = resp.UploadId; + chunkSessions.set(uploadId, { key, bucket, parts: [], partCount: parseInt(totalParts) }); + setTimeout(() => chunkSessions.delete(uploadId), 4 * 60 * 60 * 1000); // 4-hour TTL + console.log(`[chunk] Initiated: ${key} — ${totalParts} parts, uploadId=${uploadId}`); + res.json({ success: true, uploadId, key }); + } catch (err) { + console.error("[chunk] Initiate error:", err.message); + res.status(500).json({ success: false, error: err.message }); + } +}); + +// 2. Chunk — receive one chunk (multipart/form-data), proxy to S3 +const chunkMulter = multer({ storage: multer.memoryStorage(), limits: { fileSize: 200 * 1024 * 1024 } }); +app.post("/api/upload/chunk", requireAuth, chunkMulter.single("chunk"), async (req, res) => { + if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" }); + const { uploadId, partNumber } = req.body; + const partNum = parseInt(partNumber); + if (!uploadId || !partNum) return res.status(400).json({ success: false, error: "uploadId and partNumber required" }); + + const session = chunkSessions.get(uploadId); + if (!session) return res.status(404).json({ success: false, error: "Session not found or expired" }); + + const body = req.file?.buffer; + if (!body || body.length === 0) return res.status(400).json({ success: false, error: "No chunk data" }); + + try { + const resp = await s3Client.send(new UploadPartCommand({ + Bucket: session.bucket, Key: session.key, + UploadId: uploadId, PartNumber: partNum, Body: body, + })); + session.parts.push({ PartNumber: partNum, ETag: resp.ETag }); + console.log(`[chunk] Part ${partNum}/${session.partCount} OK for ${session.key}`); + res.json({ success: true, partNumber: partNum, etag: resp.ETag }); + } catch (err) { + console.error(`[chunk] Part ${partNum} error:`, err.message); + res.status(500).json({ success: false, error: err.message }); + } +}); + +// 3. Complete — assemble all parts into final S3 object +app.post("/api/upload/complete", requireAuth, async (req, res) => { + if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" }); + const { uploadId } = req.body; + if (!uploadId) return res.status(400).json({ success: false, error: "uploadId required" }); + + const session = chunkSessions.get(uploadId); + if (!session) return res.status(404).json({ success: false, error: "Session not found or expired" }); + + const parts = session.parts.slice().sort((a, b) => a.PartNumber - b.PartNumber); + try { + await s3Client.send(new CompleteMultipartUploadCommand({ + Bucket: session.bucket, Key: session.key, UploadId: uploadId, + MultipartUpload: { Parts: parts }, + })); + chunkSessions.delete(uploadId); + console.log(`[chunk] Completed: ${session.key} (${parts.length} parts)`); + res.json({ success: true, key: session.key }); + } catch (err) { + console.error("[chunk] Complete error:", err.message); + try { await s3Client.send(new AbortMultipartUploadCommand({ Bucket: session.bucket, Key: session.key, UploadId: uploadId })); } catch (_) {} + chunkSessions.delete(uploadId); + res.status(500).json({ success: false, error: err.message }); + } +}); + +// 4. Abort — cancel a multipart upload (called on client-side error) +app.post("/api/upload/abort", requireAuth, async (req, res) => { + const { uploadId } = req.body; + const session = chunkSessions.get(uploadId); + if (session && s3Client) { + try { await s3Client.send(new AbortMultipartUploadCommand({ Bucket: session.bucket, Key: session.key, UploadId: uploadId })); } catch (_) {} + } + if (uploadId) chunkSessions.delete(uploadId); + res.json({ success: true }); +}); From 172250b279bcc961ae3a131d200f6cc0e875fb07 Mon Sep 17 00:00:00 2001 From: Zac Gaetano Date: Mon, 6 Apr 2026 23:30:00 -0400 Subject: [PATCH 02/17] fix: AMPP Monitor field mapping for Grassvalley API colon-namespaced keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AMPP API returns keys like 'state:jobState', 'name:text', 'type:jobType', 'creator:id', 'created:dateTime', 'job:id' — not plain 'status', 'name', etc. Frontend was falling back to 'Job' / 'Unknown' for every entry. Updated field lookups to read colon-namespaced keys first, with fallbacks for compatibility. Also added 'aborted' to the failed status detection. --- public/index.html | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/public/index.html b/public/index.html index 555e4dc..a315813 100644 --- a/public/index.html +++ b/public/index.html @@ -1241,10 +1241,14 @@ async function loadAmppJobs() { } jobs.forEach(job => { const el=document.createElement('div'); el.className='job-item'; - const st=(job.status||job.state||job.jobStatus||'unknown').toLowerCase(); - const cls=st.includes('run')||st.includes('active')?'running':st.includes('complet')||st.includes('success')||st.includes('done')?'completed':st.includes('fail')||st.includes('error')?'failed':st.includes('queue')||st.includes('wait')||st.includes('pend')?'queued':'unknown'; - const name=job.name||job.displayName||job.id||'Job'; - const meta=[job.created?new Date(job.created).toLocaleString():'', job.type||job.jobType||''].filter(Boolean).join(' · '); + // AMPP API uses colon-namespaced keys e.g. "state:jobState", "name:text", "job:id" + const st=(job['state:jobState']||job.status||job.state||job.jobStatus||'unknown').toLowerCase(); + const cls=st.includes('run')||st.includes('active')?'running':st.includes('complet')||st.includes('success')||st.includes('done')?'completed':st.includes('fail')||st.includes('error')||st.includes('abort')?'failed':st.includes('queue')||st.includes('wait')||st.includes('pend')?'queued':'unknown'; + const name=job['name:text']||job['assetName:text']||job.name||job.displayName||job['job:id']||job.id||'Job'; + const jobType=(job['type:jobType']||job['subtype:jobSubtype']||job.type||job.jobType||'').replace(/([A-Z])/g,' $1').trim(); + const creator=job['creator:id']||''; + const created=job['created:dateTime']||job.created||''; + const meta=[created?new Date(created).toLocaleString():'', jobType, creator].filter(Boolean).join(' · '); el.innerHTML=`
${esc(name)}
${esc(meta)}
${cls.charAt(0).toUpperCase()+cls.slice(1)}`; list.appendChild(el); }); From 2b713fd169f9ceeecc351f9724fe71b370ab8100 Mon Sep 17 00:00:00 2001 From: Zac Gaetano Date: Mon, 6 Apr 2026 23:32:08 -0400 Subject: [PATCH 03/17] fix: remove duplicate require causing SyntaxError on startup CreateMultipartUploadCommand etc. were already declared at top of file. Appended block re-declared them causing 'Identifier already declared' crash. --- server.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/server.js b/server.js index 646d390..4132474 100644 --- a/server.js +++ b/server.js @@ -3,7 +3,7 @@ const express = require("express"); const multer = require("multer"); const path = require("path"); const crypto = require("crypto"); -const { S3Client, PutObjectCommand, HeadBucketCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3"); +const { S3Client, PutObjectCommand, HeadBucketCommand, DeleteObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require("@aws-sdk/client-s3"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); const { Upload } = require("@aws-sdk/lib-storage"); const { NodeHttpHandler } = require("@smithy/node-http-handler"); @@ -925,13 +925,6 @@ server.requestTimeout = 0; // Achieves Aspera-class throughput over plain HTTP — no UDP, no port forwarding needed. // Based on the same approach used by MASV. -const { - CreateMultipartUploadCommand, - UploadPartCommand, - CompleteMultipartUploadCommand, - AbortMultipartUploadCommand, -} = require("@aws-sdk/client-s3"); - // In-memory multipart session map // uploadId → { key, bucket, parts: [{PartNumber, ETag}], partCount } const chunkSessions = new Map(); From f3aa44104c1f9038e7f379eaaed24f46aa1ee9de Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:21:31 -0400 Subject: [PATCH 04/17] Remove Extension tab, fix S3 save, update footer, add Electron uploader coming soon to UDP Relay --- public/index.html | 13 ++++++++----- server.js | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/public/index.html b/public/index.html index a315813..75b6f57 100644 --- a/public/index.html +++ b/public/index.html @@ -521,11 +521,10 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
S3 Storage
AMPP
-
🧩 Extension
Users
🔗 Share Links
Folders
-
⚡ UDP Relay
+
⚡ UDP Relay COMING SOON
@@ -556,6 +555,10 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
UDP Relay Configuration
+
+
⚡ Coming Soon — Electron Desktop Uploader
+

UDP relay mode is being replaced by a native Electron desktop app that delivers Aspera-class transfer speeds over a direct connection — no browser limitations, no port forwarding, and significantly higher throughput. Configuration below is reserved for the upcoming release.

+
Not checked
Internal URL the server uses to reach the relay container (Docker service name or localhost)
The externally reachable URL for the relay — sent to uploaders' browsers. Must be reachable on port 3001 from the internet.
@@ -587,8 +590,8 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- -
+ +
Chrome Extension

The Chrome extension is required to use fast UDP uploads. Install it once in Chrome — no other setup needed on your end.

@@ -724,7 +727,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- Built by Zac Gaetano & Wild Dragon LLC + Built by Zac Gaetano · In partnership with Broadcast Management Group
diff --git a/server.js b/server.js index 4132474..de32bc5 100644 --- a/server.js +++ b/server.js @@ -427,7 +427,7 @@ app.put("/api/s3/config", requireAdmin, (req, res) => { if (!region || !bucket || !accessKeyId) return res.status(400).json({ success: false, error: "region, bucket, and accessKeyId are required" }); if (!db.s3Config) db.s3Config = {}; - db.s3Config.endpoint = endpoint.trim(); + db.s3Config.endpoint = (endpoint || "").trim(); db.s3Config.region = region.trim(); db.s3Config.bucket = bucket.trim(); db.s3Config.accessKeyId = accessKeyId.trim(); From db25255472a9857396800f7c3fb280db25881aaf Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:26:43 -0400 Subject: [PATCH 05/17] Fix S3 save, remove UDP relay entirely, remove Extension tab --- docker-compose.yml | 26 +------------------------- public/index.html | 4 +--- server.js | 6 +++++- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d442d74..1f85ba3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,7 @@ version: "3.9" # ============================================================= -# Dragon Wind — Full Stack -# Services: -# dragon-wind — Main upload web app (port 3000) -# udp-relay — UDP relay server (TCP 3001 + UDP 5000) +# Dragon Wind — Upload Portal # ============================================================= services: @@ -30,30 +27,9 @@ services: - S3_BUCKET=${S3_BUCKET:-} - S3_ACCESS_KEY=${S3_ACCESS_KEY:-} - S3_SECRET_KEY=${S3_SECRET_KEY:-} - # Relay URL for UDP mode - - RELAY_URL=${RELAY_URL:-http://udp-relay:3001} - - UDP_PORT=${UDP_PORT:-5000} # AMPP (optional) - AMPP_BASE_URL=${AMPP_BASE_URL:-https://us-east-1.gvampp.com} - AMPP_API_KEY=${AMPP_API_KEY:-} - depends_on: - - udp-relay - networks: - - dragon-wind-net - - udp-relay: - build: - context: ./udp-relay - dockerfile: Dockerfile - container_name: dragon-wind-relay - restart: unless-stopped - ports: - - "${RELAY_TCP_PORT:-3001}:3001" # Control API (TCP) - - "${RELAY_UDP_PORT:-5000}:5000/udp" # Data transfer (UDP) - environment: - - PORT=3001 - - UDP_PORT=5000 - - MAX_SESSIONS=${MAX_RELAY_SESSIONS:-50} networks: - dragon-wind-net diff --git a/public/index.html b/public/index.html index 75b6f57..257c68b 100644 --- a/public/index.html +++ b/public/index.html @@ -468,7 +468,6 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
HTTP Mode: Parallel chunked HTTP upload (6 concurrent 32 MB parts). Aspera-class speed — no UDP needed. -
@@ -524,7 +523,6 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
Users
🔗 Share Links
Folders
-
⚡ UDP Relay COMING SOON
@@ -553,7 +551,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips -
+
UDP Relay Configuration
⚡ Coming Soon — Electron Desktop Uploader
diff --git a/server.js b/server.js index de32bc5..c34727d 100644 --- a/server.js +++ b/server.js @@ -426,12 +426,16 @@ app.put("/api/s3/config", requireAdmin, (req, res) => { // endpoint is optional for AWS S3 (leave blank to use AWS default) if (!region || !bucket || !accessKeyId) return res.status(400).json({ success: false, error: "region, bucket, and accessKeyId are required" }); + // First-time setup requires a secret key + const existingSecret = db.s3Config?.secretAccessKey; + if (!secretAccessKey && !existingSecret) + return res.status(400).json({ success: false, error: "Secret Access Key is required (no existing secret on file)" }); if (!db.s3Config) db.s3Config = {}; db.s3Config.endpoint = (endpoint || "").trim(); db.s3Config.region = region.trim(); db.s3Config.bucket = bucket.trim(); db.s3Config.accessKeyId = accessKeyId.trim(); - if (secretAccessKey) db.s3Config.secretAccessKey = secretAccessKey; + if (secretAccessKey) db.s3Config.secretAccessKey = secretAccessKey.trim(); saveData(db); initS3(); res.json({ success: true, message: "S3 configuration saved" }); From ae4360e85a4b649b544d502130bf4d76c90c51f9 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:30:28 -0400 Subject: [PATCH 06/17] Fix S3 test (use HeadBucket like VPM-Uploader), replace UDP button with grayed Electron App --- public/index.html | 3 ++- server.js | 16 +++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/public/index.html b/public/index.html index 257c68b..3468742 100644 --- a/public/index.html +++ b/public/index.html @@ -464,11 +464,12 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- +
HTTP Mode: Parallel chunked HTTP upload (6 concurrent 32 MB parts). Aspera-class speed — no UDP needed.
+
Electron App — Aspera-speed desktop application (coming soon)
diff --git a/server.js b/server.js index c34727d..c64490f 100644 --- a/server.js +++ b/server.js @@ -455,22 +455,12 @@ app.post("/api/s3/test", requireAdmin, async (req, res) => { return res.status(400).json({ success: false, error: "Bucket, Access Key ID, and Secret Key are required to test" }); const testClient = buildS3Client(testCfg); - const testKey = `_dragonwind_test_${Date.now()}.txt`; try { - // Upload a tiny test file - await testClient.send(new PutObjectCommand({ - Bucket: testCfg.bucket, - Key: testKey, - Body: Buffer.from("Dragon Wind S3 connection test"), - ContentType: "text/plain", - })); - // Delete it immediately - try { - await testClient.send(new DeleteObjectCommand({ Bucket: testCfg.bucket, Key: testKey })); - } catch (_) { /* delete failure is non-fatal */ } + // Test by checking if bucket exists and is accessible + await testClient.send(new HeadBucketCommand({ Bucket: testCfg.bucket })); - res.json({ success: true, message: "S3 connection confirmed! Test file uploaded and deleted successfully." }); + res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." }); } catch (err) { let friendly = err.message; if (err.name === "NoSuchBucket") friendly = `Bucket "${testCfg.bucket}" does not exist`; From 89291a4baf58fc78772a5c7e2ba5e3452224b8cd Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:32:59 -0400 Subject: [PATCH 07/17] Fix S3 test: always merge with saved config, add debug logging --- server.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server.js b/server.js index c64490f..c7be6c7 100644 --- a/server.js +++ b/server.js @@ -442,22 +442,22 @@ app.put("/api/s3/config", requireAdmin, (req, res) => { }); app.post("/api/s3/test", requireAdmin, async (req, res) => { - // Support testing with submitted credentials (may not be saved yet) - const { endpoint, region, bucket, accessKeyId, secretAccessKey } = req.body; + // Merge submitted values with saved config — secret is often blank (keep existing) + const saved = db.s3Config || {}; const testCfg = { - endpoint: endpoint || db.s3Config?.endpoint || "", - region: region || db.s3Config?.region || "us-east-1", - bucket: bucket || db.s3Config?.bucket || "", - accessKeyId: accessKeyId || db.s3Config?.accessKeyId || "", - secretAccessKey: secretAccessKey || db.s3Config?.secretAccessKey || "", + endpoint: (req.body.endpoint || saved.endpoint || "").trim(), + region: (req.body.region || saved.region || "us-east-1").trim(), + bucket: (req.body.bucket || saved.bucket || "").trim(), + accessKeyId: (req.body.accessKeyId || saved.accessKeyId || "").trim(), + secretAccessKey: (req.body.secretAccessKey || saved.secretAccessKey || "").trim(), }; + console.log("[S3 Test] Config:", { endpoint: testCfg.endpoint, region: testCfg.region, bucket: testCfg.bucket, accessKeyId: testCfg.accessKeyId, hasSecret: !!testCfg.secretAccessKey }); if (!testCfg.bucket || !testCfg.accessKeyId || !testCfg.secretAccessKey) return res.status(400).json({ success: false, error: "Bucket, Access Key ID, and Secret Key are required to test" }); const testClient = buildS3Client(testCfg); try { - // Test by checking if bucket exists and is accessible await testClient.send(new HeadBucketCommand({ Bucket: testCfg.bucket })); res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." }); From 4b0f56ac2a281b3cb3c6e11b9010ad768c0a634d Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:34:46 -0400 Subject: [PATCH 08/17] Add full S3 error dump for debugging --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index c7be6c7..63437a1 100644 --- a/server.js +++ b/server.js @@ -469,7 +469,7 @@ app.post("/api/s3/test", requireAdmin, async (req, res) => { else if (err.code === "ECONNREFUSED" || err.message?.includes("ECONNREFUSED")) friendly = "Connection refused — check endpoint URL and port"; else if (err.code === "ENOTFOUND" || err.message?.includes("ENOTFOUND")) friendly = "Endpoint not found — check the URL"; else if (err.message?.includes("TLS") || err.message?.includes("certificate")) friendly = "TLS/SSL error — endpoint certificate issue"; - console.error("[S3 Test] Error:", err.message); + console.error("[S3 Test] Error:", err.message, "| name:", err.name, "| code:", err.Code || err.code, "| statusCode:", err.$metadata?.httpStatusCode, "| full:", JSON.stringify(err, Object.getOwnPropertyNames(err))); res.status(400).json({ success: false, error: friendly }); } }); From 1bc30010a443cc49a8ff8fe9d0145eaf6f1c7bf5 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:38:44 -0400 Subject: [PATCH 09/17] Replace AWS SDK test with raw S3v4 signed HTTP request for RustFS/MinIO compat --- server.js | 81 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/server.js b/server.js index 63437a1..a6e8523 100644 --- a/server.js +++ b/server.js @@ -455,22 +455,77 @@ app.post("/api/s3/test", requireAdmin, async (req, res) => { if (!testCfg.bucket || !testCfg.accessKeyId || !testCfg.secretAccessKey) return res.status(400).json({ success: false, error: "Bucket, Access Key ID, and Secret Key are required to test" }); - const testClient = buildS3Client(testCfg); - + // Use raw AWS Signature V4 HTTP request — compatible with RustFS/MinIO/generic S3 try { - await testClient.send(new HeadBucketCommand({ Bucket: testCfg.bucket })); + const url = new URL(`${testCfg.endpoint}/${testCfg.bucket}/`); + const now = new Date(); + const dateStamp = now.toISOString().replace(/[-:]/g, "").split(".")[0] + "Z"; + const shortDate = dateStamp.slice(0, 8); + const host = url.host; + const method = "GET"; + const canonicalUri = `/${testCfg.bucket}/`; - res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." }); + // AWS Signature V4 + function hmacSha256(key, data) { + return crypto.createHmac("sha256", key).update(data).digest(); + } + function sha256Hex(data) { + return crypto.createHash("sha256").update(data).digest("hex"); + } + + const credentialScope = `${shortDate}/${testCfg.region}/s3/aws4_request`; + const canonicalHeaders = `host:${host}\nx-amz-content-sha256:UNSIGNED-PAYLOAD\nx-amz-date:${dateStamp}\n`; + const signedHeaders = "host;x-amz-content-sha256;x-amz-date"; + const canonicalRequest = `${method}\n${canonicalUri}\n\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`; + const stringToSign = `AWS4-HMAC-SHA256\n${dateStamp}\n${credentialScope}\n${sha256Hex(canonicalRequest)}`; + + const signingKey = hmacSha256( + hmacSha256( + hmacSha256( + hmacSha256(`AWS4${testCfg.secretAccessKey}`, shortDate), + testCfg.region + ), + "s3" + ), + "aws4_request" + ); + const signature = hmacSha256(signingKey, stringToSign).toString("hex"); + const authHeader = `AWS4-HMAC-SHA256 Credential=${testCfg.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; + + const fetchUrl = `${testCfg.endpoint}/${testCfg.bucket}/?list-type=2&max-keys=1`; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 10000); + + const fetchRes = await fetch(fetchUrl, { + method: "GET", + headers: { + "Host": host, + "x-amz-date": dateStamp, + "x-amz-content-sha256": "UNSIGNED-PAYLOAD", + "Authorization": authHeader, + }, + signal: controller.signal, + }); + clearTimeout(timeout); + + const body = await fetchRes.text(); + console.log("[S3 Test] Status:", fetchRes.status, "Body:", body.slice(0, 300)); + + if (fetchRes.ok) { + res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." }); + } else if (fetchRes.status === 403) { + res.status(400).json({ success: false, error: "Access denied — check your Access Key and Secret Key" }); + } else if (fetchRes.status === 404) { + res.status(400).json({ success: false, error: `Bucket "${testCfg.bucket}" does not exist` }); + } else { + res.status(400).json({ success: false, error: `S3 returned ${fetchRes.status}: ${body.slice(0, 200)}` }); + } } catch (err) { - let friendly = err.message; - if (err.name === "NoSuchBucket") friendly = `Bucket "${testCfg.bucket}" does not exist`; - else if (err.name === "InvalidAccessKeyId") friendly = "Invalid Access Key ID"; - else if (err.name === "SignatureDoesNotMatch") friendly = "Invalid Secret Access Key"; - else if (err.code === "ECONNREFUSED" || err.message?.includes("ECONNREFUSED")) friendly = "Connection refused — check endpoint URL and port"; - else if (err.code === "ENOTFOUND" || err.message?.includes("ENOTFOUND")) friendly = "Endpoint not found — check the URL"; - else if (err.message?.includes("TLS") || err.message?.includes("certificate")) friendly = "TLS/SSL error — endpoint certificate issue"; - console.error("[S3 Test] Error:", err.message, "| name:", err.name, "| code:", err.Code || err.code, "| statusCode:", err.$metadata?.httpStatusCode, "| full:", JSON.stringify(err, Object.getOwnPropertyNames(err))); - res.status(400).json({ success: false, error: friendly }); + console.error("[S3 Test] Error:", err.message); + if (err.name === "AbortError") { + return res.status(400).json({ success: false, error: "Connection timed out — check the endpoint URL" }); + } + res.status(400).json({ success: false, error: err.message }); } }); From 43aa18f9632c0ee18bb631d9f5aa18fd96847243 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:42:52 -0400 Subject: [PATCH 10/17] Use ListObjectsV2Command for S3 test instead of raw HTTP signing --- server.js | 85 +++++++++---------------------------------------------- 1 file changed, 14 insertions(+), 71 deletions(-) diff --git a/server.js b/server.js index a6e8523..c99b9e1 100644 --- a/server.js +++ b/server.js @@ -3,7 +3,7 @@ const express = require("express"); const multer = require("multer"); const path = require("path"); const crypto = require("crypto"); -const { S3Client, PutObjectCommand, HeadBucketCommand, DeleteObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require("@aws-sdk/client-s3"); +const { S3Client, PutObjectCommand, HeadBucketCommand, DeleteObjectCommand, ListObjectsV2Command, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require("@aws-sdk/client-s3"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); const { Upload } = require("@aws-sdk/lib-storage"); const { NodeHttpHandler } = require("@smithy/node-http-handler"); @@ -442,7 +442,6 @@ app.put("/api/s3/config", requireAdmin, (req, res) => { }); app.post("/api/s3/test", requireAdmin, async (req, res) => { - // Merge submitted values with saved config — secret is often blank (keep existing) const saved = db.s3Config || {}; const testCfg = { endpoint: (req.body.endpoint || saved.endpoint || "").trim(), @@ -455,77 +454,21 @@ app.post("/api/s3/test", requireAdmin, async (req, res) => { if (!testCfg.bucket || !testCfg.accessKeyId || !testCfg.secretAccessKey) return res.status(400).json({ success: false, error: "Bucket, Access Key ID, and Secret Key are required to test" }); - // Use raw AWS Signature V4 HTTP request — compatible with RustFS/MinIO/generic S3 + const testClient = buildS3Client(testCfg); try { - const url = new URL(`${testCfg.endpoint}/${testCfg.bucket}/`); - const now = new Date(); - const dateStamp = now.toISOString().replace(/[-:]/g, "").split(".")[0] + "Z"; - const shortDate = dateStamp.slice(0, 8); - const host = url.host; - const method = "GET"; - const canonicalUri = `/${testCfg.bucket}/`; - - // AWS Signature V4 - function hmacSha256(key, data) { - return crypto.createHmac("sha256", key).update(data).digest(); - } - function sha256Hex(data) { - return crypto.createHash("sha256").update(data).digest("hex"); - } - - const credentialScope = `${shortDate}/${testCfg.region}/s3/aws4_request`; - const canonicalHeaders = `host:${host}\nx-amz-content-sha256:UNSIGNED-PAYLOAD\nx-amz-date:${dateStamp}\n`; - const signedHeaders = "host;x-amz-content-sha256;x-amz-date"; - const canonicalRequest = `${method}\n${canonicalUri}\n\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`; - const stringToSign = `AWS4-HMAC-SHA256\n${dateStamp}\n${credentialScope}\n${sha256Hex(canonicalRequest)}`; - - const signingKey = hmacSha256( - hmacSha256( - hmacSha256( - hmacSha256(`AWS4${testCfg.secretAccessKey}`, shortDate), - testCfg.region - ), - "s3" - ), - "aws4_request" - ); - const signature = hmacSha256(signingKey, stringToSign).toString("hex"); - const authHeader = `AWS4-HMAC-SHA256 Credential=${testCfg.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; - - const fetchUrl = `${testCfg.endpoint}/${testCfg.bucket}/?list-type=2&max-keys=1`; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 10000); - - const fetchRes = await fetch(fetchUrl, { - method: "GET", - headers: { - "Host": host, - "x-amz-date": dateStamp, - "x-amz-content-sha256": "UNSIGNED-PAYLOAD", - "Authorization": authHeader, - }, - signal: controller.signal, - }); - clearTimeout(timeout); - - const body = await fetchRes.text(); - console.log("[S3 Test] Status:", fetchRes.status, "Body:", body.slice(0, 300)); - - if (fetchRes.ok) { - res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." }); - } else if (fetchRes.status === 403) { - res.status(400).json({ success: false, error: "Access denied — check your Access Key and Secret Key" }); - } else if (fetchRes.status === 404) { - res.status(400).json({ success: false, error: `Bucket "${testCfg.bucket}" does not exist` }); - } else { - res.status(400).json({ success: false, error: `S3 returned ${fetchRes.status}: ${body.slice(0, 200)}` }); - } + const result = await testClient.send(new ListObjectsV2Command({ Bucket: testCfg.bucket, MaxKeys: 1 })); + console.log("[S3 Test] ListObjects OK, KeyCount:", result.KeyCount); + res.json({ success: true, message: "✓ S3 connection confirmed! Credentials are valid and the bucket is accessible." }); } catch (err) { - console.error("[S3 Test] Error:", err.message); - if (err.name === "AbortError") { - return res.status(400).json({ success: false, error: "Connection timed out — check the endpoint URL" }); - } - res.status(400).json({ success: false, error: err.message }); + console.error("[S3 Test] Full error:", JSON.stringify({ name: err.name, message: err.message, code: err.Code || err.code, status: err.$metadata?.httpStatusCode })); + let friendly = err.message || "Unknown error"; + if (err.name === "NoSuchBucket" || err.Code === "NoSuchBucket") friendly = `Bucket "${testCfg.bucket}" does not exist`; + else if (err.name === "InvalidAccessKeyId" || err.Code === "InvalidAccessKeyId") friendly = "Invalid Access Key ID"; + else if (err.name === "SignatureDoesNotMatch" || err.Code === "SignatureDoesNotMatch") friendly = "Invalid Secret Access Key"; + else if (err.name === "AccessDenied" || err.Code === "AccessDenied") friendly = "Access denied — credentials don't have permission for this bucket"; + else if (err.message?.includes("ECONNREFUSED")) friendly = "Connection refused — check endpoint URL"; + else if (err.message?.includes("ENOTFOUND")) friendly = "Endpoint not found — check the URL"; + res.status(400).json({ success: false, error: friendly }); } }); From d5192e88479e1138f42b42471c1c1a9c28a82f38 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:49:13 -0400 Subject: [PATCH 11/17] Replace chunked multipart upload with simple Upload class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manual CreateMultipartUploadCommand/UploadPartCommand/CompleteMultipartUploadCommand flow fails with RustFS due to non-standard XML responses (same issue as HeadBucketCommand). Switch front-end to use /api/upload endpoint which uses @aws-sdk/lib-storage Upload class — the same method that worked reliably in VPM-Uploader with RustFS. Uses XMLHttpRequest for upload progress tracking. Co-Authored-By: Claude Opus 4.6 --- public/index.html | 101 ++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 62 deletions(-) diff --git a/public/index.html b/public/index.html index 3468742..513c1d1 100644 --- a/public/index.html +++ b/public/index.html @@ -1090,71 +1090,48 @@ async function startUpload() { } // ============================================================ -// PARALLEL CHUNK UPLOAD (Option 4 — HTTP parallelism, Aspera-class speed) -// Slices each file into 32 MB chunks, uploads 6 concurrently via POST, -// server proxies to S3 multipart. Same approach as MASV — no UDP needed. +// SIMPLE UPLOAD — uses /api/upload with @aws-sdk/lib-storage on server +// Compatible with RustFS / MinIO / generic S3 endpoints. +// XMLHttpRequest used for upload progress tracking. // ============================================================ -const CHUNK_SIZE = 32 * 1024 * 1024; // 32 MB per part -const CHUNK_WORKERS = 6; // concurrent chunk POSTs per file -async function uploadFileChunked(item, idx) { - const file = item.file; - const totalParts = Math.ceil(file.size / CHUNK_SIZE); - const mime = file.type || 'application/octet-stream'; +async function uploadFileDirect(item, idx) { + return new Promise((resolve, reject) => { + const fd = new FormData(); + fd.append('prefix', selectedPrefix); + fd.append('files', item.file, item.name); - // 1. Initiate S3 multipart upload - const init = await api('POST','/api/upload/initiate',{ - filename: item.name, prefix: selectedPrefix, - contentType: mime, totalParts, + const xhr = new XMLHttpRequest(); + const pb = document.getElementById(`progbar-${idx}`); + + xhr.upload.addEventListener('progress', (e) => { + if (e.lengthComputable) { + const pct = Math.round(e.loaded / e.total * 100); + if (pb) pb.style.width = pct + '%'; + setFileStatus(idx, 'uploading', pct + '%'); + } + }); + + xhr.addEventListener('load', () => { + try { + const data = JSON.parse(xhr.responseText); + if (xhr.status >= 200 && xhr.status < 300 && data.success) { + resolve(data); + } else { + reject(new Error(data.error || `Server returned ${xhr.status}`)); + } + } catch (e) { + reject(new Error(`Bad response: ${xhr.status}`)); + } + }); + + xhr.addEventListener('error', () => reject(new Error('Network error'))); + xhr.addEventListener('abort', () => reject(new Error('Upload aborted'))); + + xhr.open('POST', '/api/upload'); + xhr.setRequestHeader('x-auth-token', authToken); + xhr.send(fd); }); - if (!init.success) throw new Error(init.error || 'Failed to initiate upload'); - const { uploadId } = init; - - const pb = document.getElementById(`progbar-${idx}`); - let done = 0; - - // 2. Upload all parts with bounded concurrency - const queue = Array.from({length: totalParts}, (_,i) => i+1); // 1-indexed - - async function worker() { - while (queue.length) { - const partNumber = queue.shift(); - if (partNumber === undefined) break; - const start = (partNumber - 1) * CHUNK_SIZE; - const blob = file.slice(start, Math.min(start + CHUNK_SIZE, file.size)); - const fd = new FormData(); - fd.append('uploadId', uploadId); - fd.append('partNumber', String(partNumber)); - fd.append('chunk', blob, item.name); - const resp = await fetch('/api/upload/chunk', { - method: 'POST', - headers: { 'x-auth-token': authToken }, - body: fd, - }); - const data = await resp.json(); - if (!data.success) throw new Error(`Part ${partNumber} failed: ${data.error}`); - done++; - const pct = Math.round(done / totalParts * 100); - if (pb) pb.style.width = pct + '%'; - setFileStatus(idx, 'uploading', pct + '%'); - } - } - - try { - await Promise.all(Array.from({length: Math.min(CHUNK_WORKERS, totalParts)}, worker)); - } catch (err) { - // Abort orphaned multipart upload on any chunk failure - fetch('/api/upload/abort', { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'x-auth-token': authToken }, - body: JSON.stringify({ uploadId }), - }).catch(() => {}); - throw err; - } - - // 3. Complete the multipart upload - const complete = await api('POST','/api/upload/complete',{ uploadId }); - if (!complete.success) throw new Error(complete.error || 'Failed to complete upload'); } async function uploadHTTP(files) { @@ -1170,7 +1147,7 @@ async function uploadHTTP(files) { setFileStatus(idx,'uploading','Uploading…'); document.getElementById(`prog-${idx}`).style.display='block'; try { - await uploadFileChunked(item, idx); + await uploadFileDirect(item, idx); document.getElementById(`progbar-${idx}`).style.width='100%'; setFileStatus(idx,'done','✓ Done'); item.status='done'; showToast(`Uploaded: ${item.name}`,'success'); From 44c22bd95ec759b04b8cb638a3fafdac34b8072a Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:52:54 -0400 Subject: [PATCH 12/17] Replace Upload class with PutObjectCommand for RustFS compatibility The @aws-sdk/lib-storage Upload class internally uses CreateMultipartUploadCommand for files over the part size threshold, which returns non-standard XML from RustFS causing UnknownError. PutObjectCommand does a simple single PUT request that RustFS handles correctly. Fixed in both /api/upload and /api/link/:id/upload endpoints. Co-Authored-By: Claude Opus 4.6 --- server.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/server.js b/server.js index c99b9e1..6a5f636 100644 --- a/server.js +++ b/server.js @@ -535,14 +535,14 @@ app.post("/api/upload", requireAuth, upload.array("files", 50), async (req, res) const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${file.originalname}` : file.originalname; const contentType = getMimeType(file.originalname, file.mimetype); console.log(`Uploading: ${key} (${file.size} bytes, ${contentType})`); - const uploadPromise = new Upload({ - client: s3Client, - params: { Bucket: bucket, Key: key, Body: fs.createReadStream(file.path), ContentType: contentType }, - queueSize: 4, - partSize: 10 * 1024 * 1024, - leavePartsOnError: false, - }).done(); - const result = await withTimeout(uploadPromise, UPLOAD_TIMEOUT_MS, key); + // Use PutObjectCommand (single PUT) — compatible with RustFS/MinIO/generic S3. + // The @aws-sdk/lib-storage Upload class uses CreateMultipartUpload internally + // which returns non-standard XML from RustFS and causes UnknownError. + const fileBuffer = fs.readFileSync(file.path); + const putPromise = s3Client.send(new PutObjectCommand({ + Bucket: bucket, Key: key, Body: fileBuffer, ContentType: contentType, + })); + const result = await withTimeout(putPromise, UPLOAD_TIMEOUT_MS, key); if (result?.assumed) console.log(`Assumed success (timeout): ${key}`); else console.log(`Confirmed success: ${key}`); try { fs.unlinkSync(file.path); } catch (_) {} @@ -841,12 +841,11 @@ app.post("/api/sharelinks/:token/upload", upload.array("files", 50), async (req, for (const file of req.files) { const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${file.originalname}` : file.originalname; const contentType = getMimeType(file.originalname, file.mimetype); - const uploadPromise = new Upload({ - client: s3Client, - params: { Bucket: bucket, Key: key, Body: fs.createReadStream(file.path), ContentType: contentType }, - queueSize: 4, partSize: 10 * 1024 * 1024, leavePartsOnError: false, - }).done(); - await withTimeout(uploadPromise, UPLOAD_TIMEOUT_MS, key); + const fileBuffer = fs.readFileSync(file.path); + const putPromise = s3Client.send(new PutObjectCommand({ + Bucket: bucket, Key: key, Body: fileBuffer, ContentType: contentType, + })); + await withTimeout(putPromise, UPLOAD_TIMEOUT_MS, key); try { fs.unlinkSync(file.path); } catch (_) {} results.push({ originalName: file.originalname, key, size: file.size }); } From ad5f9a4186f1fcf4be20cd2ba491264f87370b9b Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 00:56:04 -0400 Subject: [PATCH 13/17] Update HTTP mode description to reflect actual upload method Co-Authored-By: Claude Opus 4.6 --- public/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/index.html b/public/index.html index 513c1d1..bd2e62d 100644 --- a/public/index.html +++ b/public/index.html @@ -467,7 +467,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- HTTP Mode: Parallel chunked HTTP upload (6 concurrent 32 MB parts). Aspera-class speed — no UDP needed. + HTTP Mode: Direct HTTP upload to S3-compatible storage (up to 4 concurrent files). Compatible with RustFS, MinIO, and AWS S3.
Electron App — Aspera-speed desktop application (coming soon)
@@ -934,7 +934,7 @@ function setMode(mode) { if (mode === 'http') { btn.className = 'btn-upload'; if (label) label.textContent = 'HTTP Mode:'; - if (detail) detail.textContent = 'Parallel chunked HTTP upload (6 concurrent 32 MB parts). Aspera-class speed — no UDP needed.'; + if (detail) detail.textContent = 'Direct HTTP upload to S3-compatible storage (up to 4 concurrent files). Compatible with RustFS, MinIO, and AWS S3.'; if (hint) hint.style.display = 'none'; if (btnHttp) { btnHttp.className = 'mode-btn active-http'; } if (btnUdp) { btnUdp.className = 'mode-btn'; } @@ -1090,7 +1090,7 @@ async function startUpload() { } // ============================================================ -// SIMPLE UPLOAD — uses /api/upload with @aws-sdk/lib-storage on server +// SIMPLE UPLOAD — uses /api/upload with PutObjectCommand on server // Compatible with RustFS / MinIO / generic S3 endpoints. // XMLHttpRequest used for upload progress tracking. // ============================================================ From 0f81cac6ecc61fd8ab8bfb80c661060107c249ad Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 01:03:28 -0400 Subject: [PATCH 14/17] QOL improvements: folder search, user audit, admin folders fix, AMPP debug - Add search/filter input to destination folder browser with 320px scrollable container - Add User Audit button showing each user's visible pages, role, quota, and folder access permissions - Fix admin Folders tab delete buttons (were broken due to JSON.stringify quote conflicts in innerHTML onclick handlers) - Add more AMPP job name field fallbacks and debug logging to diagnose asset name display issue Co-Authored-By: Claude Opus 4.6 --- public/index.html | 141 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 130 insertions(+), 11 deletions(-) diff --git a/public/index.html b/public/index.html index bd2e62d..cf77d10 100644 --- a/public/index.html +++ b/public/index.html @@ -474,7 +474,8 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
Destination Folder
-
+ +
@@ -671,6 +672,17 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
+ + +
@@ -970,14 +982,29 @@ function renderFolderTree() { const box = document.getElementById('folder-tree-box'); if (!box) return; box.innerHTML = ''; - // Root row - const rootRow = document.createElement('div'); - rootRow.className = 'folder-tree-row' + (selectedPrefix==='' ? ' active' : ''); - rootRow.innerHTML = `🏠Root (no folder)`; - rootRow.onclick = () => { selectedPrefix=''; updatePrefixDisplay(); renderFolderTree(); }; - box.appendChild(rootRow); + const searchEl = document.getElementById('folder-search'); + const filter = (searchEl ? searchEl.value.trim().toLowerCase() : ''); + + // Helper: does a node (or any descendant) match the filter? + function matchesFilter(node, pathArr) { + const key = [...pathArr, node.name].join('/'); + if (key.toLowerCase().includes(filter)) return true; + if (node.children) return node.children.some(c => matchesFilter(c, [...pathArr, node.name])); + return false; + } + + // Root row (always shown unless filtering) + if (!filter) { + const rootRow = document.createElement('div'); + rootRow.className = 'folder-tree-row' + (selectedPrefix==='' ? ' active' : ''); + rootRow.innerHTML = `🏠Root (no folder)`; + rootRow.onclick = () => { selectedPrefix=''; updatePrefixDisplay(); renderFolderTree(); }; + box.appendChild(rootRow); + } + function addRows(nodes, pathArr, container) { nodes.forEach(n => { + if (filter && !matchesFilter(n, pathArr)) return; const fullPath = [...pathArr, n.name]; const key = fullPath.join('/'); const indent = pathArr.length; @@ -999,6 +1026,10 @@ function renderFolderTree() { }); } addRows(folderTree, [], box); + + if (!box.children.length) { + box.innerHTML = '
No folders match your search
'; + } } // Legacy aliases so other code still works @@ -1024,7 +1055,7 @@ async function deleteFolder(pathArr) { try { await api('POST','/api/folders/delete',{path:pathArr}); if (selectedPrefix.startsWith(pathArr.join('/'))) { selectedPrefix=''; updatePrefixDisplay(); } - await loadFolders(); showToast('Folder deleted','success'); + await loadFolders(); await loadAdminFolders(); showToast('Folder deleted','success'); } catch(e) { showToast(e.message,'error'); } } @@ -1218,14 +1249,17 @@ async function loadAmppJobs() { list.innerHTML='
No jobs in queue
'; return; } + // Log first job's keys for debugging field names + if (jobs.length) console.log('[AMPP] Sample job keys:', Object.keys(jobs[0]), 'Full:', JSON.stringify(jobs[0]).substring(0, 500)); jobs.forEach(job => { const el=document.createElement('div'); el.className='job-item'; // AMPP API uses colon-namespaced keys e.g. "state:jobState", "name:text", "job:id" const st=(job['state:jobState']||job.status||job.state||job.jobStatus||'unknown').toLowerCase(); const cls=st.includes('run')||st.includes('active')?'running':st.includes('complet')||st.includes('success')||st.includes('done')?'completed':st.includes('fail')||st.includes('error')||st.includes('abort')?'failed':st.includes('queue')||st.includes('wait')||st.includes('pend')?'queued':'unknown'; - const name=job['name:text']||job['assetName:text']||job.name||job.displayName||job['job:id']||job.id||'Job'; + // Try many possible asset name fields — AMPP responses vary by job type + const name=job['name:text']||job['assetName:text']||job['source:text']||job['sourceFile:text']||job['inputFile:text']||job['input:text']||job.name||job.displayName||job.assetName||job.sourceName||job.sourceFile||job.inputFile||job['job:id']||job.id||'Job'; const jobType=(job['type:jobType']||job['subtype:jobSubtype']||job.type||job.jobType||'').replace(/([A-Z])/g,' $1').trim(); - const creator=job['creator:id']||''; + const creator=job['creator:id']||job.creator||''; const created=job['created:dateTime']||job.created||''; const meta=[created?new Date(created).toLocaleString():'', jobType, creator].filter(Boolean).join(' · '); el.innerHTML=`
${esc(name)}
${esc(meta)}
${cls.charAt(0).toUpperCase()+cls.slice(1)}`; @@ -1352,6 +1386,7 @@ async function loadUsers() { ${u.created?new Date(u.created).toLocaleDateString():''} + ${u.username!==currentUser?``:'(you)'} `; tbody.appendChild(tr); @@ -1452,6 +1487,83 @@ function fmtBytes(b) { return (b/1073741824).toFixed(2) + ' GB'; } +// ============================================================ +// USER AUDIT +// ============================================================ +async function openUserAudit(username) { + const modal = document.getElementById('audit-modal'); + modal.style.display = 'flex'; + document.getElementById('audit-modal-title').textContent = `Audit — ${username}`; + const content = document.getElementById('audit-content'); + content.innerHTML = '
Loading…
'; + try { + const [pd, fd] = await Promise.all([ + api('GET', `/api/users/${encodeURIComponent(username)}/permissions`), + api('GET', '/api/folders') + ]); + if (!pd.success) throw new Error(pd.error); + const allFolders = flattenFolders(fd.tree || []); + const allowed = pd.allowedFolders || []; + const hasRestrictions = allowed.length > 0; + const visibleFolders = hasRestrictions ? allFolders.filter(f => allowed.some(a => f === a || f.startsWith(a + '--'))) : allFolders; + + let html = ''; + + // Role & Quota + html += `
`; + html += `
+
Role
+
${pd.role||'user'}
+
`; + html += `
+
Upload Quota
+
${pd.quotaMB ? `${fmtBytes(pd.uploadedBytes||0)} / ${pd.quotaMB} MB` : 'Unlimited'}
+
`; + html += `
`; + + // Visible pages + const pages = ['Upload']; + if (pd.role === 'admin') pages.push('AMPP Monitor', 'Admin Panel'); + else pages.push('AMPP Monitor'); + html += `
+
Visible Pages
+
${pages.map(p => `${p}`).join('')}
+
`; + + // Admin features + if (pd.role === 'admin') { + html += `
+
Admin Capabilities
+
+ S3 Storage settings, AMPP config, User management, Share Links, Folder management, Add/delete folders on upload page +
+
`; + } + + // Folder access + html += `
+
Folder Access ${hasRestrictions ? '(restricted)' : '(all folders)'}
+
`; + if (!visibleFolders.length) { + html += '
No folders configured
'; + } else { + visibleFolders.forEach(f => { + const depth = (f.match(/--/g) || []).length; + const name = f.includes('--') ? f.split('--').pop() : f; + const isAllowed = !hasRestrictions || allowed.includes(f); + html += `
+ ${isAllowed ? '✅' : '🚫'} ${esc(name)} +
`; + }); + } + html += `
`; + + content.innerHTML = html; + } catch(e) { + content.innerHTML = `
Error: ${e.message}
`; + } +} + // ============================================================ // SHARE LINKS // ============================================================ @@ -1540,7 +1652,14 @@ function renderAdminFolderTree(nodes,container,pathArr) { nodes.forEach(n=>{ const fp=[...pathArr,n.name]; const div=document.createElement('div');div.style.paddingLeft=pathArr.length*20+'px';div.style.marginBottom='.3rem'; - div.innerHTML=`
${n.children.length?'📁':'📄'}${esc(n.name)}
`; + const row=document.createElement('div'); + row.style.cssText='display:flex;align-items:center;gap:.5rem;padding:.3rem .5rem;border:1px solid var(--border);border-radius:7px;background:var(--bg-card)'; + row.innerHTML=`${n.children.length?'📁':'📄'}${esc(n.name)}`; + const delBtn=document.createElement('button'); + delBtn.className='btn-danger';delBtn.style.cssText='padding:.18rem .48rem;font-size:.66rem';delBtn.textContent='Delete'; + delBtn.onclick=()=>{ deleteFolder(fp).then(()=>{loadAdminFolders();}); }; + row.appendChild(delBtn); + div.appendChild(row); container.appendChild(div); if(n.children.length){const sub=document.createElement('div');renderAdminFolderTree(n.children,sub,fp);container.appendChild(sub);} }); From ecdfe0f7cdb52d6f1421a0c5e06507da352af330 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 01:08:55 -0400 Subject: [PATCH 15/17] =?UTF-8?q?Presigned=20direct-to-S3=20uploads=20?= =?UTF-8?q?=E2=80=94=20bypass=20Node=20server=20entirely?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Browser now uploads files directly to RustFS/S3 via presigned PUT URLs. The Node server only generates signed URLs and tracks quota — file data never touches the server. 6 concurrent file uploads. Falls back to server-proxied PutObjectCommand upload if presigned fails. Server changes: - /api/presigned now checks folder permissions, quota, and blocked files - /api/presigned/complete endpoint for post-upload quota tracking Co-Authored-By: Claude Opus 4.6 --- public/index.html | 107 ++++++++++++++++++++++++++++++++++------------ server.js | 31 +++++++++++++- 2 files changed, 109 insertions(+), 29 deletions(-) diff --git a/public/index.html b/public/index.html index cf77d10..d5c97d2 100644 --- a/public/index.html +++ b/public/index.html @@ -467,7 +467,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- HTTP Mode: Direct HTTP upload to S3-compatible storage (up to 4 concurrent files). Compatible with RustFS, MinIO, and AWS S3. + HTTP Mode: Direct-to-S3 upload via presigned URLs (up to 6 concurrent files). Browser uploads straight to storage, bypassing the server.
Electron App — Aspera-speed desktop application (coming soon)
@@ -946,7 +946,7 @@ function setMode(mode) { if (mode === 'http') { btn.className = 'btn-upload'; if (label) label.textContent = 'HTTP Mode:'; - if (detail) detail.textContent = 'Direct HTTP upload to S3-compatible storage (up to 4 concurrent files). Compatible with RustFS, MinIO, and AWS S3.'; + if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs (up to 6 concurrent files). Browser uploads straight to storage, bypassing the server.'; if (hint) hint.style.display = 'none'; if (btnHttp) { btnHttp.className = 'mode-btn active-http'; } if (btnUdp) { btnUdp.className = 'mode-btn'; } @@ -1121,12 +1121,60 @@ async function startUpload() { } // ============================================================ -// SIMPLE UPLOAD — uses /api/upload with PutObjectCommand on server -// Compatible with RustFS / MinIO / generic S3 endpoints. -// XMLHttpRequest used for upload progress tracking. +// PRESIGNED DIRECT-TO-S3 UPLOAD +// Browser uploads directly to RustFS/S3 via presigned PUT URLs. +// Server only generates signed URLs — file data never touches Node. +// Falls back to server-proxied upload if presigned fails. // ============================================================ +const UPLOAD_CONCURRENCY = 6; // concurrent file uploads -async function uploadFileDirect(item, idx) { +async function uploadFilePresigned(item, idx) { + const pb = document.getElementById(`progbar-${idx}`); + const mime = item.file.type || 'application/octet-stream'; + + // 1. Get presigned PUT URL from server + setFileStatus(idx, 'uploading', 'Getting URL…'); + const pre = await api('POST', '/api/presigned', { + filename: item.name, + prefix: selectedPrefix, + contentType: mime, + size: item.file.size, + }); + if (!pre.success) throw new Error(pre.error || 'Failed to get presigned URL'); + + // 2. PUT directly to S3 via presigned URL + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + + xhr.upload.addEventListener('progress', (e) => { + if (e.lengthComputable) { + const pct = Math.round(e.loaded / e.total * 100); + if (pb) pb.style.width = pct + '%'; + setFileStatus(idx, 'uploading', pct + '%'); + } + }); + + xhr.addEventListener('load', async () => { + if (xhr.status >= 200 && xhr.status < 300) { + // 3. Notify server for quota tracking + try { await api('POST', '/api/presigned/complete', { key: pre.key, size: item.file.size }); } catch(_) {} + resolve({ key: pre.key }); + } else { + reject(new Error(`S3 returned ${xhr.status}: ${xhr.statusText}`)); + } + }); + + xhr.addEventListener('error', () => reject(new Error('Direct upload failed — network error'))); + xhr.addEventListener('abort', () => reject(new Error('Upload aborted'))); + + xhr.open('PUT', pre.url); + xhr.setRequestHeader('Content-Type', mime); + xhr.send(item.file); + }); +} + +// Fallback: upload through Node server (for when presigned fails) +async function uploadFileViaServer(item, idx) { return new Promise((resolve, reject) => { const fd = new FormData(); fd.append('prefix', selectedPrefix); @@ -1139,21 +1187,16 @@ async function uploadFileDirect(item, idx) { if (e.lengthComputable) { const pct = Math.round(e.loaded / e.total * 100); if (pb) pb.style.width = pct + '%'; - setFileStatus(idx, 'uploading', pct + '%'); + setFileStatus(idx, 'uploading', `${pct}% (server)`); } }); xhr.addEventListener('load', () => { try { const data = JSON.parse(xhr.responseText); - if (xhr.status >= 200 && xhr.status < 300 && data.success) { - resolve(data); - } else { - reject(new Error(data.error || `Server returned ${xhr.status}`)); - } - } catch (e) { - reject(new Error(`Bad response: ${xhr.status}`)); - } + if (xhr.status >= 200 && xhr.status < 300 && data.success) resolve(data); + else reject(new Error(data.error || `Server returned ${xhr.status}`)); + } catch (e) { reject(new Error(`Bad response: ${xhr.status}`)); } }); xhr.addEventListener('error', () => reject(new Error('Network error'))); @@ -1166,8 +1209,6 @@ async function uploadFileDirect(item, idx) { } async function uploadHTTP(files) { - // Upload files with up to 4 concurrent file uploads - const FILE_CONCURRENCY = 4; const fileQueue = [...files]; async function fileWorker() { @@ -1175,22 +1216,34 @@ async function uploadHTTP(files) { const item = fileQueue.shift(); if (!item) break; const idx = selectedFiles.indexOf(item); - setFileStatus(idx,'uploading','Uploading…'); - document.getElementById(`prog-${idx}`).style.display='block'; + setFileStatus(idx, 'uploading', 'Starting…'); + document.getElementById(`prog-${idx}`).style.display = 'block'; try { - await uploadFileDirect(item, idx); - document.getElementById(`progbar-${idx}`).style.width='100%'; - setFileStatus(idx,'done','✓ Done'); item.status='done'; - showToast(`Uploaded: ${item.name}`,'success'); - } catch(e) { - setFileStatus(idx,'error','✗ Error'); item.status='error'; - showToast(`Failed: ${item.name} — ${e.message}`,'error'); + // Try presigned direct-to-S3 first + await uploadFilePresigned(item, idx); + document.getElementById(`progbar-${idx}`).style.width = '100%'; + setFileStatus(idx, 'done', '✓ Done'); item.status = 'done'; + showToast(`Uploaded: ${item.name}`, 'success'); + } catch(presignedErr) { + // Fallback to server-proxied upload + console.warn(`[upload] Presigned failed for ${item.name}: ${presignedErr.message}, falling back to server upload`); + try { + setFileStatus(idx, 'uploading', 'Retrying via server…'); + if (document.getElementById(`progbar-${idx}`)) document.getElementById(`progbar-${idx}`).style.width = '0%'; + await uploadFileViaServer(item, idx); + document.getElementById(`progbar-${idx}`).style.width = '100%'; + setFileStatus(idx, 'done', '✓ Done (server)'); item.status = 'done'; + showToast(`Uploaded: ${item.name}`, 'success'); + } catch(serverErr) { + setFileStatus(idx, 'error', '✗ Error'); item.status = 'error'; + showToast(`Failed: ${item.name} — ${serverErr.message}`, 'error'); + } } updateStats(); } } - await Promise.all(Array.from({length: Math.min(FILE_CONCURRENCY, files.length)}, fileWorker)); + await Promise.all(Array.from({length: Math.min(UPLOAD_CONCURRENCY, files.length)}, fileWorker)); } async function uploadUDP(files) { diff --git a/server.js b/server.js index 6a5f636..3129a2b 100644 --- a/server.js +++ b/server.js @@ -558,11 +558,26 @@ app.post("/api/upload", requireAuth, upload.array("files", 50), async (req, res) } }); -// ==================== PRESIGNED URL (for Chrome extension HTTP mode) ==================== +// ==================== PRESIGNED URL (direct-to-S3 upload) ==================== +// Client uploads directly to RustFS/S3 using presigned PUT URLs, bypassing the Node server. +// Server only generates the signed URL and tracks quota/permissions. app.post("/api/presigned", requireAuth, async (req, res) => { if (!s3Client) return res.status(503).json({ success: false, error: "S3 not configured" }); - const { filename, prefix, contentType } = req.body; + const { filename, prefix, contentType, size } = req.body; if (!filename) return res.status(400).json({ success: false, error: "filename required" }); + if (isBlockedFile(filename)) return res.status(400).json({ success: false, error: `Blocked file type: ${filename}` }); + + const user = db.users.find(u => u.username === req.sessionData.user); + // Folder permission check + if (user && !isFolderAllowed(user, prefix || "")) { + return res.status(403).json({ success: false, error: "You do not have permission to upload to this folder" }); + } + // Quota check + if (size && user && isQuotaExceeded(user, size)) { + const usedMB = ((user.uploadedBytes || 0) / 1024 / 1024).toFixed(1); + return res.status(403).json({ success: false, error: `Upload quota exceeded (${usedMB} MB of ${user.quotaMB} MB used)` }); + } + const bucket = db.s3Config?.bucket || ""; const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename; const mime = contentType || getMimeType(filename, "application/octet-stream"); @@ -575,6 +590,18 @@ app.post("/api/presigned", requireAuth, async (req, res) => { } }); +// Called by client after a successful direct-to-S3 upload to update quota tracking +app.post("/api/presigned/complete", requireAuth, (req, res) => { + const { key, size } = req.body; + const user = db.users.find(u => u.username === req.sessionData.user); + if (user && size) { + user.uploadedBytes = (user.uploadedBytes || 0) + size; + saveData(db); + } + console.log(`[presigned] Completed: ${key} (${size} bytes) by ${req.sessionData.user}`); + res.json({ success: true }); +}); + // ==================== UDP UPLOAD SESSION ==================== // Sessions stored in memory; relay handles actual transfer const udpSessions = new Map(); From 5b0a3ef2cc679aae6bb7c526bfc4ae2e548cc5cc Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 7 Apr 2026 09:16:24 -0400 Subject: [PATCH 16/17] Add server-side AMPP job debug logging to identify field names Logs the first job's keys and data (truncated to 1000 chars) on each /api/ampp/jobs request to help identify which field contains the asset name. Co-Authored-By: Claude Opus 4.6 --- server.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server.js b/server.js index 3129a2b..66adf76 100644 --- a/server.js +++ b/server.js @@ -775,7 +775,14 @@ app.get("/api/ampp/jobs", requireAuth, async (req, res) => { return res.json({ success: true, jobs: await r2.json() }); } if (!r.ok) return res.status(r.status).json({ success: false, error: `AMPP ${r.status}` }); - res.json({ success: true, jobs: await r.json() }); + const jobData = await r.json(); + // Debug: log the first job's full structure to identify field names + const items = Array.isArray(jobData) ? jobData : (jobData?.items || jobData?.results || []); + if (items.length > 0) { + console.log("[AMPP] Sample job keys:", Object.keys(items[0])); + console.log("[AMPP] Sample job data:", JSON.stringify(items[0]).substring(0, 1000)); + } + res.json({ success: true, jobs: jobData }); } catch (err) { if (err.name === "AbortError") return res.status(504).json({ success: false, error: "AMPP timeout" }); res.status(500).json({ success: false, error: err.message }); From 84cf9cccbe051315e85cc60ec633cc8e4e559537 Mon Sep 17 00:00:00 2001 From: Zac Gaetano Date: Wed, 8 Apr 2026 21:40:44 -0400 Subject: [PATCH 17/17] Fix folder sorting, default selection, subfolder S3 keys, and HTTP mode description Addresses feedback from Gavin (VPM): - Sort folders alphabetically in both upload tree and admin tree views - Auto-select VPM as default folder on login instead of Root - Fix S3 key construction for nested folders: convert "/" to "--" so FLX correctly maps subfolders (e.g. Content/TEST - AMPP Demo now produces Content--TEST - AMPP Demo--file.ext instead of Content/TEST - AMPP Demo--file.ext) - Clarify HTTP mode note: "Files are processed 6 at a time" instead of "up to 6 concurrent files" which implied a total file limit Co-Authored-By: Claude Opus 4.6 --- lib/upload-manager.js | 4 +++- public/index.html | 15 +++++++++++---- server.js | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/upload-manager.js b/lib/upload-manager.js index 0eb6148..0a64c7b 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -21,7 +21,9 @@ class UploadManager { createSession({ filename, size, mode = "http", prefix = "" }) { if (!["http", "udp"].includes(mode)) throw new Error("mode must be 'http' or 'udp'"); const sessionId = crypto.randomBytes(16).toString("hex"); - const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename; + // Normalize prefix: UI uses "/" for nested folders, FLX expects "--" as delimiter + const normalized = prefix ? prefix.replace(/\//g, "--").replace(/[-]+$/, "") : ""; + const key = normalized ? `${normalized}--${filename}` : filename; const session = { sessionId, filename, size, mode, key, prefix, status: "pending", diff --git a/public/index.html b/public/index.html index d5c97d2..7a4a818 100644 --- a/public/index.html +++ b/public/index.html @@ -467,7 +467,7 @@ body::before{content:'';position:fixed;inset:0;background:radial-gradient(ellips
- HTTP Mode: Direct-to-S3 upload via presigned URLs (up to 6 concurrent files). Browser uploads straight to storage, bypassing the server. + HTTP Mode: Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server. Files are processed 6 at a time.
Electron App — Aspera-speed desktop application (coming soon)
@@ -946,7 +946,7 @@ function setMode(mode) { if (mode === 'http') { btn.className = 'btn-upload'; if (label) label.textContent = 'HTTP Mode:'; - if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs (up to 6 concurrent files). Browser uploads straight to storage, bypassing the server.'; + if (detail) detail.textContent = 'Direct-to-S3 upload via presigned URLs. Browser uploads straight to storage, bypassing the server. Files are processed 6 at a time.'; if (hint) hint.style.display = 'none'; if (btnHttp) { btnHttp.className = 'mode-btn active-http'; } if (btnUdp) { btnUdp.className = 'mode-btn'; } @@ -974,6 +974,11 @@ async function loadFolders() { try { const d = await api('GET','/api/folders'); folderTree = d.tree || []; + // Auto-select VPM as default folder if nothing is selected yet + if (!selectedPrefix && folderTree.some(n => n.name === 'VPM')) { + selectedPrefix = 'VPM'; + updatePrefixDisplay(); + } renderFolderTree(); } catch(e) { console.error('loadFolders:',e); } } @@ -1003,7 +1008,8 @@ function renderFolderTree() { } function addRows(nodes, pathArr, container) { - nodes.forEach(n => { + const sorted = [...nodes].sort((a, b) => a.name.localeCompare(b.name, undefined, {sensitivity: 'base'})); + sorted.forEach(n => { if (filter && !matchesFilter(n, pathArr)) return; const fullPath = [...pathArr, n.name]; const key = fullPath.join('/'); @@ -1702,7 +1708,8 @@ async function loadAdminFolders() { function renderAdminFolderTree(nodes,container,pathArr) { container.innerHTML=''; if(!nodes.length){container.innerHTML='
No folders. Add one below.
';return;} - nodes.forEach(n=>{ + const sorted=[...nodes].sort((a,b)=>a.name.localeCompare(b.name,undefined,{sensitivity:'base'})); + sorted.forEach(n=>{ const fp=[...pathArr,n.name]; const div=document.createElement('div');div.style.paddingLeft=pathArr.length*20+'px';div.style.marginBottom='.3rem'; const row=document.createElement('div'); diff --git a/server.js b/server.js index 66adf76..55741b0 100644 --- a/server.js +++ b/server.js @@ -186,6 +186,15 @@ function cleanName(s) { return (s || "").trim().replace(/[^a-zA-Z0-9'\-.,&()! ]/g, ""); } +// Build S3 object key from a folder prefix and filename. +// The prefix may use "/" to separate nested folders (from the UI tree), +// but FLX expects "--" as the folder delimiter in object keys. +function buildS3Key(prefix, filename) { + if (!prefix) return filename; + const normalized = prefix.replace(/\//g, "--").replace(/[-]+$/, ""); + return `${normalized}--${filename}`; +} + function findNode(pathArr) { let current = getTree(); for (const segment of pathArr) { @@ -532,7 +541,7 @@ app.post("/api/upload", requireAuth, upload.array("files", 50), async (req, res) } const bucket = db.s3Config?.bucket || ""; for (const file of req.files) { - const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${file.originalname}` : file.originalname; + const key = buildS3Key(prefix, file.originalname); const contentType = getMimeType(file.originalname, file.mimetype); console.log(`Uploading: ${key} (${file.size} bytes, ${contentType})`); // Use PutObjectCommand (single PUT) — compatible with RustFS/MinIO/generic S3. @@ -579,7 +588,7 @@ app.post("/api/presigned", requireAuth, async (req, res) => { } const bucket = db.s3Config?.bucket || ""; - const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename; + const key = buildS3Key(prefix, filename); const mime = contentType || getMimeType(filename, "application/octet-stream"); try { const url = await getSignedUrl(s3Client, new PutObjectCommand({ Bucket: bucket, Key: key, ContentType: mime }), { expiresIn: 3600 }); @@ -614,7 +623,7 @@ app.post("/api/udp/session", requireAuth, async (req, res) => { if (!relayUrl) return res.status(503).json({ success: false, error: "UDP relay not configured. Go to Admin → Relay Settings." }); const publicRelayUrl = (db.relayConfig?.publicRelayUrl || "").trim() || relayUrl; const sessionId = crypto.randomBytes(16).toString("hex"); - const key = (prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename); + const key = buildS3Key(prefix, filename); const s3Cfg = db.s3Config || {}; // Register session on the relay server so it can accept chunks and upload to S3 @@ -873,7 +882,7 @@ app.post("/api/sharelinks/:token/upload", upload.array("files", 50), async (req, const bucket = db.s3Config?.bucket || ""; const results = []; for (const file of req.files) { - const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${file.originalname}` : file.originalname; + const key = buildS3Key(prefix, file.originalname); const contentType = getMimeType(file.originalname, file.mimetype); const fileBuffer = fs.readFileSync(file.path); const putPromise = s3Client.send(new PutObjectCommand({ @@ -964,7 +973,7 @@ app.post("/api/upload/initiate", requireAuth, async (req, res) => { const bucket = db.s3Config?.bucket || ""; if (!bucket) return res.status(503).json({ success: false, error: "S3 bucket not configured" }); - const key = prefix ? `${prefix.replace(/[-\/]+$/, "")}--${filename}` : filename; + const key = buildS3Key(prefix, filename); const mime = contentType || getMimeType(filename, "application/octet-stream"); try {