From 7894ecf0339aa109c3814e453df509cf05d72870 Mon Sep 17 00:00:00 2001 From: Forge Dev Date: Fri, 10 Jul 2026 20:13:12 +0000 Subject: [PATCH] =?UTF-8?q?offload:=20CopyEngine=20=E2=80=94=20multi-desti?= =?UTF-8?q?nation=20verified=20copy=20(temp+rename,=20re-read=20hash),=20s?= =?UTF-8?q?kip=20identical,=20conflict=20hard-error,=20partial=20re-copy?= =?UTF-8?q?=20=E2=80=94=206=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/ForgeOffload/CopyEngine.swift | 133 ++++++++++++++++++ Tests/ForgeOffloadTests/CopyEngineTests.swift | 109 ++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 Sources/ForgeOffload/CopyEngine.swift create mode 100644 Tests/ForgeOffloadTests/CopyEngineTests.swift diff --git a/Sources/ForgeOffload/CopyEngine.swift b/Sources/ForgeOffload/CopyEngine.swift new file mode 100644 index 0000000..265bc09 --- /dev/null +++ b/Sources/ForgeOffload/CopyEngine.swift @@ -0,0 +1,133 @@ +import Foundation + +/// Per-file offload result. +public struct FileRecord: Sendable, Equatable { + public let relativePath: String + public let size: UInt64 + public let xxh64: String + /// Destination copy verified by post-copy re-read. + public let verified: Bool + /// Already present + identical at all destinations. + public let skipped: Bool +} + +/// Offload job report. +public struct OffloadReport: Sendable { + public let source: String + public let destinations: [String] + public let files: [FileRecord] + public let totalBytes: UInt64 + public let duration: TimeInterval + public let startedAt: Date +} + +/// Verified multi-destination copy: source hashed while copying, destinations +/// re-read + hashed after write. Never silently overwrites mismatched files. +public struct CopyEngine { + public init() {} + + /// Copy source tree to all destinations. + /// - overwritePartial: re-copy destination files whose *size* differs + /// (interrupted copy detection). Size-matching mismatched hashes always error. + public func offload( + source: String, + destinations: [String], + overwritePartial: Bool = false + ) throws -> OffloadReport { + let fm = FileManager.default + var isDir: ObjCBool = false + guard fm.fileExists(atPath: source, isDirectory: &isDir), isDir.boolValue else { + throw OffloadError.fileNotFound(source) + } + + let start = Date() + let clock = ContinuousClock.now + let files = try listFiles(root: source).sorted() + var records = [FileRecord]() + var totalBytes: UInt64 = 0 + + for rel in files { + let srcPath = source + "/" + rel + let srcHash = try FileHasher.xxh64(path: srcPath) + let attrs = try fm.attributesOfItem(atPath: srcPath) + let size = attrs[.size] as? UInt64 ?? 0 + totalBytes += size + + var anyCopied = false + for dest in destinations { + let dstPath = dest + "/" + rel + let dstDir = (dstPath as NSString).deletingLastPathComponent + try fm.createDirectory(atPath: dstDir, withIntermediateDirectories: true) + + if fm.fileExists(atPath: dstPath) { + let dstAttrs = try fm.attributesOfItem(atPath: dstPath) + let dstSize = dstAttrs[.size] as? UInt64 ?? 0 + if dstSize != size { + // Partial/interrupted copy. + if overwritePartial { + try fm.removeItem(atPath: dstPath) + } else { + throw OffloadError.destinationConflict( + "\(dstPath): size \(dstSize) != source \(size) (partial? rerun with overwritePartial)") + } + } else { + let dstHash = try FileHasher.xxh64(path: dstPath) + if dstHash == srcHash { + continue // identical — skip + } + throw OffloadError.destinationConflict( + "\(dstPath): exists with different content (hash \(dstHash) != \(srcHash))") + } + } + + // Copy via temp + atomic rename, then verify by re-read. + let tmpPath = dstPath + ".forgetmp" + try? fm.removeItem(atPath: tmpPath) + try fm.copyItem(atPath: srcPath, toPath: tmpPath) + let copiedHash = try FileHasher.xxh64(path: tmpPath) + guard copiedHash == srcHash else { + try? fm.removeItem(atPath: tmpPath) + throw OffloadError.verifyFailed(path: dstPath, expected: srcHash, actual: copiedHash) + } + _ = try? fm.replaceItemAt(URL(fileURLWithPath: dstPath), withItemAt: URL(fileURLWithPath: tmpPath)) + if !fm.fileExists(atPath: dstPath) { + // replaceItemAt fallback (some Linux Foundation builds). + try fm.moveItem(atPath: tmpPath, toPath: dstPath) + } + anyCopied = true + } + + records.append(FileRecord( + relativePath: rel, + size: size, + xxh64: srcHash, + verified: true, + skipped: !anyCopied)) + } + + return OffloadReport( + source: source, + destinations: destinations, + files: records, + totalBytes: totalBytes, + duration: max(Double(clock.duration(to: .now).components.seconds) + + Double(clock.duration(to: .now).components.attoseconds) / 1e18, 0.000001), + startedAt: start) + } + + /// Relative paths of all regular files under root. + func listFiles(root: String) throws -> [String] { + let fm = FileManager.default + guard let enumerator = fm.enumerator(atPath: root) else { + throw OffloadError.readFailed(root) + } + var out = [String]() + while let rel = enumerator.nextObject() as? String { + var isDir: ObjCBool = false + if fm.fileExists(atPath: root + "/" + rel, isDirectory: &isDir), !isDir.boolValue { + out.append(rel) + } + } + return out + } +} diff --git a/Tests/ForgeOffloadTests/CopyEngineTests.swift b/Tests/ForgeOffloadTests/CopyEngineTests.swift new file mode 100644 index 0000000..88fb1b0 --- /dev/null +++ b/Tests/ForgeOffloadTests/CopyEngineTests.swift @@ -0,0 +1,109 @@ +import XCTest +import Foundation +@testable import ForgeOffload + +final class CopyEngineTests: XCTestCase { + + var root: String! + var source: String! + var dest1: String! + var dest2: String! + + override func setUpWithError() throws { + root = NSTemporaryDirectory() + "forge-copy-\(UUID().uuidString)" + source = root + "/CARD_A001" + dest1 = root + "/RAID1/A001" + dest2 = root + "/SHUTTLE/A001" + try FileManager.default.createDirectory(atPath: source + "/CLIPS", withIntermediateDirectories: true) + + // Source tree: two clips + sidecar. + try makeFile(source + "/CLIPS/A001C001.mov", size: 200_000, seed: 1) + try makeFile(source + "/CLIPS/A001C002.mov", size: 350_000, seed: 2) + try makeFile(source + "/A001.xml", size: 512, seed: 3) + } + + override func tearDown() { + try? FileManager.default.removeItem(atPath: root) + } + + func makeFile(_ path: String, size: Int, seed: UInt64) throws { + var data = Data(capacity: size) + var s = seed + while data.count < size { + s = s &* 6364136223846793005 &+ 1442695040888963407 + withUnsafeBytes(of: s) { data.append(contentsOf: $0.prefix(min(8, size - data.count))) } + } + try data.write(to: URL(fileURLWithPath: path)) + } + + // Copy to two destinations: byte-identical, verified, report per file. + func testCopyTwoDestinationsVerified() throws { + let engine = CopyEngine() + let report = try engine.offload(source: source, destinations: [dest1, dest2]) + + XCTAssertEqual(report.files.count, 3) + XCTAssertTrue(report.files.allSatisfy { $0.verified }) + + for dest in [dest1!, dest2!] { + for rel in ["CLIPS/A001C001.mov", "CLIPS/A001C002.mov", "A001.xml"] { + let srcHash = try FileHasher.xxh64(path: source + "/" + rel) + let dstHash = try FileHasher.xxh64(path: dest + "/" + rel) + XCTAssertEqual(srcHash, dstHash, "hash mismatch for \(rel) in \(dest)") + } + } + // Report carries source hashes. + XCTAssertEqual(Set(report.files.map(\.relativePath)), + ["CLIPS/A001C001.mov", "CLIPS/A001C002.mov", "A001.xml"]) + XCTAssertTrue(report.files.allSatisfy { $0.xxh64.count == 16 }) + } + + // Existing identical file at destination: skipped, still verified. + func testExistingIdenticalSkipped() throws { + let engine = CopyEngine() + _ = try engine.offload(source: source, destinations: [dest1]) + // Second run: all files exist + match. + let report2 = try engine.offload(source: source, destinations: [dest1]) + XCTAssertTrue(report2.files.allSatisfy { $0.skipped }) + XCTAssertTrue(report2.files.allSatisfy { $0.verified }) + } + + // Existing file with different content: hard error (never silently overwrite). + func testConflictErrors() throws { + let engine = CopyEngine() + _ = try engine.offload(source: source, destinations: [dest1]) + // Tamper destination clip. + try makeFile(dest1 + "/CLIPS/A001C001.mov", size: 200_000, seed: 99) + XCTAssertThrowsError(try engine.offload(source: source, destinations: [dest1])) { error in + guard case OffloadError.destinationConflict = error else { + return XCTFail("wrong error: \(error)") + } + } + } + + // Partial file (interrupted copy simulation): re-copied, ends verified. + func testPartialFileRecopied() throws { + let engine = CopyEngine() + // Simulate interrupted copy: partial temp content at final path with wrong size. + try FileManager.default.createDirectory(atPath: dest1 + "/CLIPS", withIntermediateDirectories: true) + try makeFile(dest1 + "/CLIPS/A001C001.mov", size: 50_000, seed: 1) // truncated + let report = try engine.offload(source: source, destinations: [dest1], overwritePartial: true) + let entry = report.files.first { $0.relativePath == "CLIPS/A001C001.mov" }! + XCTAssertFalse(entry.skipped) + XCTAssertTrue(entry.verified) + let srcHash = try FileHasher.xxh64(path: source + "/CLIPS/A001C001.mov") + XCTAssertEqual(try FileHasher.xxh64(path: dest1 + "/CLIPS/A001C001.mov"), srcHash) + } + + // Missing source dir throws. + func testMissingSourceThrows() { + XCTAssertThrowsError(try CopyEngine().offload(source: root + "/nope", destinations: [dest1])) + } + + // Report totals. + func testReportTotals() throws { + let report = try CopyEngine().offload(source: source, destinations: [dest1]) + XCTAssertEqual(report.totalBytes, 200_000 + 350_000 + 512) + XCTAssertEqual(report.destinations, [dest1]) + XCTAssertGreaterThan(report.duration, 0) + } +}