offload: CopyEngine — multi-destination verified copy (temp+rename, re-read hash), skip identical, conflict hard-error, partial re-copy — 6 tests

Sources/ForgeOffload/CopyEngine.swift

@@ -0,0 +1,133 @@
+import Foundation
+
+/// Per-file offload result.
+public struct FileRecord: Sendable, Equatable {
+    public let relativePath: String
+    public let size: UInt64
+    public let xxh64: String
+    /// Destination copy verified by post-copy re-read.
+    public let verified: Bool
+    /// Already present + identical at all destinations.
+    public let skipped: Bool
+}
+
+/// Offload job report.
+public struct OffloadReport: Sendable {
+    public let source: String
+    public let destinations: [String]
+    public let files: [FileRecord]
+    public let totalBytes: UInt64
+    public let duration: TimeInterval
+    public let startedAt: Date
+}
+
+/// Verified multi-destination copy: source hashed while copying, destinations
+/// re-read + hashed after write. Never silently overwrites mismatched files.
+public struct CopyEngine {
+    public init() {}
+
+    /// Copy source tree to all destinations.
+    /// - overwritePartial: re-copy destination files whose *size* differs
+    ///   (interrupted copy detection). Size-matching mismatched hashes always error.
+    public func offload(
+        source: String,
+        destinations: [String],
+        overwritePartial: Bool = false
+    ) throws -> OffloadReport {
+        let fm = FileManager.default
+        var isDir: ObjCBool = false
+        guard fm.fileExists(atPath: source, isDirectory: &isDir), isDir.boolValue else {
+            throw OffloadError.fileNotFound(source)
+        }
+
+        let start = Date()
+        let clock = ContinuousClock.now
+        let files = try listFiles(root: source).sorted()
+        var records = [FileRecord]()
+        var totalBytes: UInt64 = 0
+
+        for rel in files {
+            let srcPath = source + "/" + rel
+            let srcHash = try FileHasher.xxh64(path: srcPath)
+            let attrs = try fm.attributesOfItem(atPath: srcPath)
+            let size = attrs[.size] as? UInt64 ?? 0
+            totalBytes += size
+
+            var anyCopied = false
+            for dest in destinations {
+                let dstPath = dest + "/" + rel
+                let dstDir = (dstPath as NSString).deletingLastPathComponent
+                try fm.createDirectory(atPath: dstDir, withIntermediateDirectories: true)
+
+                if fm.fileExists(atPath: dstPath) {
+                    let dstAttrs = try fm.attributesOfItem(atPath: dstPath)
+                    let dstSize = dstAttrs[.size] as? UInt64 ?? 0
+                    if dstSize != size {
+                        // Partial/interrupted copy.
+                        if overwritePartial {
+                            try fm.removeItem(atPath: dstPath)
+                        } else {
+                            throw OffloadError.destinationConflict(
+                                "\(dstPath): size \(dstSize) != source \(size) (partial? rerun with overwritePartial)")
+                        }
+                    } else {
+                        let dstHash = try FileHasher.xxh64(path: dstPath)
+                        if dstHash == srcHash {
+                            continue // identical — skip
+                        }
+                        throw OffloadError.destinationConflict(
+                            "\(dstPath): exists with different content (hash \(dstHash) != \(srcHash))")
+                    }
+                }
+
+                // Copy via temp + atomic rename, then verify by re-read.
+                let tmpPath = dstPath + ".forgetmp"
+                try? fm.removeItem(atPath: tmpPath)
+                try fm.copyItem(atPath: srcPath, toPath: tmpPath)
+                let copiedHash = try FileHasher.xxh64(path: tmpPath)
+                guard copiedHash == srcHash else {
+                    try? fm.removeItem(atPath: tmpPath)
+                    throw OffloadError.verifyFailed(path: dstPath, expected: srcHash, actual: copiedHash)
+                }
+                _ = try? fm.replaceItemAt(URL(fileURLWithPath: dstPath), withItemAt: URL(fileURLWithPath: tmpPath))
+                if !fm.fileExists(atPath: dstPath) {
+                    // replaceItemAt fallback (some Linux Foundation builds).
+                    try fm.moveItem(atPath: tmpPath, toPath: dstPath)
+                }
+                anyCopied = true
+            }
+
+            records.append(FileRecord(
+                relativePath: rel,
+                size: size,
+                xxh64: srcHash,
+                verified: true,
+                skipped: !anyCopied))
+        }
+
+        return OffloadReport(
+            source: source,
+            destinations: destinations,
+            files: records,
+            totalBytes: totalBytes,
+            duration: max(Double(clock.duration(to: .now).components.seconds) +
+                          Double(clock.duration(to: .now).components.attoseconds) / 1e18, 0.000001),
+            startedAt: start)
+    }
+
+    /// Relative paths of all regular files under root.
+    func listFiles(root: String) throws -> [String] {
+        let fm = FileManager.default
+        guard let enumerator = fm.enumerator(atPath: root) else {
+            throw OffloadError.readFailed(root)
+        }
+        var out = [String]()
+        while let rel = enumerator.nextObject() as? String {
+            var isDir: ObjCBool = false
+            if fm.fileExists(atPath: root + "/" + rel, isDirectory: &isDir), !isDir.boolValue {
+                out.append(rel)
+            }
+        }
+        return out
+    }
+}

Tests/ForgeOffloadTests/CopyEngineTests.swift

@@ -0,0 +1,109 @@
+import XCTest
+import Foundation
+@testable import ForgeOffload
+
+final class CopyEngineTests: XCTestCase {
+
+    var root: String!
+    var source: String!
+    var dest1: String!
+    var dest2: String!
+
+    override func setUpWithError() throws {
+        root = NSTemporaryDirectory() + "forge-copy-\(UUID().uuidString)"
+        source = root + "/CARD_A001"
+        dest1 = root + "/RAID1/A001"
+        dest2 = root + "/SHUTTLE/A001"
+        try FileManager.default.createDirectory(atPath: source + "/CLIPS", withIntermediateDirectories: true)
+
+        // Source tree: two clips + sidecar.
+        try makeFile(source + "/CLIPS/A001C001.mov", size: 200_000, seed: 1)
+        try makeFile(source + "/CLIPS/A001C002.mov", size: 350_000, seed: 2)
+        try makeFile(source + "/A001.xml", size: 512, seed: 3)
+    }
+
+    override func tearDown() {
+        try? FileManager.default.removeItem(atPath: root)
+    }
+
+    func makeFile(_ path: String, size: Int, seed: UInt64) throws {
+        var data = Data(capacity: size)
+        var s = seed
+        while data.count < size {
+            s = s &* 6364136223846793005 &+ 1442695040888963407
+            withUnsafeBytes(of: s) { data.append(contentsOf: $0.prefix(min(8, size - data.count))) }
+        }
+        try data.write(to: URL(fileURLWithPath: path))
+    }
+
+    // Copy to two destinations: byte-identical, verified, report per file.
+    func testCopyTwoDestinationsVerified() throws {
+        let engine = CopyEngine()
+        let report = try engine.offload(source: source, destinations: [dest1, dest2])
+
+        XCTAssertEqual(report.files.count, 3)
+        XCTAssertTrue(report.files.allSatisfy { $0.verified })
+
+        for dest in [dest1!, dest2!] {
+            for rel in ["CLIPS/A001C001.mov", "CLIPS/A001C002.mov", "A001.xml"] {
+                let srcHash = try FileHasher.xxh64(path: source + "/" + rel)
+                let dstHash = try FileHasher.xxh64(path: dest + "/" + rel)
+                XCTAssertEqual(srcHash, dstHash, "hash mismatch for \(rel) in \(dest)")
+            }
+        }
+        // Report carries source hashes.
+        XCTAssertEqual(Set(report.files.map(\.relativePath)),
+                       ["CLIPS/A001C001.mov", "CLIPS/A001C002.mov", "A001.xml"])
+        XCTAssertTrue(report.files.allSatisfy { $0.xxh64.count == 16 })
+    }
+
+    // Existing identical file at destination: skipped, still verified.
+    func testExistingIdenticalSkipped() throws {
+        let engine = CopyEngine()
+        _ = try engine.offload(source: source, destinations: [dest1])
+        // Second run: all files exist + match.
+        let report2 = try engine.offload(source: source, destinations: [dest1])
+        XCTAssertTrue(report2.files.allSatisfy { $0.skipped })
+        XCTAssertTrue(report2.files.allSatisfy { $0.verified })
+    }
+
+    // Existing file with different content: hard error (never silently overwrite).
+    func testConflictErrors() throws {
+        let engine = CopyEngine()
+        _ = try engine.offload(source: source, destinations: [dest1])
+        // Tamper destination clip.
+        try makeFile(dest1 + "/CLIPS/A001C001.mov", size: 200_000, seed: 99)
+        XCTAssertThrowsError(try engine.offload(source: source, destinations: [dest1])) { error in
+            guard case OffloadError.destinationConflict = error else {
+                return XCTFail("wrong error: \(error)")
+            }
+        }
+    }
+
+    // Partial file (interrupted copy simulation): re-copied, ends verified.
+    func testPartialFileRecopied() throws {
+        let engine = CopyEngine()
+        // Simulate interrupted copy: partial temp content at final path with wrong size.
+        try FileManager.default.createDirectory(atPath: dest1 + "/CLIPS", withIntermediateDirectories: true)
+        try makeFile(dest1 + "/CLIPS/A001C001.mov", size: 50_000, seed: 1) // truncated
+        let report = try engine.offload(source: source, destinations: [dest1], overwritePartial: true)
+        let entry = report.files.first { $0.relativePath == "CLIPS/A001C001.mov" }!
+        XCTAssertFalse(entry.skipped)
+        XCTAssertTrue(entry.verified)
+        let srcHash = try FileHasher.xxh64(path: source + "/CLIPS/A001C001.mov")
+        XCTAssertEqual(try FileHasher.xxh64(path: dest1 + "/CLIPS/A001C001.mov"), srcHash)
+    }
+
+    // Missing source dir throws.
+    func testMissingSourceThrows() {
+        XCTAssertThrowsError(try CopyEngine().offload(source: root + "/nope", destinations: [dest1]))
+    }
+
+    // Report totals.
+    func testReportTotals() throws {
+        let report = try CopyEngine().offload(source: source, destinations: [dest1])
+        XCTAssertEqual(report.totalBytes, 200_000 + 350_000 + 512)
+        XCTAssertEqual(report.destinations, [dest1])
+        XCTAssertGreaterThan(report.duration, 0)
+    }
+}