zgaetano/rainbow-dragon
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

offload: checksum breadth — vendored xxHash 0.8.2 (XXH3-64/128, xxhsum-verified vectors), pure-Swift SHA1 (RFC 3174), HashAlgorithm dispatch, MHL per-algo elements + algo-aware verify, forge offload --checksum — 8 tests, 214 total

Browse source
This commit is contained in:
Forge Dev 2026-07-11 19:46:41 +00:00
parent e32ee6eced
commit 03fa5ca2d4
10 changed files with 7237 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Package.swift
View file

@ -43,10 +43,12 @@ let package = Package(
// Shot logging + storage + exports
.target(name: "ForgeLibrary", dependencies: ["ForgeGrade", "CSQLite"]),
.systemLibrary(name: "CSQLite", path: "Sources/CSQLite"),
// Vendored xxHash v0.8.2 (BSD-2) for XXH3-64/128
.target(name: "CXXHash"),
.target(name: "ForgeLogger", dependencies: ["ForgeCamera", "ForgeLibrary"]),
.target(name: "ForgeExport", dependencies: ["ForgeLibrary", "ForgeGrade"]),
// Offload + proxy
.target(name: "ForgeOffload", dependencies: ["ForgeLibrary"]),
.target(name: "ForgeOffload", dependencies: ["ForgeLibrary", "CXXHash"]),
.target(name: "ForgeProxy", dependencies: ["ForgeLibrary", "ForgeGrade", "ForgeColor"]),
// Panel + video seams
.target(name: "ForgePanel", dependencies: [

48
Sources/CXXHash/cxxhash.c Normal file
View file

@ -0,0 +1,48 @@
#define XXH_INLINE_ALL
#include "vendor_xxhash.h"
#include "include/cxxhash.h"
#include <stdlib.h>
uint64_t cxx_xxh3_64(const void *data, size_t len) {
return XXH3_64bits(data, len);
}
cxx_hash128 cxx_xxh3_128(const void *data, size_t len) {
XXH128_hash_t h = XXH3_128bits(data, len);
cxx_hash128 out = { h.low64, h.high64 };
return out;
}
struct cxx_xxh3_state {
XXH3_state_t *st;
int is128;
};
cxx_xxh3_state *cxx_xxh3_create(int is128) {
cxx_xxh3_state *s = malloc(sizeof(cxx_xxh3_state));
if (!s) return NULL;
s->st = XXH3_createState();
s->is128 = is128;
if (is128) XXH3_128bits_reset(s->st);
else XXH3_64bits_reset(s->st);
return s;
}
void cxx_xxh3_update(cxx_xxh3_state *s, const void *data, size_t len) {
if (s->is128) XXH3_128bits_update(s->st, data, len);
else XXH3_64bits_update(s->st, data, len);
}
uint64_t cxx_xxh3_digest64(cxx_xxh3_state *s) {
return XXH3_64bits_digest(s->st);
}
cxx_hash128 cxx_xxh3_digest128(cxx_xxh3_state *s) {
XXH128_hash_t h = XXH3_128bits_digest(s->st);
cxx_hash128 out = { h.low64, h.high64 };
return out;
}
void cxx_xxh3_free(cxx_xxh3_state *s) {
if (s) { XXH3_freeState(s->st); free(s); }
}

20
Sources/CXXHash/include/cxxhash.h Normal file
View file

@ -0,0 +1,20 @@
/* Forge shim over vendored xxHash v0.8.2 (BSD-2, Yann Collet).
Exposes stable one-shot + streaming XXH3-64/128 entry points for Swift. */
#ifndef CXXHASH_H
#define CXXHASH_H
#include <stddef.h>
#include <stdint.h>
typedef struct { uint64_t low64; uint64_t high64; } cxx_hash128;
uint64_t cxx_xxh3_64(const void *data, size_t len);
cxx_hash128 cxx_xxh3_128(const void *data, size_t len);
/* streaming */
typedef struct cxx_xxh3_state cxx_xxh3_state;
cxx_xxh3_state *cxx_xxh3_create(int is128);
void cxx_xxh3_update(cxx_xxh3_state *s, const void *data, size_t len);
uint64_t cxx_xxh3_digest64(cxx_xxh3_state *s);
cxx_hash128 cxx_xxh3_digest128(cxx_xxh3_state *s);
void cxx_xxh3_free(cxx_xxh3_state *s);
#endif

6773
Sources/CXXHash/vendor_xxhash.h Normal file
View file

File diff suppressed because it is too large Load diff

66
Sources/ForgeOffload/HashAlgorithm.swift Normal file
View file

@ -0,0 +1,66 @@
import Foundation
/// Checksum algorithms supported for offload/MHL. Element names match
/// ASC MHL v2.0 spec hash-element vocabulary.
public enum HashAlgorithm: String, Sendable, CaseIterable {
case xxh64
case xxh3
case xxh128
case md5
case sha1
public var mhlElement: String {
rawValue
}
/// In-memory one-shot (test/reference path).
public static func hashInMemory(_ data: Data, algorithm: HashAlgorithm) -> String {
switch algorithm {
case .xxh64:
return XXHash64.hexString(XXHash64.hash(data))
case .xxh3:
return XXH3.hexString64(XXH3.hash64(data))
case .xxh128:
return XXH3.hash128(data).hexString
case .md5:
return MD5.hexString(data)
case .sha1:
return SHA1.hexString(data)
}
}
}
extension FileHasher {
/// Chunked file hash with algorithm dispatch.
public static func hash(path: String, algorithm: HashAlgorithm) throws -> String {
switch algorithm {
case .xxh64:
return try xxh64(path: path)
case .md5:
return try md5(path: path)
case .xxh3:
let s = XXH3.Streaming(bits: .h64)
try streamFile(path: path) { s.update($0) }
return XXH3.hexString64(s.digest64())
case .xxh128:
let s = XXH3.Streaming(bits: .h128)
try streamFile(path: path) { s.update($0) }
return s.digest128().hexString
case .sha1:
var sha = SHA1()
try streamFile(path: path) { sha.update($0) }
return sha.finalize().map { String(format: "%02x", $0) }.joined()
}
}
static func streamFile(path: String, _ body: (Data) -> Void) throws {
guard let handle = FileHandle(forReadingAtPath: path) else {
throw OffloadError.fileNotFound(path)
}
defer { try? handle.close() }
while true {
guard let chunk = try handle.read(upToCount: chunkSize), !chunk.isEmpty else { break }
body(chunk)
}
}
}

44
Sources/ForgeOffload/MHL.swift
View file

@ -49,7 +49,8 @@ public enum MHL {
root: String,
creator: String,
files: [String],
process: String = "transfer"
process: String = "transfer",
algorithm: HashAlgorithm = .xxh64
) throws -> String {
let fm = FileManager.default
let now = isoNow()
@ -73,11 +74,12 @@ public enum MHL {
throw OffloadError.fileNotFound(full)
}
let size = (try fm.attributesOfItem(atPath: full)[.size] as? UInt64) ?? 0
let hash = try FileHasher.xxh64(path: full)
let hash = try FileHasher.hash(path: full, algorithm: algorithm)
let el = algorithm.mhlElement
out += """
<hash>
<path size="\(size)">\(escape(rel))</path>
<xxh64 action="original" hashdate="\(now)">\(hash)</xxh64>
<\(el) action="original" hashdate="\(now)">\(hash)</\(el)>
</hash>
"""
@ -92,9 +94,10 @@ public enum MHL {
creator: String,
files: [String],
to path: String,
process: String = "transfer"
process: String = "transfer",
algorithm: HashAlgorithm = .xxh64
) throws {
let xml = try generate(root: root, creator: creator, files: files, process: process)
let xml = try generate(root: root, creator: creator, files: files, process: process, algorithm: algorithm)
try xml.write(toFile: path, atomically: true, encoding: .utf8)
}
@ -116,8 +119,8 @@ public enum MHL {
continue
}
}
let hash = try FileHasher.xxh64(path: full)
if hash != e.xxh64 {
let hash = try FileHasher.hash(path: full, algorithm: e.algorithm)
if hash != e.hashValue64 {
failures.append(Failure(path: e.path, reason: .hashMismatch))
}
}
@ -127,7 +130,8 @@ public enum MHL {
struct Entry {
let path: String
let size: UInt64?
let xxh64: String
let algorithm: HashAlgorithm
let hashValue64: String
}
/// Scanner-based parse of <hash> blocks. Accepts v2.0 attribute-style size
@ -163,15 +167,25 @@ public enum MHL {
size = UInt64(block[sizeOpen.upperBound..<sizeClose.lowerBound].trimmingCharacters(in: .whitespacesAndNewlines))
}
// xxh64 element with optional attributes.
guard let hOpen = block.range(of: "<xxh64"),
let hTagEnd = block.range(of: ">", range: hOpen.upperBound..<block.endIndex),
let hClose = block.range(of: "</xxh64>", range: hTagEnd.upperBound..<block.endIndex) else {
throw OffloadError.readFailed("malformed <hash> entry: no xxh64")
// Hash element: first recognized algorithm element in the block.
var found: (HashAlgorithm, String)?
for algo in HashAlgorithm.allCases {
let el = algo.mhlElement
guard let hOpen = block.range(of: "<\(el)"),
let hTagEnd = block.range(of: ">", range: hOpen.upperBound..<block.endIndex),
let hClose = block.range(of: "</\(el)>", range: hTagEnd.upperBound..<block.endIndex) else {
continue
}
let value = String(block[hTagEnd.upperBound..<hClose.lowerBound])
.trimmingCharacters(in: .whitespacesAndNewlines)
found = (algo, value)
break
}
guard let (algorithm, hash) = found else {
throw OffloadError.readFailed("malformed <hash> entry: no recognized hash element")
}
let hash = String(block[hTagEnd.upperBound..<hClose.lowerBound]).trimmingCharacters(in: .whitespacesAndNewlines)
entries.append(Entry(path: path, size: size, xxh64: hash))
entries.append(Entry(path: path, size: size, algorithm: algorithm, hashValue64: hash))
searchRange = blockEnd.upperBound..<xml.endIndex
}
guard !entries.isEmpty else {

98
Sources/ForgeOffload/SHA1.swift Normal file
View file

@ -0,0 +1,98 @@
import Foundation
/// SHA-1 (RFC 3174) — pure Swift streaming. Legacy post-house compatibility.
public struct SHA1 {
private var h0: UInt32 = 0x67452301
private var h1: UInt32 = 0xEFCDAB89
private var h2: UInt32 = 0x98BADCFE
private var h3: UInt32 = 0x10325476
private var h4: UInt32 = 0xC3D2E1F0
private var buffer = [UInt8]()
private var totalLength: UInt64 = 0
public init() {}
public mutating func update(_ data: Data) {
totalLength += UInt64(data.count)
buffer.append(contentsOf: data)
var offset = 0
while buffer.count - offset >= 64 {
processBlock(Array(buffer[offset..<offset + 64]))
offset += 64
}
if offset > 0 {
buffer.removeFirst(offset)
}
}
@inline(__always)
private static func rotl(_ x: UInt32, _ n: UInt32) -> UInt32 {
(x << n) | (x >> (32 - n))
}
private mutating func processBlock(_ block: [UInt8]) {
var w = [UInt32](repeating: 0, count: 80)
for i in 0..<16 {
w[i] = UInt32(block[i * 4]) << 24 | UInt32(block[i * 4 + 1]) << 16
| UInt32(block[i * 4 + 2]) << 8 | UInt32(block[i * 4 + 3])
}
for i in 16..<80 {
w[i] = Self.rotl(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1)
}
var a = h0, b = h1, c = h2, d = h3, e = h4
for i in 0..<80 {
let (f, k): (UInt32, UInt32)
switch i {
case 0..<20: (f, k) = ((b & c) | (~b & d), 0x5A827999)
case 20..<40: (f, k) = (b ^ c ^ d, 0x6ED9EBA1)
case 40..<60: (f, k) = ((b & c) | (b & d) | (c & d), 0x8F1BBCDC)
default: (f, k) = (b ^ c ^ d, 0xCA62C1D6)
}
let temp = Self.rotl(a, 5) &+ f &+ e &+ k &+ w[i]
e = d
d = c
c = Self.rotl(b, 30)
b = a
a = temp
}
h0 = h0 &+ a
h1 = h1 &+ b
h2 = h2 &+ c
h3 = h3 &+ d
h4 = h4 &+ e
}
public mutating func finalize() -> Data {
let bitLength = totalLength * 8
var padding: [UInt8] = [0x80]
let remainder = (buffer.count + 1) % 64
let zeros = remainder <= 56 ? 56 - remainder : 120 - remainder
padding.append(contentsOf: [UInt8](repeating: 0, count: zeros))
// Length big-endian.
for shift in stride(from: 56, through: 0, by: -8) {
padding.append(UInt8((bitLength >> UInt64(shift)) & 0xFF))
}
buffer.append(contentsOf: padding)
var offset = 0
while buffer.count - offset >= 64 {
processBlock(Array(buffer[offset..<offset + 64]))
offset += 64
}
buffer.removeAll()
var out = Data()
for v in [h0, h1, h2, h3, h4] {
out.append(UInt8((v >> 24) & 0xFF))
out.append(UInt8((v >> 16) & 0xFF))
out.append(UInt8((v >> 8) & 0xFF))
out.append(UInt8(v & 0xFF))
}
return out
}
public static func hexString(_ data: Data) -> String {
var sha = SHA1()
sha.update(data)
return sha.finalize().map { String(format: "%02x", $0) }.joined()
}
}

67
Sources/ForgeOffload/XXH3.swift Normal file
View file

@ -0,0 +1,67 @@
import Foundation
import CXXHash
/// XXH3 64/128 via vendored official xxHash v0.8.2 (BSD-2).
public enum XXH3 {
public struct Hash128: Equatable, Sendable {
public let low64: UInt64
public let high64: UInt64
public var hexString: String {
String(format: "%016lx%016lx", high64, low64)
}
}
public static func hash64(_ data: Data) -> UInt64 {
data.withUnsafeBytes { buf in
cxx_xxh3_64(buf.baseAddress, buf.count)
}
}
public static func hash128(_ data: Data) -> Hash128 {
let h = data.withUnsafeBytes { buf in
cxx_xxh3_128(buf.baseAddress, buf.count)
}
return Hash128(low64: h.low64, high64: h.high64)
}
public static func hexString64(_ v: UInt64) -> String {
String(format: "%016lx", v)
}
/// Streaming state.
public final class Streaming {
public enum Bits {
case h64
case h128
}
private let state: OpaquePointer?
private let bits: Bits
public init(bits: Bits) {
self.bits = bits
state = cxx_xxh3_create(bits == .h128 ? 1 : 0)
}
deinit {
cxx_xxh3_free(state)
}
public func update(_ data: Data) {
data.withUnsafeBytes { buf in
cxx_xxh3_update(state, buf.baseAddress, buf.count)
}
}
public func digest64() -> UInt64 {
cxx_xxh3_digest64(state)
}
public func digest128() -> Hash128 {
let h = cxx_xxh3_digest128(state)
return Hash128(low64: h.low64, high64: h.high64)
}
}
}

11
Sources/forge-cli/Main.swift
View file

@ -167,17 +167,26 @@ struct Offload: AsyncParsableCommand {
@Option(name: .long, help: "Write HTML report to this path")
var report: String?
@Option(name: .long, help: "Checksum: xxh64|xxh3|xxh128|md5|sha1")
var checksum: String = "xxh64"
func run() async throws {
guard !destinations.isEmpty else {
throw ValidationError("at least one --to destination required")
}
guard let algo = HashAlgorithm(rawValue: checksum) else {
throw ValidationError("unknown checksum '\(checksum)' (xxh64|xxh3|xxh128|md5|sha1)")
}
let engine = CopyEngine()
let result = try engine.offload(source: source, destinations: destinations, overwritePartial: overwritePartial)
// MHL manifest at each destination root.
let files = result.files.map(\.relativePath)
for dest in destinations {
try MHL.write(root: dest, creator: "Forge", files: files, to: dest + "/" + ((source as NSString).lastPathComponent) + ".mhl")
try MHL.write(
root: dest, creator: "Forge", files: files,
to: dest + "/" + ((source as NSString).lastPathComponent) + ".mhl",
algorithm: algo)
}
if let reportPath = report {
try OffloadReportWriter.html(result).write(toFile: reportPath, atomically: true, encoding: .utf8)

123
Tests/ForgeOffloadTests/ChecksumBreadthTests.swift Normal file
View file

@ -0,0 +1,123 @@
import XCTest
import Foundation
@testable import ForgeOffload
final class ChecksumBreadthTests: XCTestCase {
// XXH3-64 vectors verified against official xxhsum 0.8.2 (-H3, seed 0).
func testXXH3_64Vectors() {
XCTAssertEqual(XXH3.hash64(Data()), 0x2D06_800538D394C2)
XCTAssertEqual(XXH3.hash64(Data("xxhash".utf8)), 0xAA4C_2B42AE6B13DE)
XCTAssertEqual(
XXH3.hash64(Data("Nobody inspects the spammish repetition".utf8)),
0x6CB0_0603B5CC47E9)
}
// XXH3-128 vectors verified against official xxhsum 0.8.2 (-H128).
// Canonical hex = high64 then low64.
func testXXH3_128Vectors() {
let empty = XXH3.hash128(Data())
XCTAssertEqual(empty.hexString, "99aa06d3014798d86001c324468d497f")
let xx = XXH3.hash128(Data("xxhash".utf8))
XCTAssertEqual(xx.hexString, "9c8b437c78cac00a376072e24bfdf4d2")
}
// Streaming XXH3 == one-shot across chunk boundaries.
func testXXH3StreamingMatchesOneShot() {
var data = Data()
var seed: UInt64 = 0xABCDEF
for _ in 0..<(2 * 1024 * 1024 / 8) {
seed = seed &* 6364136223846793005 &+ 1442695040888963407
withUnsafeBytes(of: seed) { data.append(contentsOf: $0) }
}
let whole64 = XXH3.hash64(data)
let whole128 = XXH3.hash128(data)
let s64 = XXH3.Streaming(bits: .h64)
let s128 = XXH3.Streaming(bits: .h128)
var offset = 0
let chunks = [1, 63, 64, 65, 4096, 999_999]
var i = 0
while offset < data.count {
let size = min(chunks[i % chunks.count], data.count - offset)
let chunk = data.subdata(in: offset..<offset + size)
s64.update(chunk)
s128.update(chunk)
offset += size
i += 1
}
XCTAssertEqual(s64.digest64(), whole64)
let d128 = s128.digest128()
XCTAssertEqual(d128.low64, whole128.low64)
XCTAssertEqual(d128.high64, whole128.high64)
}
// SHA1 RFC 3174 vectors.
func testSHA1Vectors() {
XCTAssertEqual(SHA1.hexString(Data("abc".utf8)), "a9993e364706816aba3e25717850c26c9cd0d89d")
XCTAssertEqual(
SHA1.hexString(Data("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq".utf8)),
"84983e441c3bd26ebaae4aa1f95129e5e54670f1")
XCTAssertEqual(SHA1.hexString(Data()), "da39a3ee5e6b4b0d3255bfef95601890afd80709")
}
// SHA1 streaming across block boundary.
func testSHA1Streaming() {
let text = String(repeating: "a", count: 1_000_000)
// RFC 3174 test: one million 'a' -> 34aa973cd4c4daa4f61eeb2bdbad27316534016f
var sha = SHA1()
let data = Data(text.utf8)
var offset = 0
while offset < data.count {
let size = min(77_777, data.count - offset)
sha.update(data.subdata(in: offset..<offset + size))
offset += size
}
let digest = sha.finalize()
XCTAssertEqual(digest.map { String(format: "%02x", $0) }.joined(),
"34aa973cd4c4daa4f61eeb2bdbad27316534016f")
}
// FileHasher algorithm dispatch: all algos over same temp file.
func testFileHasherAllAlgorithms() throws {
let path = NSTemporaryDirectory() + "forge-algo-\(UUID().uuidString).bin"
defer { try? FileManager.default.removeItem(atPath: path) }
var data = Data()
for i in 0..<50_000 { data.append(UInt8(truncatingIfNeeded: i)) }
try data.write(to: URL(fileURLWithPath: path))
for algo in HashAlgorithm.allCases {
let fileHash = try FileHasher.hash(path: path, algorithm: algo)
let memHash = HashAlgorithm.hashInMemory(data, algorithm: algo)
XCTAssertEqual(fileHash, memHash, "\(algo) file != memory")
XCTAssertFalse(fileHash.isEmpty)
}
}
// MHL element names per algorithm (ASC MHL v2 spec names).
func testMHLElementNames() {
XCTAssertEqual(HashAlgorithm.xxh64.mhlElement, "xxh64")
XCTAssertEqual(HashAlgorithm.xxh3.mhlElement, "xxh3")
XCTAssertEqual(HashAlgorithm.xxh128.mhlElement, "xxh128")
XCTAssertEqual(HashAlgorithm.md5.mhlElement, "md5")
XCTAssertEqual(HashAlgorithm.sha1.mhlElement, "sha1")
}
// MHL generate with alternate algorithm embeds right element + verifies.
func testMHLWithXXH3() throws {
let root = NSTemporaryDirectory() + "forge-mhl3-\(UUID().uuidString)"
defer { try? FileManager.default.removeItem(atPath: root) }
try FileManager.default.createDirectory(atPath: root, withIntermediateDirectories: true)
try Data("clip data".utf8).write(to: URL(fileURLWithPath: root + "/C1.mov"))
let xml = try MHL.generate(root: root, creator: "Forge", files: ["C1.mov"], algorithm: .xxh3)
XCTAssertTrue(xml.contains("<xxh3 action=\"original\""))
let result = try MHL.verify(manifest: xml, root: root)
XCTAssertTrue(result.passed)
// Tamper still caught with xxh3.
try Data("clip DATA".utf8).write(to: URL(fileURLWithPath: root + "/C1.mov"))
let bad = try MHL.verify(manifest: xml, root: root)
XCTAssertFalse(bad.passed)
}
}