swift: SHA1 from file without copy it to ram memory [duplicate] - swift

Question

I'm working on creating the MD5 Checksum for large video files. I'm currently using the code:
extension NSData {
func MD5() -> NSString {
let digestLength = Int(CC_MD5_DIGEST_LENGTH)
let md5Buffer = UnsafeMutablePointer<CUnsignedChar>.allocate(capacity: digestLength)
CC_MD5(bytes, CC_LONG(length), md5Buffer)
let output = NSMutableString(capacity: Int(CC_MD5_DIGEST_LENGTH * 2))
for i in 0..<digestLength {
output.appendFormat("%02x", md5Buffer[i])
}
return NSString(format: output)
}
}
But that creates a memory buffer, and for large video files would not be ideal. Is there a way in Swift to calculate the MD5 Checksum reading a file stream, so the memory footprint will be minimal?

Answer 1

You can compute the MD5 checksum in chunks, as demonstrated
e.g. in Is there a MD5 library that doesn't require the whole input at the same time?.
Here is a possible implementation using Swift (now updated for Swift 5)
import CommonCrypto
func md5File(url: URL) -> Data? {
let bufferSize = 1024 * 1024
do {
// Open file for reading:
let file = try FileHandle(forReadingFrom: url)
defer {
file.closeFile()
}
// Create and initialize MD5 context:
var context = CC_MD5_CTX()
CC_MD5_Init(&context)
// Read up to `bufferSize` bytes, until EOF is reached, and update MD5 context:
while autoreleasepool(invoking: {
let data = file.readData(ofLength: bufferSize)
if data.count > 0 {
data.withUnsafeBytes {
_ = CC_MD5_Update(&context, $0.baseAddress, numericCast(data.count))
}
return true // Continue
} else {
return false // End of file
}
}) { }
// Compute the MD5 digest:
var digest: [UInt8] = Array(repeating: 0, count: Int(CC_MD5_DIGEST_LENGTH))
_ = CC_MD5_Final(&digest, &context)
return Data(digest)
} catch {
print("Cannot open file:", error.localizedDescription)
return nil
}
}
The autorelease pool is needed to release the memory returned by
file.readData(), without it the entire (potentially huge) file
would be loaded into memory. Thanks to Abhi Beckert for noticing that
and providing an implementation.
If you need the digest as a hex-encoded string then change the
return type to String? and replace
return digest
by
let hexDigest = digest.map { String(format: "%02hhx", $0) }.joined()
return hexDigest

Answer 2

Since iOS13
'CC_MD5_Init' was deprecated in iOS 13.0
You may replace the code with CryptoKit
import Foundation
import CryptoKit
extension URL {
func checksumInBase64() -> String? {
let bufferSize = 16*1024
do {
// Open file for reading:
let file = try FileHandle(forReadingFrom: self)
defer {
file.closeFile()
}
// Create and initialize MD5 context:
var md5 = CryptoKit.Insecure.MD5()
// Read up to `bufferSize` bytes, until EOF is reached, and update MD5 context:
while autoreleasepool(invoking: {
let data = file.readData(ofLength: bufferSize)
if data.count > 0 {
md5.update(data: data)
return true // Continue
} else {
return false // End of file
}
}) { }
// Compute the MD5 digest:
let data = Data(md5.finalize())
return data.base64EncodedString()
} catch {
error_log(error)
return nil
}
}
}

Answer 3

Solution (based on Martin R's answer) for SHA256 hash:
func sha256(url: URL) -> Data? {
do {
let bufferSize = 1024 * 1024
// Open file for reading:
let file = try FileHandle(forReadingFrom: url)
defer {
file.closeFile()
}
// Create and initialize SHA256 context:
var context = CC_SHA256_CTX()
CC_SHA256_Init(&context)
// Read up to `bufferSize` bytes, until EOF is reached, and update SHA256 context:
while autoreleasepool(invoking: {
// Read up to `bufferSize` bytes
let data = file.readData(ofLength: bufferSize)
if data.count > 0 {
data.withUnsafeBytes {
_ = CC_SHA256_Update(&context, $0, numericCast(data.count))
}
// Continue
return true
} else {
// End of file
return false
}
}) { }
// Compute the SHA256 digest:
var digest = Data(count: Int(CC_SHA256_DIGEST_LENGTH))
digest.withUnsafeMutableBytes {
_ = CC_SHA256_Final($0, &context)
}
return digest
} catch {
print(error)
return nil
}
}
Usage with instance of type URL with name fileURL previously created:
if let digestData = sha256(url: fileURL) {
let calculatedHash = digestData.map { String(format: "%02hhx", $0) }.joined()
DDLogDebug(calculatedHash)
}