How to parse Hex with String, Float - swift

Question

I have binary files which containing names of place and coordinates ( latitude, longitude ), whenever I parse it to String using encoding .ascii it won't parse it well. I assume that parsing from Float values (coordinates) failing.
Reading InputStream
extension Data {
init(reading input: InputStream) {
self.init()
input.open()
let bufferSize = 1024
let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: bufferSize)
while input.hasBytesAvailable {
let read = input.read(buffer, maxLength: bufferSize)
self.append(buffer, count: read)
}
buffer.deallocate()
input.close()
}
}
File to parse
let filepath = Bundle.main.path(forResource: "MN", ofType: "dat")
let data = Data.init(reading: InputStream(fileAtPath: filepath)!)
let parsedData = String.init(data: data, encoding: .ascii)
Any ideas how could I parse it in correct way ?
For example Java ObjectInputStream have methods called:
inputStreamObj.readUTF()
inputStreamObj.readFloat()
Java

Answer 1

As I wrote in the comment, you need to read the spec Object Serialization Stream Protocol.
So, first 4 bytes represents STREAM_MAGIC, STREAM_VERSION, expected to be always the same value. And 5 byte sequence 0x7A 0xhh 0xhh 0xhh 0xhh represents TC_BLOCKDATALONG(0xhhhhhhhh).
And all blocks needs to be concatenated before parsing strings and floats.
So, preparing the DataReader:
(Nearly the same as Sulthan's, but this treats Modified UTF-8 correctly.)
struct DataReader {
enum DataReaderError: Error {
case invalidFirstByte(byte: UInt16, offset: Int)
case invalidFollowingByte
case missingFollowingByte
case insufficientData
}
var data: Data
var currentPosition: Int
init(data: Data) {
self.data = data
self.currentPosition = 0
}
mutating func skipBytes(_ n: Int) {
currentPosition += n
}
private mutating func readBigEndian<T: FixedWidthInteger>() throws -> T {
guard currentPosition + MemoryLayout<T>.size <= data.count else {
throw DataReaderError.insufficientData
}
var fixedWithInteger: T = 0
let range: Range<Int> = currentPosition ..< currentPosition + MemoryLayout<T>.size
withUnsafeMutableBytes(of: &fixedWithInteger) {ptrT in
let uint8Ptr = ptrT.baseAddress!.assumingMemoryBound(to: UInt8.self)
data.copyBytes(to: uint8Ptr, from: range)
}
currentPosition += MemoryLayout<T>.size
return fixedWithInteger.bigEndian
}
mutating func readFloat() throws -> Float {
let floatBits: UInt32 = try readBigEndian()
return Float(bitPattern: floatBits)
}
mutating func readUnsignedShort() throws -> Int {
let ushortValue: UInt16 = try readBigEndian()
return Int(ushortValue)
}
mutating func readInt() throws -> Int {
let intValue: Int32 = try readBigEndian()
return Int(intValue)
}
mutating func readUnsignedByte() throws -> Int {
guard currentPosition < data.count else {
throw DataReaderError.insufficientData
}
let byte = data[currentPosition]
currentPosition += 1
return Int(byte)
}
mutating func readBytes(_ n: Int) throws -> Data {
guard currentPosition + n <= data.count else {
throw DataReaderError.insufficientData
}
let subdata = data[currentPosition ..< currentPosition+n]
currentPosition += n
return subdata
}
mutating func readUTF() throws -> String {
//Get byte size of the string
let count = try readUnsignedShort()
//Decoding Modified UTF-8
var utf16: [UInt16] = []
var offset = 0
while offset < count {
let firstByte = UInt16(data[currentPosition + offset])
if firstByte & 0b1_0000000 == 0b0_0000000 {
utf16.append(firstByte)
offset += 1
} else if firstByte & 0b111_00000 == 0b110_00000 {
guard offset + 1 < count else {throw DataReaderError.missingFollowingByte}
let secondByte = UInt16(data[currentPosition + offset + 1])
guard secondByte & 0b11_000000 == 0b10_000000 else {throw DataReaderError.invalidFollowingByte}
let codeUnit = ((firstByte & 0b000_11111) << 6) | (secondByte & 0b00_111111)
utf16.append(codeUnit)
offset += 2
} else if firstByte & 0b1111_0000 == 0b1110_0000 {
guard offset + 2 < count else {throw DataReaderError.missingFollowingByte}
let secondByte = UInt16(data[currentPosition + offset + 1])
guard secondByte & 0b11_000000 == 0b10_000000 else {throw DataReaderError.invalidFollowingByte}
let thirdByte = UInt16(data[currentPosition + offset + 2])
guard thirdByte & 0b11_000000 == 0b10_000000 else {throw DataReaderError.invalidFollowingByte}
let codeUnit = ((firstByte & 0b0000_1111) << 12) | ((secondByte & 0b00_111111) << 6) | (thirdByte & 0b00_111111)
utf16.append(codeUnit)
offset += 3
} else {
throw DataReaderError.invalidFirstByte(byte: firstByte, offset: currentPosition+offset)
}
}
currentPosition += offset
return String(utf16CodeUnits: &utf16, count: utf16.count)
}
var isAtEnd: Bool {
return currentPosition == data.count
}
}
We can parse your MN.dat as follows:
let mnUrl = Bundle.main.url(forResource: "MN", withExtension: "dat")!
do {
let data = try Data(contentsOf: mnUrl)
var reader = DataReader(data: data)
reader.skipBytes(4)
//First collect all blocks
var blockData = Data()
while !reader.isAtEnd {
let contentType = try reader.readUnsignedByte()
if contentType == 0x7A {//TC_BLOCKDATALONG
let size = try reader.readInt()
let block = try reader.readBytes(size)
blockData.append(block)
} else if contentType == 0x77 {//TC_BLOCKDATA
let size = try reader.readUnsignedByte()
let block = try reader.readBytes(size)
blockData.append(block)
} else {
print("Unsupported content type")
break
}
}
//Then read the contents of blockData
var blockReader = DataReader(data: blockData)
while !blockReader.isAtEnd {
let string = try blockReader.readUTF()
print(string)
let float1 = try blockReader.readFloat()
print(float1)
let float2 = try blockReader.readFloat()
print(float2)
//Use string, float1, float2 as you like
}
} catch {
print(error)
}
Output:
Albert Lea
43.648
-93.3683
Albertville
45.2377
-93.6544
Alexandria
45.8852
-95.3775
(... no errors...)
Woodbury
44.9239
-92.9594
Worthington
43.62
-95.5964
Wyoming
45.3364
-92.9972
Zimmerman
45.4433
-93.59
You may need to modify the code above if your binary data may contain other content types.

Answer 2

I will show you how to parse Java-encoded data. However, since I cannot understand the format of the file, the response will not be complete:
First, load the file:
// load the file
let fileUrl = URL(fileURLWithPath: "/Users/sulthan/Downloads/MN.dat")
let data = try! Data(contentsOf: fileUrl)
Second, create a simple Java data reader:
// create a simple data reader
class Reader {
let data: Data
private var offset = 0
init(data: Data) {
self.data = data
}
var hasMoreData: Bool {
return offset < data.count
}
func skip(length: Int) {
offset += length
}
func readByte() -> UInt8 {
defer { offset += 1}
return data[offset]
}
// java bytes are unsigned
func readJavaByte() -> Int8 {
return Int8(bitPattern: readByte())
}
func readBytes(length: Int) -> Data {
defer { offset += length }
return data.subdata(in: offset ..< offset + length)
}
private func readJavaUShort() -> UInt16 {
let byte1 = UInt16(exactly: readByte())!
let byte2 = UInt16(exactly: readByte())!
return (byte1 << 8) | byte2
}
func readJavaShort() -> Int16 {
return Int16(bitPattern: readJavaUShort())
}
// Java UTF-8 encodes the length as first two bytes (unsigned java short)
func readJavaUtf() -> String? {
let length = readJavaUShort()
let data = readBytes(length: Int(length))
return String(data: data, encoding: .utf8)
}
private func readUInt32() -> UInt32 {
let short1 = UInt32(exactly: readJavaUShort())!
let short2 = UInt32(exactly: readJavaUShort())!
return (short1 << 16) | short2
}
func readJavaInt() -> Int32 {
let short1 = Int32(exactly: readJavaShort())!
let short2 = Int32(exactly: readJavaShort())!
return (short1 << 16) | short2
}
// interpret the 4 bytes as a floating point number
func readJavaFloat() -> Float {
let bits = readUInt32()
return Float(bitPattern: bits)
}
}
Third, parse the data. I cannot do this completely since the data format is unknown:
// create a reader from our data
let reader = Reader(data: data)
// some data I don't understand
reader.skip(length: 4)
var offset = 0
while reader.hasMoreData {
// some data I don't understand in the beginning and after every 52 items
if offset % 53 == 0 {
reader.skip(length: 5)
}
print(reader.readJavaUtf())
print(reader.readJavaFloat())
print(reader.readJavaFloat())
offset += 1
}
The data parsing will crash with the provided data after some items are parsed. I am assuming you know how to handle that since you know the format.