How to setup bone hierarchy for SCNSkinner from glb data? - swift

Question

I'm trying to show a 3d model of a human body scan in SceneKit. The model is stored in a glb file. Because SceneKit does not support the import of that file format I manually read the file in. I'm trying to use the SCNSkinner from SceneKit to rig the model with the Mixamo-rig included in the glb file. The model is correctly shown as long as I don't apply any bone positions unequal (0, 0, 0), rotations or set boneInverseBindTransforms unequal the Identity matrix. But as soon as I apply any of the afore mentioned properties, the model starts to look unnatural.
Here is the code to create the SCNSkinner object:
private func createSCNSkinnerFrom(bgArmature: BgArmature, baseGeometry: SCNGeometry) -> SCNSkinner? {
var joints: [UInt8] = bgArmature.mesh.joints.joined().map { element in UInt8(element) }
var weights: [Float] = bgArmature.mesh.weights.joined().map { element in element }
let bones: [SCNNode] = self.createBoneNodesList(bgRig: bgArmature.rig!)
let boneIndicesData = Data(bytesNoCopy: &joints, count: joints.count * MemoryLayout<UInt8>.size, deallocator: .none)
let boneIndicesGeometrySource = SCNGeometrySource(data: boneIndicesData, semantic: .boneIndices, vectorCount: joints.count/4, usesFloatComponents: false, componentsPerVector: 4, bytesPerComponent: MemoryLayout<UInt8>.size, dataOffset: 0, dataStride: MemoryLayout<UInt8>.size * 4)
let boneWeightsData = Data(bytesNoCopy: &weights, count: weights.count * MemoryLayout<Float>.size, deallocator: .none)
let boneWeightsGeometrySource = SCNGeometrySource(data: boneWeightsData, semantic: .boneWeights, vectorCount: weights.count/4, usesFloatComponents: true, componentsPerVector: 4, bytesPerComponent: MemoryLayout<Float>.size, dataOffset: 0, dataStride: MemoryLayout<Float>.size * 4)
let boneInverseBindTransforms: [NSValue]? = self.createListOfBoneInverseBindTransforms(bgBones: bgArmature.rig!.bones)
let skinner = SCNSkinner(baseGeometry: baseGeometry,
bones: bones,
boneInverseBindTransforms: boneInverseBindTransforms,
boneWeights: boneWeightsGeometrySource,
boneIndices: boneIndicesGeometrySource)
return skinner
}
Here the function to create a bone node and the list of bones:
private func createBoneNodeWithoutChildren(bgBone: BgBone) -> SCNNode {
let bone = SCNNode()
if let name = bgBone.name {
bone.name = name
}
if let translation = bgBone.translation {
bone.simdPosition = SIMD3<Float>(translation[0]!, translation[1]!, translation[2]!)
}
if let rotation = bgBone.rotation {
bone.simdOrientation = simd_quatf(ix: rotation[0]!, iy: rotation[1]!, iz: rotation[2]!, r: rotation[3]!)
}
if let scale = bgBone.scale {
bone.simdScale = SIMD3<Float>(scale[0]!, scale[1]!, scale[2]!)
}
return bone
}
private func createBoneNodesList(bgRig: BgRig) -> [SCNNode] {
var bonesList: [SCNNode] = []
for bone in bgRig.bones {
bonesList.append(self.createBoneNodeWithoutChildren(bgBone: bone))
}
return bonesList
}
The function to create the boneInverseTransforms:
private func createListOfBoneInverseBindTransforms(bgBones: [BgBone]!) -> [NSValue]? {
var boneInverseBindTransforms: [NSValue]? = []
for bone in bgBones {
boneInverseBindTransforms?.append(NSValue(scnMatrix4: bone.inverseBindMatrix!.getSCNMatrix4()))
}
return boneInverseBindTransforms
}
And the BgBone class with the BgMat4 struct:
class BgBone {
var index: Int?
var children: [Int?]?
var name: String?
var rotation: [Float?]?
var scale: [Float?]?
var translation: [Float?]?
var inverseBindMatrix: BgMat4?
init(index: Int? = nil, children: [Int?]? = nil, name: String? = nil, rotation: [Float?]? = nil, scale: [Float?]? = nil, translation: [Float?]? = nil) {
self.index = index
self.children = children
self.name = name
self.rotation = rotation
self.scale = scale
self.translation = translation
}
}
struct BgMat4: sizeable {
var r1c1: Float
var r2c1: Float
var r3c1: Float
var r4c1: Float
var r1c2: Float
var r2c2: Float
var r3c2: Float
var r4c2: Float
var r1c3: Float
var r2c3: Float
var r3c3: Float
var r4c3: Float
var r1c4: Float
var r2c4: Float
var r3c4: Float
var r4c4: Float
init(fromData: Data) {
/// Accessors of matrix type have data stored in column-major order; start of each column MUST be aligned to 4-byte boundaries.
/// Specifically, when ROWS * SIZE_OF_COMPONENT (where ROWS is the number of rows of the matrix) is not a multiple of 4,
/// then (ROWS * SIZE_OF_COMPONENT) % 4 padding bytes MUST be inserted at the end of each column.
self.r1c1 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 0, as: Float32.self) }
self.r2c1 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 4, as: Float32.self) }
self.r3c1 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 8, as: Float32.self) }
self.r4c1 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 12, as: Float32.self) }
self.r1c2 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 16, as: Float32.self) }
self.r2c2 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 20, as: Float32.self) }
self.r3c2 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 24, as: Float32.self) }
self.r4c2 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 28, as: Float32.self) }
self.r1c3 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 32, as: Float32.self) }
self.r2c3 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 36, as: Float32.self) }
self.r3c3 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 40, as: Float32.self) }
self.r4c3 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 44, as: Float32.self) }
self.r1c4 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 48, as: Float32.self) }
self.r2c4 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 52, as: Float32.self) }
self.r3c4 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 56, as: Float32.self) }
self.r4c4 = fromData.withUnsafeBytes { rawBuffer in rawBuffer.load(fromByteOffset: 60, as: Float32.self) }
}
func getSCNMatrix4() -> SCNMatrix4 {
return SCNMatrix4(m11: self.r1c1,
m12: self.r2c1,
m13: self.r3c1,
m14: self.r4c1,
m21: self.r1c2,
m22: self.r2c2,
m23: self.r3c2,
m24: self.r4c2,
m31: self.r1c3,
m32: self.r2c3,
m33: self.r3c3,
m34: self.r4c3,
m41: self.r1c4,
m42: self.r2c4,
m43: self.r3c4,
m44: self.r4c4)
}
}
With this code the model looks like this:
With Positions==(0, 0, 0), Rotation(0, 0, 0), bindInverseTransformation=Identity it looks like this:
But with the Positions, Rotations and bindInverseTransformations applied I expect the model to be in the A-Pose (similar to the following image):
What am I doing wrong with my bones (SCNNodes) or skinner (SCNSkinner) object?