Collide RealityKit 3D model with LiDAR mesh - swift

Question

I spent many days trying to understand and follow examples without success.
My goal is to place a virtual AR object to the real world scanned previously with the LiDAR. With the showSceneUnderstanding I can see the realtime mesh created ok that fine.
With a tap function I can insert a usdz file, that also fine.
Because I have toyModel.physicsBody?.mode = .kinematic and self.arView.installGestures(for: toyRobot) I can move/scale the model.
Now want to be able move the model AND collide with the mesh generated by the LiDAR. When I move the model to a scanned wall the mesh it's stopped for example.
Here is my complete code :
import UIKit
import RealityKit
import ARKit
class ViewController: UIViewController, ARSessionDelegate {
#IBOutlet var arView: ARView!
var tapRecognizer = UITapGestureRecognizer()
override func viewDidLoad() {
super.viewDidLoad()
self.arView.session.delegate = self
//Scene Understanding options
self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
//Only for dev
self.arView.debugOptions.insert(.showSceneUnderstanding)
self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(placeObject(_:)))
self.arView.addGestureRecognizer(self.tapRecognizer)
}
#objc func placeObject(_ sender: UITapGestureRecognizer) {
// Perform a ray cast against the mesh (sceneUnderstanding)
// Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
let tapLocation = sender.location(in: arView)
if let result = arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {
// Load the "Toy robot"
let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")
// Add gestures to the toy (only available is physicsBody mode == kinematic)
self.arView.installGestures(for: toyRobot)
// Toy Anchor to place the toy on surface
let toyAnchor = AnchorEntity(world: result.worldTransform)
toyAnchor.addChild(toyRobot)
// Create a "Physics" model of the toy in order to add physics mode
guard let toyModel = toyAnchor.children.first as? HasPhysics else {
return
}
// Because toyModel is a fresh new model we need to init physics
toyModel.generateCollisionShapes(recursive: true)
toyModel.physicsBody = .init()
// Add the physics body mode
toyModel.physicsBody?.mode = .kinematic
let test = ShapeResource.generateConvex(from: toyRobot.model!.mesh)
toyModel.components[CollisionComponent] = CollisionComponent(shapes: [test], mode: .default, filter: .default)
// Finally add the toy anchor to the scene
self.arView.scene.addAnchor(toyAnchor)
}
}
}
Someone knows if it's possible to achieve that ? Many thanks in advance!

Answer 1

Following the previous discussion with #AndyFedoroff I added convext raycast in order, always, to collide the placed 3d object with the LiDAR created mesh. Here is my full code. I don't know if I'm doing well... In any case still doesn't work.
import UIKit
import RealityKit
import ARKit
class ViewController: UIViewController, ARSessionDelegate {
#IBOutlet var arView: ARView!
var tapRecognizer = UITapGestureRecognizer()
var panGesture = UIPanGestureRecognizer()
var panGestureEntity: Entity? = nil
override func viewDidLoad() {
super.viewDidLoad()
self.arView.session.delegate = self
//Scene Understanding options
self.arView.environment.sceneUnderstanding.options.insert([.physics, .collision, .occlusion])
//Only for dev
self.arView.debugOptions.insert(.showSceneUnderstanding)
self.tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(self.placeObject))
self.arView.addGestureRecognizer(self.tapRecognizer)
self.panGesture = UIPanGestureRecognizer(target: self, action: #selector(self.didPan))
self.arView.addGestureRecognizer(self.panGesture)
}
#objc func placeObject(_ sender: UITapGestureRecognizer) {
// Perform a ray cast against the mesh (sceneUnderstanding)
// Note: Ray-cast option ".estimatedPlane" with alignment ".any" also takes the mesh into account.
let tapLocation = sender.location(in: arView)
if let result = self.arView.raycast(from: tapLocation, allowing: .estimatedPlane, alignment: .any).first {
// Load the "Toy robot"
let toyRobot = try! ModelEntity.loadModel(named: "toy_robot_vintage.usdz")
// Add gestures to the toy (only available is physicsBody mode == kinematic)
self.arView.installGestures(for: toyRobot)
// Toy Anchor to place the toy on surface
let toyAnchor = AnchorEntity(world: result.worldTransform)
toyAnchor.addChild(toyRobot)
// Create a "Physics" model of the toy in order to add physics mode
guard let model = toyAnchor.children.first as? (Entity & HasPhysics)
else { return }
model.generateCollisionShapes(recursive: true)
model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
mass: 1.0,
material: .default,
mode: .kinematic)
// Finally add the toy anchor to the scene
self.arView.scene.addAnchor(toyAnchor)
}
}
#objc public func didPan(_ sender: UIPanGestureRecognizer) {
print(sender.state)
let point = sender.location(in: self.arView)
switch sender.state {
case .began:
if let entity = self.arView.hitTest(point).first?.entity {
self.panGestureEntity = entity
} else {
self.panGestureEntity = nil
}
case .changed:
if let entity = self.panGestureEntity {
if let raycast = arView
.scene
.raycast(origin: .zero, direction: .one, length: 1, query: .all, mask: .all, relativeTo: entity)
.first
{
print("hit", raycast.entity, raycast.distance)
}
}
case .ended:
self.panGestureEntity = nil
default: break;
}
}
}

Answer 2

Try downcasting to Entity & HasPhysics:
guard let model = anchor.children.first as? (Entity & HasPhysics)
else { return }
model.generateCollisionShapes(recursive: true)
model.physicsBody = PhysicsBodyComponent(shapes: [.generateBox(size: .one)],
mass: 1.0,
material: .default,
mode: .kinematic)
Official documentation says:
For non-model entities, generateCollisionShapes(recursive:) method has no effect. Nevertheless, the method is defined for all entities so that you can call it on any entity, and have the calculation propagate recursively to all that entity’s descendants.