I’m trying to render a face mesh with RealityKit, no success yet. So when ARKit detected a human face, then ARSession generates an ARFaceAnchor which has a face geometry mesh in it.
But it cannot being generated as a model entity.
Could anyone help on this?
Canonical Face Mesh in RealityKit
To programmatically generate and render an ARKit's canonical face mesh (ARFaceGeometry object consisting of 1220 vertices) in RealityKit 2.0 use the following code:
import ARKit
import RealityKit
class ControllerView: UIViewController {
#IBOutlet var arView: ARView!
var anchor = AnchorEntity()
var model = ModelEntity()
override func viewDidLoad() {
super.viewDidLoad()
arView.automaticallyConfigureSession = false
arView.session.delegate = self
guard ARFaceTrackingConfiguration.isSupported
else {
fatalError("We can't run face tracking config")
}
let config = ARFaceTrackingConfiguration()
config.maximumNumberOfTrackedFaces = 1
arView.session.run(config)
}
}
Then create a method for converting face anchor's sub-properties. Note that I used for-in loop to convert indices from [Int16] to [UInt32] type (type casting doesn't help here).
extension ControllerView {
private func nutsAndBoltsOf(_ anchor: ARFaceAnchor) -> MeshDescriptor {
let vertices: [simd_float3] = anchor.geometry.vertices
var triangleIndices: [UInt32] = []
let texCoords: [simd_float2] = anchor.geometry.textureCoordinates
for index in anchor.geometry.triangleIndices { // [Int16]
triangleIndices.append(UInt32(index))
}
print(vertices.count) // 1220 vertices
var descriptor = MeshDescriptor(name: "canonical_face_mesh")
descriptor.positions = MeshBuffers.Positions(vertices)
descriptor.primitives = .triangles(triangleIndices)
descriptor.textureCoordinates = MeshBuffers.TextureCoordinates(texCoords)
return descriptor
}
}
And, at last, let's run a delegate's method to feed a mesh resource:
extension ControllerView: ARSessionDelegate {
func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
guard let faceAnchor = anchors[0] as? ARFaceAnchor else { return }
arView.session.add(anchor: faceAnchor)
self.anchor = AnchorEntity(anchor: faceAnchor)
self.anchor.scale *= 1.2
let mesh: MeshResource = try! .generate(from: [nutsAndBoltsOf(faceAnchor)])
var material = SimpleMaterial(color: .magenta, isMetallic: true)
self.model = ModelEntity(mesh: mesh, materials: [material])
self.anchor.addChild(self.model)
arView.scene.anchors.append(self.anchor)
}
}
Result (tested on iPad Pro 4th gen in iPadOS 16.2).
I also recommend you take a look at the post about visualizing detected planes in RealityKit 2.0.
Merry Christmas!
Related
I have set a virtual object in rear camera view. I want to move that object using facial expression with respect to world origin and measure the displacement angles of the virtual object.
Is that possible using ARKit or RealityKit?
Use the following solution. At first setup a configuration:
import RealityKit
import ARKit
class ViewController: UIViewController, ARSessionDelegate {
#IBOutlet var arView: ARView!
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
arView.session.delegate = self
arView.automaticallyConfigureSession = false
let config = ARFaceTrackingConfiguration()
config.isWorldTrackingEnabled = true // Simultaneous tracking
arView.session.run(config)
}
}
Run your transform animation when a defined facial expression occurs:
func facialExpression(anchor: ARFaceAnchor) {
let eyeUpLeft = anchor.blendShapes[.eyeLookUpLeft]
let eyeUpRight = anchor.blendShapes[.eyeLookUpRight]
if ((eyeUpLeft?.decimalValue ?? 0.0) +
(eyeUpRight?.decimalValue ?? 0.0)) > 0.75 {
// ModelEntity's animation goes here
}
}
Delegate's method (running at 60 fps):
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
guard let faceAnchor = anchors[0] as? ARFaceAnchor else { return }
self.facialExpression(anchor: faceAnchor)
}
The answer to your second question you can see HERE.
I have an rcproject file with about 12 scenes (500mb or so). In order to lessen the load on iOS devices I tried breaking it apart into separate rcproject files and change the scene using notification triggers. However when doing this and adding the new scene as a child to the main anchor, the new scene renders in a new spot, breaking the AR experience. There must be a way to add the new scenes to the exact same anchor/position. Alternatively, is there a better way than seperating the rcproject to lessen load on ram etc?
Here is my ARView
struct ARViewContainer: UIViewRepresentable {
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
// The experience consists of a "Base" (it acts as a permanent platform for all the scenes to be rendered on)
let baseAnchor = try! Base.loadIntro()
let introAnchor = try! IntroSceneOM.loadIntro()
introAnchor.actions.changeStoriesWithTrigger.onAction = loadStories
arView.scene.anchors.append(baseAnchor)
arView.scene.anchors.append(introAnchor)
func loadStories(_ entity: Entity?) -> Void {
arView.scene.anchors.remove(introAnchor)
let storiesAnchor = try! StoriesSceneOM.loadStoriesScene()
baseAnchor.addChild(storiesAnchor)
}
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {} }
EDIT:
Recreated the project using an implementation of Andy Jazz's code.
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
var anchor = AnchorEntity()
let scene01 = try! Experience.loadBoxScene()
let scene02 = try! Experience.loadBallScene()
// Base Scene
let scene03 = try! Experience.loadFloppyScene()
anchor = AnchorEntity(.plane(.horizontal, classification: .any,
minimumBounds: [0.1, 0.1]))
scene01.actions.boxTapped.onAction = loadScene02
scene02.actions.ballTapped.onAction = loadScene01
anchor.addChild(scene01)
anchor.addChild(scene03)
arView.scene.anchors.append(anchor)
func loadScene02(_ entity: Entity?) -> Void {
scene01.removeFromParent()
anchor.addChild(scene02)
}
func loadScene01(_ entity: Entity?) -> Void {
scene02.removeFromParent()
anchor.addChild(scene01)
}
return arView
}
However I still get the same issue where the anchor moves each time a new scene is added.
The code is quite simple, but regarding the issue of loading scenes with a large number of polygons, remains unresolved. At the maximum, the current scene should contain no more than 100K polygons, but ideally they should be within 50...70K. Texture resolution should not exceed 2K.
import RealityKit
class ViewController: UIViewController {
#IBOutlet var arView: ARView!
#IBOutlet var label: UILabel!
var anchor = AnchorEntity()
let scene01 = try! Experience.loadBox()
let scene02 = try! Experience.loadBall()
var cube = ModelEntity()
var sphere = ModelEntity()
override func viewDidLoad() {
super.viewDidLoad()
self.cube = scene01.steelBox?.children[0] as! ModelEntity
self.sphere = scene02.ball?.children[0] as! ModelEntity
self.anchor = AnchorEntity(.plane(.horizontal, classification: .any,
minimumBounds: [0.1, 0.1]))
self.anchor.addChild(cube)
arView.scene.anchors.append(anchor)
DispatchQueue.main.asyncAfter(deadline: .now() + 5.0) {
self.label.text = String(describing: self.anchor.id)
}
}
#IBAction func pressed(_ sender: UIButton) {
self.cube.removeFromParent()
self.anchor.addChild(sphere)
self.label.text = String(describing: self.anchor.id)
}
}
I'm adding a scene to my ARView that contains one entity. I was wondering how I would be able to get the distance between the entity's anchor and the camera. I know how to get the current position of the camera, but the position I'm getting for the arScene must be wrong since subtracting both positions doesn't yield the right position. Here is some code below.
import RealityKit
class ViewController: UIViewController, ARSessionDelegate {
let arScene = try! TestProj.loadScene()
#IBOutlet var sceneView: ARView!
override func viewDidLoad() {
...
sceneView.scene.addAnchor(arScene)
...
}
func session(_ session: ARSession, didUpdate frame: ARFrame) {
// Calculate distance here
let cameraPos = frame.camera.transform.columns.3
let entityPos = // Position of entity's anchor
let distance = // Find distance here
}
}
You need to perform a Convex Raycast against all the geometry in the RealityKit's scene for a ray between two end points:
import UIKit
import RealityKit
class ViewController: UIViewController {
#IBOutlet var arView: ARView!
override func viewDidLoad() {
super.viewDidLoad()
let entity = ModelEntity(mesh: .generateBox(size: 0.4))
entity.name = "Cube"
let anchor = AnchorEntity(world: [0,0,0])
anchor.addChild(entity)
arView.scene.anchors.append(anchor)
// For every entity that could be hit,
// we must generate a collision shape.
entity.generateCollisionShapes(recursive: true)
}
#IBAction func onTap(_ sender: UITapGestureRecognizer) {
let query: CollisionCastQueryType = .nearest
let mask: CollisionGroup = .default
let camera = arView.session.currentFrame?.camera
let x = (camera?.transform.columns.3.x)!
let y = (camera?.transform.columns.3.y)!
let z = (camera?.transform.columns.3.z)!
let transform: SIMD3<Float> = [x, y, z]
let raycasts: [CollisionCastHit] = arView.scene.raycast(
from: transform,
to: [0, 0, 0],
query: query,
mask: mask,
relativeTo: nil)
guard let raycast: CollisionCastHit = raycasts.first
else { return }
print(raycast.distance) // Distance from the ray origin to the hit
print(raycast.entity.name) // The entity that was hit
print(raycast.position) // The position of the hit
}
}
Also, as #maxxfrazer suggested, you can access camera transforms more easily:
let translate = arView.cameraTransform.translation
let x = translate.x
let y = translate.y
let z = translate.z
let transform: SIMD3<Float> = [x, y, z]
I want to show 3d modeling when reading Qr code and when it matches. But after reading the Qr code, the camera cannot see the surface, and when I look at the output, I get the error "VIO error callback: 161457.637109, 1, Frame processing rate has fallen below pre-set threshold". The camera works very slowly, I think there is a situation with this. Despite reading the barcode, 3d modeling does not turn on after reading the barcode.
enum FunctionMode {
case none
case placeObject(String)
case measure
}
class ARKitTestViewController: UIViewController {
#IBOutlet var sceneView: ARSCNView!
#IBOutlet weak var crosshair: UIView!
#IBOutlet weak var messageLabel: UILabel!
#IBOutlet weak var trackingInfo: UILabel!
var currentMode: FunctionMode = .none
var objects: [SCNNode] = []
// Current touch location
private var currTouchLocation: CGPoint?
let sequenceHandler = VNSequenceRequestHandler()
var isObjectAdded: Bool = false
var isQRCodeFound: Bool = false
var viewCenter:CGPoint = CGPoint()
override func viewDidLoad() {
super.viewDidLoad()
runARSession()
trackingInfo.text = ""
messageLabel.text = ""
viewCenter = CGPoint(x: view.bounds.width / 2.0, y: view.bounds.height / 2.0)
}
#IBAction func didTapReset(_ sender: Any) {
removeAllObjects()
}
func removeAllObjects() {
for object in objects {
object.removeFromParentNode()
}
objects = []
}
// MARK: - barcode handling
func searchQRCode(){
guard let frame = sceneView.session.currentFrame else {
return
}
let handler = VNImageRequestHandler(ciImage: CIImage(cvPixelBuffer: frame.capturedImage), options: [.properties : ""])
//DispatchQueue.global(qos: .userInteractive).async {
do {
try handler.perform([self.barcodeRequest])
} catch {
print(error)
}
//}
}
lazy var barcodeRequest: VNDetectBarcodesRequest = {
return VNDetectBarcodesRequest(completionHandler: self.handleBarcodes)
}()
func handleBarcodes(request: VNRequest, error: Error?) {
//print("handleBarcodes called")
guard let observations = request.results as? [VNBarcodeObservation]
else { fatalError("unexpected result type from VNBarcodeRequest") }
guard observations.first != nil else {
/*DispatchQueue.main.async {
print("No Barcode detected.")
}*/
return
}
// Loop through the found results
for result in request.results! {
print("Barcode detected")
// Cast the result to a barcode-observation
if let barcode = result as? VNBarcodeObservation {
if let payload = barcode.payloadStringValue {
let screenCentre : CGPoint = CGPoint(x: self.sceneView.bounds.midX, y: self.sceneView.bounds.midY)
let hitTestResults = sceneView.hitTest(screenCentre, types: [.existingPlaneUsingExtent])
//check payload
if let hitResult = hitTestResults.first {
// Get Coordinates of HitTest
let transform : matrix_float4x4 = hitResult.worldTransform
let worldCoord : SCNVector3 = SCNVector3Make(transform.columns.3.x, transform.columns.3.y, transform.columns.3.z)
let plane = SCNPlane(width: 0.1, height: 0.1)
let material = SCNMaterial()
material.diffuse.contents = UIColor.red
plane.materials = [material]
// Holder node
let node = SCNNode()
//node.transform = SCNMatrix4MakeRotation(-Float.pi / 2, 1, 0, 0)
//node.geometry = plane
sceneView.scene.rootNode.addChildNode(node)
node.position = worldCoord
//check payload
if(payload == "target_1"){
//Add 3D object
let objectScene = SCNScene(named: "Models.scnassets/candle/candle.scn")!
if let objectNode = objectScene.rootNode.childNode(withName: "candle", recursively: true) {
node.addChildNode(objectNode)
}
}
if(payload == "target_2"){
//Add 3D object
let objectScene = SCNScene(named: "Models.scnassets/lamp/lamp.scn")!
if let objectNode = objectScene.rootNode.childNode(withName: "lamp", recursively: true) {
node.addChildNode(objectNode)
}
}
isQRCodeFound = true
}
}
}
}
}
// MARK: - AR functions
func runARSession() {
// Registers ARKitTestViewController as ARSCNView delegate. You’ll use this later to render objects.
sceneView.delegate = self
// Uses ARWorldTrackingConfiguration to make use of all degrees of movement and give the best results. Remember, it supports A9 processors and up.
let configuration = ARWorldTrackingConfiguration()
// Turns on the automatic horizontal plane detection. You’ll use this to render planes for debugging and to place objects in the world.
configuration.planeDetection = .horizontal
// This turns on the light estimation calculations. ARSCNView uses that automatically and lights your objects based on the estimated light conditions in the real world.
configuration.isLightEstimationEnabled = true
// run(_:options) starts the ARKit session along with capturing video. This method will cause your device to ask for camera capture permission. If the user denies this request, ARKit won’t work.
sceneView.session.run(configuration)
// ASRCNView has an extra feature of rendering feature points. This turns it on for debug builds.
#if DEBUG
sceneView.debugOptions = ARSCNDebugOptions.showFeaturePoints
#endif
}
//Function that gives the user some feedback of the current tracking status.
func updateTrackingInfo() {
// You can get the current ARFrame thanks to the currentFrame property on the ARSession object.
guard let frame = sceneView.session.currentFrame else {
return
}
// The trackingState property can be found in the current frame’s ARCamera object. The trackingState enum value limited has an associated TrackingStateReason value which tells you the specific tracking problem.
switch frame.camera.trackingState {
case .limited(let reason):
switch reason {
case .excessiveMotion:
trackingInfo.text = "Limited Tracking: Excessive Motion"
case .insufficientFeatures:
trackingInfo.text =
"Limited Tracking: Insufficient Details"
default:
trackingInfo.text = "Limited Tracking"
}
default:
trackingInfo.text = "Good tracking conditions"
}
// You turned on light estimation in the ARWorldTrackingConfiguration, so it’s measured and provided in each ARFrame in the lightEstimate property.
guard
let lightEstimate = frame.lightEstimate?.ambientIntensity
else {
return
}
// ambientIntensity is given in lumen units. Less than 100 lumens is usually too dark, so you communicate this to the user.
if lightEstimate < 100 {
trackingInfo.text = "Limited Tracking: Too Dark"
}
}
}
extension ARKitTestViewController: ARSCNViewDelegate {
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
DispatchQueue.main.async {
if let planeAnchor = anchor as? ARPlaneAnchor {
#if DEBUG
let planeNode = createPlaneNode(center: planeAnchor.center, extent: planeAnchor.extent)
node.addChildNode(planeNode)
#endif
// else means that ARAnchor is not ARPlaneAnchor subclass, but just a regular ARAnchor instance you added in touchesBegan(_:with:)
} else {
// currentMode is a ARKitTestViewController property already added in the starter. It represents the current UI state: placeObject value if the object button is selected, or measure value if the measuring button is selected. The switch executes different code depending on the UI state.
switch self.currentMode {
case .none:
break
// placeObject has an associated string value which represents the path to the 3D model .scn file. You can browse all the 3D models in Models.scnassets.
case .placeObject(let name):
// nodeWithModelName(_:) creates a new 3D model SCNNode with the given path name. It’s a helper function provided with the starter project.
let modelClone = nodeWithModelName(name)
// Append the node to the objects array provided with the starter.
self.objects.append(modelClone)
// Finally, you add your new object node to the SCNNode provided to the delegate method.
node.addChildNode(modelClone)
// You’ll implement measuring later.
case .measure:
break
}
}
}
}
func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
DispatchQueue.main.async {
if let planeAnchor = anchor as? ARPlaneAnchor {
// Update the child node, which is the plane node you added earlier in renderer(_:didAdd:for:). updatePlaneNode(_:center:extent:) is a function included with the starter that updates the coordinates and size of the plane to the updated values contained in ARPlaneAnchor.
updatePlaneNode(node.childNodes[0], center: planeAnchor.center, extent: planeAnchor.extent)
}
}
}
func renderer(_ renderer: SCNSceneRenderer, didRemove node: SCNNode,
for anchor: ARAnchor) {
guard anchor is ARPlaneAnchor else { return }
// Removes the plane from the node if the corresponding ARAnchorPlane has been removed. removeChildren(inNode:) was provided with the starter project as well.
removeChildren(inNode: node)
}
func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) {
DispatchQueue.main.async {
// Updates tracking info for each rendered frame.
self.updateTrackingInfo()
if(!self.isQRCodeFound){
self.searchQRCode()
}
// If the dot in the middle hit tests with existingPlaneUsingExtent type, it turns green to indicate high quality hit testing to the user.
if let _ = self.sceneView.hitTest(
self.viewCenter,
types: [.existingPlaneUsingExtent]).first {
self.crosshair.backgroundColor = UIColor.green
} else {
self.crosshair.backgroundColor = UIColor(white: 0.34, alpha: 1)
}
}
}
func session(_ session: ARSession, didFailWithError error: Error) {
print("ARSession error: \(error.localizedDescription)")
let message = error.localizedDescription
messageLabel.text = message
DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
if self.messageLabel.text == message {
self.messageLabel.text = ""
}
}
}
// sessionWasInterrupted(_:) is called when a session is interrupted, like when your app is backgrounded.
func sessionWasInterrupted(_ session: ARSession) {
print("Session interrupted")
let message = "Session interrupted"
messageLabel.text = message
DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
if self.messageLabel.text == message {
self.messageLabel.text = ""
}
}
}
func sessionInterruptionEnded(_ session: ARSession) {
print("Session resumed")
let message = "Session resumed"
messageLabel.text = message
DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
if self.messageLabel.text == message {
self.messageLabel.text = ""
}
}
// When sessionInterruptionEnded(_:) is called, you should remove all your objects and restart the AR session by calling the runSession() method you implemented before. removeAllObjects() is a helper method provided with the starter project.
removeAllObjects()
runARSession()
}
}
I want to achieve the billboard effect in RealityKit (the plane always look at the camera), I used the Entity.Look() method, but the result is weird, I can't even see the plane, the scripts I used as below, so, what is the problem?
struct ARViewContainer: UIViewRepresentable {
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
let config = ARWorldTrackingConfiguration()
config.planeDetection = .horizontal
arView.session.run(config, options:[ ])
arView.session.delegate = arView
arView.createPlane()
return arView
}
func updateUIView(_ uiView: ARView, context: Context) { }
}
var planeMesh = MeshResource.generatePlane(width: 0.15, height: 0.15)
var planeMaterial = SimpleMaterial(color:.white,isMetallic: false)
var planeEntity = ModelEntity(mesh:planeMesh,materials:[planeMaterial])
var arCameraPostion : SIMD3<Float>!
var isPlaced = false
extension ARView : ARSessionDelegate{
func createPlane(){
let planeAnchor = AnchorEntity(plane:.horizontal)
planeAnchor.addChild(planeEntity)
self.scene.addAnchor(planeAnchor)
//planeAnchor.transform.rotation = simd_quatf(angle: .pi, axis: [0,1,0])
}
public func session(_ session: ARSession, didUpdate frame: ARFrame){
guard let arCamera = session.currentFrame?.camera else { return }
if isPlaced {
arCameraPostion = SIMD3(arCamera.transform.columns.3.x,0,arCamera.transform.columns.3.z)
planeEntity.look(at: arCameraPostion, from: planeEntity.position, upVector: [0, 1, 0],relativeTo: nil)
}
}
public func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
isPlaced = true
}
}
session(_:didUpdate:) method
Try the following logic to implement a "billboard" behavior for RealityKit camera. You can use this code as a starting point. It generates a rotation of the model around its local Y axis based on camera position.
import RealityKit
import ARKit
class ViewController: UIViewController {
#IBOutlet var arView: ARView!
var model = Entity()
override func viewDidLoad() {
super.viewDidLoad()
arView.session.delegate = self
let config = ARWorldTrackingConfiguration()
arView.session.run(config)
self.model = try! ModelEntity.load(named: "drummer")
let anchor = AnchorEntity(world: [0, 0, 0])
anchor.addChild(self.model)
arView.scene.anchors.append(anchor)
}
}
A pivot point of the model must be in the center of it (not at some distance from the model).
extension ViewController: ARSessionDelegate {
func session(_ session: ARSession, didUpdate frame: ARFrame) {
let camTransform: float4x4 = arView.cameraTransform.matrix
let alongXZPlane: simd_float4 = camTransform.columns.3
let yaw: Float = atan2(alongXZPlane.x - model.position.x,
alongXZPlane.z - model.position.z)
print(yaw)
// Identity matrix 4x4
var positionAndScale = float4x4()
// position
positionAndScale.columns.3.z = -0.25
// scale
positionAndScale.columns.0.x = 0.01
positionAndScale.columns.1.y = 0.01
positionAndScale.columns.2.z = 0.01
// orientation matrix
let orientation = Transform(pitch: 0, yaw: yaw, roll: 0).matrix
// matrices multiplication
let transform = simd_mul(positionAndScale, orientation)
self.model.transform.matrix = transform
}
}
subscribe(to:on:_:) method
Alternatively, you can implement a subscription to the event stream.
import RealityKit
import Combine
class ViewController: UIViewController {
#IBOutlet var arView: ARView!
var model = Entity()
var subs: [AnyCancellable] = []
override func viewDidLoad() {
super.viewDidLoad()
self.model = try! ModelEntity.load(named: "drummer")
let anchor = AnchorEntity(world: [0, 0, 0])
anchor.addChild(self.model)
arView.scene.anchors.append(anchor)
arView.scene.subscribe(to: SceneEvents.Update.self) { _ in
let camTransform: float4x4 = self.arView.cameraTransform.matrix
let alongXZPlane: simd_float4 = camTransform.columns.3
let yaw: Float = atan2(alongXZPlane.x - self.model.position.x,
alongXZPlane.z - self.model.position.z)
var positionAndScale = float4x4()
positionAndScale.columns.3.z = -0.25
positionAndScale.columns.0.x = 0.01
positionAndScale.columns.1.y = 0.01
positionAndScale.columns.2.z = 0.01
let orientation = Transform(pitch: 0, yaw: yaw, roll: 0).matrix
let transform = simd_mul(positionAndScale, orientation)
self.model.transform.matrix = transform
}.store(in: &subs)
}
}