Reshape Face Coordinate in Swift - swift

I want to reshape the face coordinate like showing in the video: https://www.dropbox.com/s/vsttylwgt25szha/IMG_6590.TRIM.MOV?dl=0 (Sorry, unfortunetly the video is about 11 MB in size).
I've just capture the face coordinate using iOS Vision API:
// Facial landmarks are GREEN.
fileprivate func drawFeatures(onFaces faces: [VNFaceObservation], onImageWithBounds bounds: CGRect) {
CATransaction.begin()
for faceObservation in faces {
let faceBounds = boundingBox(forRegionOfInterest: faceObservation.boundingBox, withinImageBounds: bounds)
guard let landmarks = faceObservation.landmarks else {
continue
}
// Iterate through landmarks detected on the current face.
let landmarkLayer = CAShapeLayer()
let landmarkPath = CGMutablePath()
let affineTransform = CGAffineTransform(scaleX: faceBounds.size.width, y: faceBounds.size.height)
// Treat eyebrows and lines as open-ended regions when drawing paths.
let openLandmarkRegions: [VNFaceLandmarkRegion2D?] = [
//landmarks.leftEyebrow,
//landmarks.rightEyebrow,
landmarks.faceContour,
landmarks.noseCrest,
// landmarks.medianLine
]
// Draw eyes, lips, and nose as closed regions.
let closedLandmarkRegions = [
landmarks.nose
].compactMap { $0 } // Filter out missing regions.
// Draw paths for the open regions.
for openLandmarkRegion in openLandmarkRegions where openLandmarkRegion != nil {
landmarkPath.addPoints(in: openLandmarkRegion!,
applying: affineTransform,
closingWhenComplete: false)
}
// Draw paths for the closed regions.
for closedLandmarkRegion in closedLandmarkRegions {
landmarkPath.addPoints(in: closedLandmarkRegion ,
applying: affineTransform,
closingWhenComplete: true)
}
// Format the path's appearance: color, thickness, shadow.
landmarkLayer.path = landmarkPath
landmarkLayer.lineWidth = 1
landmarkLayer.strokeColor = UIColor.green.cgColor
landmarkLayer.fillColor = nil
landmarkLayer.shadowOpacity = 1.0
landmarkLayer.shadowRadius = 1
// Locate the path in the parent coordinate system.
landmarkLayer.anchorPoint = .zero
landmarkLayer.frame = faceBounds
landmarkLayer.transform = CATransform3DMakeScale(1, -1, 1)
pathLayer?.addSublayer(landmarkLayer)
}
CATransaction.commit()
}
How to step forward from here? Can anyone guide me please?

Related

Converting SCNMatrix4 from swift to Objective-c

I can not for the life of me figure out how to create a SCNMatrix4 from a transform in objective-c.
The swift code I'm trying to use in objective-c:
let affineTransform = frame.displayTransform(for: .portrait, viewportSize: sceneView.bounds.size)
let transform = SCNMatrix4(affineTransform)
faceGeometry.setValue(SCNMatrix4Invert(transform), forKey: "displayTransform")
I got the first and third line but I can't find anyway to create this SCNMatrix4 from the CGAffineTransform.
CGAffineTransform affine = [self.sceneView.session.currentFrame displayTransformForOrientation:UIInterfaceOrientationPortrait viewportSize:self.sceneView.bounds.size];
SCNMatrix4 trans = ??
[f setValue:SCNMatrix4Invert(trans) forKey:#"displayTransform"];
There is no SCNMatrix4Make, I tried simd_matrix4x4 but that didn't seem to work either.
Thank you
edit:
The swift code is from Apples Example project "ARKitFaceExample", this is the full code:
/*
See LICENSE folder for this sample’s licensing information.
Abstract:
Demonstrates using video imagery to texture and modify the face mesh.
*/
import ARKit
import SceneKit
/// - Tag: VideoTexturedFace
class VideoTexturedFace: TexturedFace {
override func renderer(_ renderer: SCNSceneRenderer, nodeFor anchor: ARAnchor) -> SCNNode? {
guard let sceneView = renderer as? ARSCNView,
let frame = sceneView.session.currentFrame,
anchor is ARFaceAnchor
else { return nil }
#if targetEnvironment(simulator)
#error("ARKit is not supported in iOS Simulator. Connect a physical iOS device and select it as your Xcode run destination, or select Generic iOS Device as a build-only destination.")
#else
// Show video texture as the diffuse material and disable lighting.
let faceGeometry = ARSCNFaceGeometry(device: sceneView.device!, fillMesh: true)!
let material = faceGeometry.firstMaterial!
material.diffuse.contents = sceneView.scene.background.contents
material.lightingModel = .constant
guard let shaderURL = Bundle.main.url(forResource: "VideoTexturedFace", withExtension: "shader"),
let modifier = try? String(contentsOf: shaderURL)
else { fatalError("Can't load shader modifier from bundle.") }
faceGeometry.shaderModifiers = [ .geometry: modifier]
// Pass view-appropriate image transform to the shader modifier so
// that the mapped video lines up correctly with the background video.
let affineTransform = frame.displayTransform(for: .portrait, viewportSize: sceneView.bounds.size)
let transform = SCNMatrix4(affineTransform)
faceGeometry.setValue(SCNMatrix4Invert(transform), forKey: "displayTransform")
contentNode = SCNNode(geometry: faceGeometry)
#endif
return contentNode
}
}
In case anyone ever needs this, here is the extension I was missing
extension SCNMatrix4 {
/**
Create a 4x4 matrix from CGAffineTransform, which represents a 3x3 matrix
but stores only the 6 elements needed for 2D affine transformations.
[ a b 0 ] [ a b 0 0 ]
[ c d 0 ] -> [ c d 0 0 ]
[ tx ty 1 ] [ 0 0 1 0 ]
. [ tx ty 0 1 ]
Used for transforming texture coordinates in the shader modifier.
(Needs to be SCNMatrix4, not SIMD float4x4, for passing to shader modifier via KVC.)
*/
init(_ affineTransform: CGAffineTransform) {
self.init()
m11 = Float(affineTransform.a)
m12 = Float(affineTransform.b)
m21 = Float(affineTransform.c)
m22 = Float(affineTransform.d)
m41 = Float(affineTransform.tx)
m42 = Float(affineTransform.ty)
m33 = 1
m44 = 1
}
}
To replicate the Swift extension for creating an SCNMatrix4 from a CGAffineTransform you can implement the following function:
Some .h file:
extern SCNMatrix4 SCNMatrix4FromTransform(CGAffineTransform transform);
Some .m file:
SCNMatrix4 SCNMatrix4FromTransform(CGAffineTransform transform) {
SCNMatrix4 matrix;
matrix.m11 = transform.a;
matrix.m12 = transform.b;
matrix.m21 = transform.c;
matrix.m22 = transform.d;
matrix.m41 = transform.tx;
matrix.m42 = transform.ty;
matrix.m33 = 1;
matrix.m44 = 1;
return matrix;
}
Then your code becomes:
CGAffineTransform affineTransform = [self.sceneView.session.currentFrame displayTransformForOrientation:UIInterfaceOrientationPortrait viewportSize:self.sceneView.bounds.size];
SCNMatrix4 transform = SCNMatrixFromTransform(affineTransform);
[f setValue:[NSValue valueWithSCNMatrix4:SCNMatrix4Invert(transform)] forKey:#"displayTransform"];
Note the use of NSValue valueWithSCNMatrix4:. This is needed to convert the struct to an object and should satisfy the use of KVC for setting the displayTransform property.

Unable to get correct coordinates for QR code, ARKit - Swift

I am trying to place an object above a QR code in swift. I am able to detect the QR code however the location is wrong for the placement of the box. I don't really understand how the placement works. I know how to place objects down on planes. Do I need to relate the detected planes somehow with where it says it detects the QR code? Any information on how this SCNVector columns thing works would be appreciated as well haha. Also if CIDector is out dated and there is a new method.
Here is a snippet of detecting the QRCode and placing the box:
var discoveredQRCodes = String
func session(_ session: ARSession, didUpdate frame: ARFrame) {
//print("Updated")
if time != 0.5 {
return
}
DispatchQueue.global(qos: .background).async {
let image = CIImage(cvPixelBuffer: frame.capturedImage)
let detector = CIDetector(ofType: CIDetectorTypeQRCode, context: nil, options: nil)
let features = detector!.features(in: image)
for feature in features as! [CIQRCodeFeature] {
if !self.discoveredQRCodes.contains(feature.messageString!) {
self.discoveredQRCodes.append(feature.messageString!)
let url = URL(string: feature.messageString!)
let position = SCNVector3(frame.camera.transform.columns.3.x,
frame.camera.transform.columns.3.y,
frame.camera.transform.columns.3.z)
// add3DModel(fromURL: url!, toPosition: getPositionBasedOnQRCode(frame: frame, position: "df"))
print(position)
print(url)
DispatchQueue.main.async {
let boxNode = SCNNode()
boxNode.geometry = SCNBox(width: 0.04, height: 0.04, length: 0.04, chamferRadius: 0.002)
boxNode.geometry?.firstMaterial?.diffuse.contents = UIColor.green
boxNode.position = position
boxNode.name = "node"
self.arView.scene.rootNode.addChildNode(boxNode)
}
//add3dInstance(fromURL: url!, toPosition: position)
}
}
}
}
Here is an image of the result:
Here is some debug output:
SCNVector3(x: 0.023941405, y: 0.040143043, z: 0.056782123)
First of all, you set up the boxNode position to the camera position. It's not what you want.
Secondly, any QR code detector provides 2d bounding box coordinates in the image space. To translate 2d coordinates to scene coordinates you need to find a ray from the camera to the QR code plane.
Please, check the code here.

How to move and rotate SCNode using ARKit and Gesture Recognizer?

I am working on an AR based iOS app using ARKit(SceneKit). I used the Apple sample code https://developer.apple.com/documentation/arkit/handling_3d_interaction_and_ui_controls_in_augmented_reality as base for this. Using this i am able to move or rotate the whole Virtual Object.
But i want to select and move/rotate a Child Node in Virtual object using user finger, similar to how we move/rotate the whole Virtual Object itself.
I tried the below two links but it is only moving the child node in particular axis but not freely moving anywhere as the user moves the finger.
ARKit - Drag a node along a specific axis (not on a plane)
Dragging SCNNode in ARKit Using SceneKit
Also i tried replacing the Virtual Object which is a SCNReferenceNode with SCNode so that whatever functionality present for existing Virtual Object applies to Child Node as well, it is not working.
Can anyone please help me on how to freely move/rotate not only the Virtual Object but also the child node of a Virtual Object?
Please find the code i am currently using below,
let tapPoint: CGPoint = gesture.location(in: sceneView)
let result = sceneView.hitTest(tapPoint, options: nil)
if result.count == 0 {
return
}
let scnHitResult: SCNHitTestResult? = result.first
movedObject = scnHitResult?.node //.parent?.parent
let hitResults = self.sceneView.hitTest(tapPoint, types: .existingPlane)
if !hitResults.isEmpty{
guard let hitResult = hitResults.last else { return }
movedObject?.position = SCNVector3Make(hitResult.worldTransform.columns.3.x, hitResult.worldTransform.columns.3.y, hitResult.worldTransform.columns.3.z)
}
To move an object:
Perform a hitTest to check where you have touched, and detect which plane you touched and get a position. Move your SCNNode to that position by changing the node.position value with an SCNVector3.
Code:
#objc func panDetected(recognizer: UIPanGestureRecognizer){
let hitResult = self.arSceneView.hitTest(loc, types: .existingPlane)
if !hitResult.isEmpty{
guard let hitResult = hitResult.last else { return }
self.yourNode.position = SCNVector3Make(hitResult.worldTransform.columns.3.x, hitResult.worldTransform.columns.3.y, hitResult.worldTransform.columns.3.z)
}
The above code is enough to move your node over a detected plane, anywhere you touch, and not just in a single axis.
Rotating a node according to your gesture is a very difficult task and I have worked on a solution for quite sometime, never reaching a perfect output.
But, I have come across this repository in GitHub which allows you to do just that with a very impressive result.
https://github.com/Xartec/ScreenSpaceRotationAndPan
The Swift version of the code you require to rotate your node using your gesture would be :
var previousLoc: CGPoint?
var touchCount: Int?
#objc func panDetected(recognizer: UIPanGestureRecognizer){
let loc = recognizer.location(in: self.view)
var delta = recognizer.translation(in: self.view)
if recognizer.state == .began {
previousLoc = loc
touchCount = recognizer.numberOfTouches
}
else if gestureRecognizer.state == .changed {
delta = CGPoint.init(x: 2 * (loc.x - previousLoc.x), y: 2 * (loc.y - previousLoc.y))
previousLoc = loc
if touchCount != recognizer.numberOfTouches {
return
}
var rotMatrix: SCNMatrix4!
let rotX = SCNMatrix4Rotate(SCNMatrix4Identity, Float((1.0/100) * delta.y), 1, 0, 0)
let rotY = SCNMatrix4Rotate(SCNMatrix4Identity, Float((1.0 / 100) * delta.x), 0, 1, 0)
rotMatrix = SCNMatrix4Mult(rotX, rotY)
let transMatrix = SCNMatrix4MakeTranslation(yourNode.position.x, yourNode.position.y, yourNode.position.z)
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, SCNMatrix4Invert(transMatrix))
let parentNoderanslationMatrix = SCNMatrix4MakeTranslation((self.yourNode.parent?.worldPosition.x)!, (self.yourNode.parent?.worldPosition.y)!, (self.yourNode.parent?.worldPosition.z)!)
let parentNodeMatWOTrans = SCNMatrix4Mult((self.yourNode.parent?.worldTransform)!, SCNMatrix4Invert(parentNoderanslationMatrix))
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, parentNodeMatWOTrans)
let camorbitNodeTransMat = SCNMatrix4MakeTranslation((self.arSceneView.pointOfView?.worldPosition.x)!, (self.arSceneView.pointOfView?.worldPosition.y)!, (self.arSceneView.pointOfView?.worldPosition.z)!)
let camorbitNodeMatWOTrans = SCNMatrix4Mult((self.arSceneView.pointOfView?.worldTransform)!, SCNMatrix4Invert(camorbitNodeTransMat))
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, SCNMatrix4Invert(camorbitNodeMatWOTrans))
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, rotMatrix)
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, camorbitNodeMatWOTrans)
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, SCNMatrix4Invert(parentNodeMatWOTrans))
self.yourNode.transform = SCNMatrix4Mult(self.yourNode.transform, transMatrix)
}
}

Placing an object in front of camera at Touch Location

The following code places the node in front of the camera but always at the center 10cm away from the camera position. I want to place the node 10cm away in z-direction but at the x and y co-ordinates of where I touch the screen. So touching on different parts of the screen should result in a node being placed 10cm away in front of the camera but at the x and y location of the touch and not always at the center.
var cameraRelativePosition = SCNVector3(0,0,-0.1)
let sphere = SCNNode()
sphere.geometry = SCNSphere(radius: 0.0025)
sphere.geometry?.firstMaterial?.diffuse.contents = UIColor.white
Service.addChildNode(sphere, toNode: self.sceneView.scene.rootNode,
inView: self.sceneView, cameraRelativePosition:
cameraRelativePosition)
Service.swift
class Service: NSObject {
static func addChildNode(_ node: SCNNode, toNode: SCNNode, inView:
ARSCNView, cameraRelativePosition: SCNVector3) {
guard let currentFrame = inView.session.currentFrame else { return }
let camera = currentFrame.camera
let transform = camera.transform
var translationMatrix = matrix_identity_float4x4
translationMatrix.columns.3.x = cameraRelativePosition.x
translationMatrix.columns.3.y = cameraRelativePosition.y
translationMatrix.columns.3.z = cameraRelativePosition.z
let modifiedMatrix = simd_mul(transform, translationMatrix)
node.simdTransform = modifiedMatrix
toNode.addChildNode(node)
}
}
The result should look exactly like this : https://justaline.withgoogle.com
We can use the unprojectPoint(_:) method of SCNSceneRenderer (SCNView and ARSCNView both conform to this protocol) to convert a point on the screen to a 3D point.
When tapping the screen we can calculate a ray this way:
func getRay(for point: CGPoint, in view: SCNSceneRenderer) -> SCNVector3 {
let farPoint = view.unprojectPoint(SCNVector3(Float(point.x), Float(point.y), 1))
let nearPoint = view.unprojectPoint(SCNVector3(Float(point.x), Float(point.y), 0))
let ray = SCNVector3Make(farPoint.x - nearPoint.x, farPoint.y - nearPoint.y, farPoint.z - nearPoint.z)
// Normalize the ray
let length = sqrt(ray.x*ray.x + ray.y*ray.y + ray.z*ray.z)
return SCNVector3Make(ray.x/length, ray.y/length, ray.z/length)
}
The ray has a length of 1, so by multiplying it by 0.1 and adding the camera location we get the point you were searching for.

ARKit – How to put 3D Object on QRCode?

I'm actually trying to put a 3D Object on QRCode with ARKit
For that I use a AVCaptureDevice to detect a QRCode and establish the area of the QRCode that gives me a CGRect.
Then, I make a hitTest on every point of the CGRect to get the average 3D coordinates like so :
positionGiven = SCNVector3(0, 0, 0)
for column in Int(qrZone.origin.x)...2*Int(qrZone.origin.x + qrZone.width) {
for row in Int(qrZone.origin.y)...2*Int(qrZone.origin.y + qrZone.height) {
for result in sceneView.hitTest(CGPoint(x: CGFloat(column)/2,y:CGFloat(row)/2), types: [.existingPlaneUsingExtent,.featurePoint]) {
positionGiven.x+=result.worldTransform.columns.3.x
positionGiven.y+=result.worldTransform.columns.3.y
positionGiven.z+=result.worldTransform.columns.3.z
cpts += 1
}
}
}
positionGiven.x=positionGiven.x/cpts
positionGiven.y=positionGiven.y/cpts
positionGiven.z=positionGiven.z/cpts
But the hitTest doesn't detect any result and freeze the camera while when I make a hitTest with a touch on screen it works.
Do you have any idea why it's not working ?
Do you have an other idea that can help me to achieve what I want to do ?
I already thought about 3D translation with CoreMotion that can give me the tilt of the device but that seems really tedious.
I also heard about ARWorldAlignmentCamera that can locked the scene coordinate to match the orientation of the camera but I don't know how to use it !
Edit : I try to move my 3D Object every time I touch the screen and the hitTest is positive, and it's pretty accurate ! I really don't understand why hitTest on an area of pixels doesn't work...
Edit 2 : Here is the code of the hitTest who works with 2-5 touches on the screen:
#objc func touch(sender : UITapGestureRecognizer) {
for result in sceneView.hitTest(CGPoint(x: sender.location(in: view).x,y: sender.location(in: view).y), types: [.existingPlaneUsingExtent,.featurePoint]) {
//Pop up message for testing
alert("\(sender.location(in: view))", message: "\(result.worldTransform.columns.3)")
//Moving the 3D Object to the new coordinates
let objectList = sceneView.scene.rootNode.childNodes
for object : SCNNode in objectList {
object.removeFromParentNode()
}
addObject(SCNVector3(result.worldTransform.columns.3.x,result.worldTransform.columns.3.y,result.worldTransform.columns.3.z))
}
}
Edit 3 :
I manage to resolve my problem partially.
I take the transform matrix of the camera (session.currentFrame.camera.transform) so that the object is in front of the camera.
Then I apply a translation on (x,y) with the position of the CGRect.
However i can't translate the z-axis because i don't have enough informations.
And I will probably need a estimation of z coordinate like the hitTest do..
Thanks in advance ! :)
You could use Apple's Vision API to detect the QR code and place an anchor.
To start detecting QR codes, use:
var qrRequests = [VNRequest]()
var detectedDataAnchor: ARAnchor?
var processing = false
func startQrCodeDetection() {
// Create a Barcode Detection Request
let request = VNDetectBarcodesRequest(completionHandler: self.requestHandler)
// Set it to recognize QR code only
request.symbologies = [.QR]
self.qrRequests = [request]
}
In ARSession's didUpdate Frame
public func session(_ session: ARSession, didUpdate frame: ARFrame) {
DispatchQueue.global(qos: .userInitiated).async {
do {
if self.processing {
return
}
self.processing = true
// Create a request handler using the captured image from the ARFrame
let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: frame.capturedImage,
options: [:])
// Process the request
try imageRequestHandler.perform(self.qrRequests)
} catch {
}
}
}
Handle the Vision QR request and trigger the hit test
func requestHandler(request: VNRequest, error: Error?) {
// Get the first result out of the results, if there are any
if let results = request.results, let result = results.first as? VNBarcodeObservation {
guard let payload = result.payloadStringValue else {return}
// Get the bounding box for the bar code and find the center
var rect = result.boundingBox
// Flip coordinates
rect = rect.applying(CGAffineTransform(scaleX: 1, y: -1))
rect = rect.applying(CGAffineTransform(translationX: 0, y: 1))
// Get center
let center = CGPoint(x: rect.midX, y: rect.midY)
DispatchQueue.main.async {
self.hitTestQrCode(center: center)
self.processing = false
}
} else {
self.processing = false
}
}
func hitTestQrCode(center: CGPoint) {
if let hitTestResults = self.latestFrame?.hitTest(center, types: [.featurePoint] ),
let hitTestResult = hitTestResults.first {
if let detectedDataAnchor = self.detectedDataAnchor,
let node = self.sceneView.node(for: detectedDataAnchor) {
let previousQrPosition = node.position
node.transform = SCNMatrix4(hitTestResult.worldTransform)
} else {
// Create an anchor. The node will be created in delegate methods
self.detectedDataAnchor = ARAnchor(transform: hitTestResult.worldTransform)
self.sceneView.session.add(anchor: self.detectedDataAnchor!)
}
}
}
Then handle adding the node when the anchor is added.
func renderer(_ renderer: SCNSceneRenderer, nodeFor anchor: ARAnchor) -> SCNNode? {
// If this is our anchor, create a node
if self.detectedDataAnchor?.identifier == anchor.identifier {
let sphere = SCNSphere(radius: 1.0)
sphere.firstMaterial?.diffuse.contents = UIColor.redColor()
let sphereNode = SCNNode(geometry: sphere)
sphereNode.transform = SCNMatrix4(anchor.transform)
return sphereNode
}
return nil
}
Source