AVAudioEngine, using 3D Audio - swift

I would like to use AVAudioEngine for a 3D audio effect, where the sound source circles the user head. The source appears to move from left to right but I've been unable to figure out how to make it circle the users head. The audio source must be mono or it cant work.
I dont understand AVAudioMake3DVectorOrientation and AVAudioMake3DAngularOrientation.
I thought my math was correct but I suspect that if it were, I would have gotten the results I was looking for.
This is bare bones, so there isn't much error checking.
Would someone provide guidance to get me on track?
Thank you,
W.
import AVFoundation
class ThreeDAudio {
var _angleIndx = 0.0
var _engine : AVAudioEngine!
var _player : AVAudioPlayerNode!
var _environment : AVAudioEnvironmentNode!
var _circleTimer : Timer!
func initTimer()
{
_circleTimer = Timer.scheduledTimer(timeInterval: 0.10, target: self,
selector: #selector(ThreeDAudio.updatePosition),userInfo: nil, repeats: true)
}
#objc func updatePosition()
{
let centerX = 0.0
let centerY = 0.0
let radius = 10.0
let degToRads = Double.pi / 180.0
let angle = _angleIndx * degToRads
let x = centerX + sin(angle) * radius
let y = centerY + cos(angle) * radius
let z = 0.0
let posInSpace = AVAudioMake3DPoint(Float(x), Float(y), Float(z))
_angleIndx += 1.0
_player.position = posInSpace
print("angle: \(_angleIndx) , \(posInSpace)")
if(_angleIndx == 360.0) { _circleTimer.invalidate() }
}
func getBufferFromFileInBundle(fileName: String, fileType: String) -> AVAudioPCMBuffer?
{
// audio MUST be a monoaural source or it cant work in 3D
var file:AVAudioFile
var audioBuffer : AVAudioPCMBuffer? = nil
let path = Bundle.main.path(forResource: fileName, ofType: fileType)!
do{
file = try AVAudioFile(forReading: URL(fileURLWithPath:path))
audioBuffer = AVAudioPCMBuffer(pcmFormat:(file.processingFormat), frameCapacity: AVAudioFrameCount(file.length))
try file.read(into: audioBuffer!)
} catch let error as NSError {
print("Error AVAudioFile:\(error)")
}
return audioBuffer
}
func outputFormat() -> AVAudioFormat
{
let outputFormat = _engine.outputNode.outputFormat(forBus: 0)
let nChannels = outputFormat.channelCount // testing, will always be 2 channels
let sampleRate = outputFormat.sampleRate
return AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: nChannels)
}
func setupEngine(_ audioBuffer: AVAudioPCMBuffer )
{
_engine = AVAudioEngine()
_player = AVAudioPlayerNode()
_environment = AVAudioEnvironmentNode()
_player.renderingAlgorithm = .HRTF
_engine.attach(_player)
_engine.attach(_environment)
_engine.connect(_player, to: _environment, format: audioBuffer.format)
_engine.connect(_environment, to: _engine.outputNode, format: outputFormat())
_environment.listenerPosition = AVAudioMake3DPoint(0.0, 0.0, 0.0);
_environment.listenerVectorOrientation = AVAudioMake3DVectorOrientation(AVAudioMake3DVector(0, 0, -1), AVAudioMake3DVector(0, 0, 0))
_environment.listenerAngularOrientation = AVAudioMake3DAngularOrientation(0.0,0.0, 0.0)
do{
try _engine.start()
} catch let error as NSError {
print("Error start:\(error)")
}
}
func startAudioTest()
{
var thisFile = (name: "", fileType: "")
thisFile = (name: "sound_voices", fileType: "wav")
thisFile = (name: "Bouncing-Ball-MONO", fileType: "aiff")
let audioBuffer = getBufferFromFileInBundle(fileName: thisFile.name, fileType: thisFile.fileType )
if ( audioBuffer != nil )
{
setupEngine( audioBuffer! )
initTimer()
_player.scheduleBuffer(audioBuffer!, at: nil, options: .loops, completionHandler: nil)
_player.play()
}
}
}

Related

Accessing Recently Stored Files (The file “insert file” couldn’t be opened because there is no such file)

In my video recording app, I am trying to implement a merge video function which takes in an array of URLs which were recently recorded in the app.
extension AVMutableComposition {
func mergeVideo(_ urls: [URL], completion: #escaping (_ url: URL?, _ error: Error?) -> Void) {
guard let documentDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else {
completion(nil, nil)
return
}
let dateFormatter = DateFormatter()
dateFormatter.dateStyle = .long
dateFormatter.timeStyle = .short
let date = dateFormatter.string(from: Date())
let outputURL = documentDirectory.appendingPathComponent("mergedVideo_\(date).mp4")
// If there is only one video, we dont to touch it to save export time.
if let url = urls.first, urls.count == 1 {
do {
try FileManager().copyItem(at: url, to: outputURL)
completion(outputURL, nil)
} catch let error {
completion(nil, error)
}
return
}
let maxRenderSize = CGSize(width: 1280.0, height: 720.0)
var currentTime = CMTime.zero
var renderSize = CGSize.zero
// Create empty Layer Instructions, that we will be passing to Video Composition and finally to Exporter.
var instructions = [AVMutableVideoCompositionInstruction]()
urls.enumerated().forEach { index, url in
let asset = AVAsset(url: url)
print(asset)
let assetTrack = asset.tracks.first!
// Create instruction for a video and append it to array.
let instruction = AVMutableComposition.instruction(assetTrack, asset: asset, time: currentTime, duration: assetTrack.timeRange.duration, maxRenderSize: maxRenderSize)
instructions.append(instruction.videoCompositionInstruction)
// Set render size (orientation) according first videro.
if index == 0 {
renderSize = instruction.isPortrait ? CGSize(width: maxRenderSize.height, height: maxRenderSize.width) : CGSize(width: maxRenderSize.width, height: maxRenderSize.height)
}
do {
let timeRange = CMTimeRangeMake(start: .zero, duration: assetTrack.timeRange.duration)
// Insert video to Mutable Composition at right time.
try insertTimeRange(timeRange, of: asset, at: currentTime)
currentTime = CMTimeAdd(currentTime, assetTrack.timeRange.duration)
} catch let error {
completion(nil, error)
}
}
// Create Video Composition and pass Layer Instructions to it.
let videoComposition = AVMutableVideoComposition()
videoComposition.instructions = instructions
// Do not forget to set frame duration and render size. It will crash if you dont.
videoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
videoComposition.renderSize = renderSize
guard let exporter = AVAssetExportSession(asset: self, presetName: AVAssetExportPresetHighestQuality) else {
completion(nil, nil)
return
}
exporter.outputURL = outputURL
exporter.outputFileType = .mp4
// Pass Video Composition to the Exporter.
exporter.videoComposition = videoComposition
exporter.shouldOptimizeForNetworkUse = true
exporter.exportAsynchronously {
DispatchQueue.main.async {
completion(exporter.outputURL, nil)
}
}
}
static func instruction(_ assetTrack: AVAssetTrack, asset: AVAsset, time: CMTime, duration: CMTime, maxRenderSize: CGSize)
-> (videoCompositionInstruction: AVMutableVideoCompositionInstruction, isPortrait: Bool) {
let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: assetTrack)
// Find out orientation from preferred transform.
let assetInfo = orientationFromTransform(assetTrack.preferredTransform)
// Calculate scale ratio according orientation.
var scaleRatio = maxRenderSize.width / assetTrack.naturalSize.width
if assetInfo.isPortrait {
scaleRatio = maxRenderSize.height / assetTrack.naturalSize.height
}
// Set correct transform.
var transform = CGAffineTransform(scaleX: scaleRatio, y: scaleRatio)
transform = assetTrack.preferredTransform.concatenating(transform)
layerInstruction.setTransform(transform, at: .zero)
// Create Composition Instruction and pass Layer Instruction to it.
let videoCompositionInstruction = AVMutableVideoCompositionInstruction()
videoCompositionInstruction.timeRange = CMTimeRangeMake(start: time, duration: duration)
videoCompositionInstruction.layerInstructions = [layerInstruction]
return (videoCompositionInstruction, assetInfo.isPortrait)
}
static func orientationFromTransform(_ transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
var assetOrientation = UIImage.Orientation.up
var isPortrait = false
switch [transform.a, transform.b, transform.c, transform.d] {
case [0.0, 1.0, -1.0, 0.0]:
assetOrientation = .right
isPortrait = true
case [0.0, -1.0, 1.0, 0.0]:
assetOrientation = .left
isPortrait = true
case [1.0, 0.0, 0.0, 1.0]:
assetOrientation = .up
case [-1.0, 0.0, 0.0, -1.0]:
assetOrientation = .down
default:
break
}
return (assetOrientation, isPortrait)
}
}
However, when I try to call the mergeVideo function, I get the error of trying to unwrap an optional value found nil. For instance, in my log I print the values in the array of URLS. I can have an array that looks like this [file:///private/var/mobile/Containers/Data/Application/FD6FFB6E-36A8-49A9-8892-15BEAC0BA817/tmp/846F105D-A2F7-4F0A-A512-643B0407B962.mp4] but get the error The file “846F105D-A2F7-4F0A-A512-643B0407B962.mp4” couldn’t be opened because there is no such file. Any suggestions on how to properly access this recently stored files or any suggestions?

The File "xxx" couldn’t be opened because there is no such file from directory

In my video recording app, I record a video and save it to the photo library. The ultimate goal is to take the recently taken videos and merge them with this merge function.
extension AVMutableComposition {
func mergeVideo(_ urls: [URL], completion: #escaping (_ url: URL?, _ error: Error?) -> Void) {
guard let documentDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else {
completion(nil, nil)
return
}
let dateFormatter = DateFormatter()
dateFormatter.dateStyle = .long
dateFormatter.timeStyle = .short
let date = dateFormatter.string(from: Date())
let outputURL = documentDirectory.appendingPathComponent("mergedVideo_\(date).mp4")
// If there is only one video, we dont to touch it to save export time.
if let url = urls.first, urls.count == 1 {
do {
try FileManager().copyItem(at: url, to: outputURL)
completion(outputURL, nil)
} catch let error {
completion(nil, error)
}
return
}
let maxRenderSize = CGSize(width: 1280.0, height: 720.0)
var currentTime = CMTime.zero
var renderSize = CGSize.zero
// Create empty Layer Instructions, that we will be passing to Video Composition and finally to Exporter.
var instructions = [AVMutableVideoCompositionInstruction]()
urls.enumerated().forEach { index, url in
let asset = AVAsset(url: url)
print(asset)
let assetTrack = asset.tracks.first!
// Create instruction for a video and append it to array.
let instruction = AVMutableComposition.instruction(assetTrack, asset: asset, time: currentTime, duration: assetTrack.timeRange.duration, maxRenderSize: maxRenderSize)
instructions.append(instruction.videoCompositionInstruction)
// Set render size (orientation) according first videro.
if index == 0 {
renderSize = instruction.isPortrait ? CGSize(width: maxRenderSize.height, height: maxRenderSize.width) : CGSize(width: maxRenderSize.width, height: maxRenderSize.height)
}
do {
let timeRange = CMTimeRangeMake(start: .zero, duration: assetTrack.timeRange.duration)
// Insert video to Mutable Composition at right time.
try insertTimeRange(timeRange, of: asset, at: currentTime)
currentTime = CMTimeAdd(currentTime, assetTrack.timeRange.duration)
} catch let error {
completion(nil, error)
}
}
// Create Video Composition and pass Layer Instructions to it.
let videoComposition = AVMutableVideoComposition()
videoComposition.instructions = instructions
// Do not forget to set frame duration and render size. It will crash if you dont.
videoComposition.frameDuration = CMTimeMake(value: 1, timescale: 30)
videoComposition.renderSize = renderSize
guard let exporter = AVAssetExportSession(asset: self, presetName: AVAssetExportPresetHighestQuality) else {
completion(nil, nil)
return
}
exporter.outputURL = outputURL
exporter.outputFileType = .mp4
// Pass Video Composition to the Exporter.
exporter.videoComposition = videoComposition
exporter.shouldOptimizeForNetworkUse = true
exporter.exportAsynchronously {
DispatchQueue.main.async {
completion(exporter.outputURL, nil)
}
}
}
static func instruction(_ assetTrack: AVAssetTrack, asset: AVAsset, time: CMTime, duration: CMTime, maxRenderSize: CGSize)
-> (videoCompositionInstruction: AVMutableVideoCompositionInstruction, isPortrait: Bool) {
let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: assetTrack)
// Find out orientation from preferred transform.
let assetInfo = orientationFromTransform(assetTrack.preferredTransform)
// Calculate scale ratio according orientation.
var scaleRatio = maxRenderSize.width / assetTrack.naturalSize.width
if assetInfo.isPortrait {
scaleRatio = maxRenderSize.height / assetTrack.naturalSize.height
}
// Set correct transform.
var transform = CGAffineTransform(scaleX: scaleRatio, y: scaleRatio)
transform = assetTrack.preferredTransform.concatenating(transform)
layerInstruction.setTransform(transform, at: .zero)
// Create Composition Instruction and pass Layer Instruction to it.
let videoCompositionInstruction = AVMutableVideoCompositionInstruction()
videoCompositionInstruction.timeRange = CMTimeRangeMake(start: time, duration: duration)
videoCompositionInstruction.layerInstructions = [layerInstruction]
return (videoCompositionInstruction, assetInfo.isPortrait)
}
static func orientationFromTransform(_ transform: CGAffineTransform) -> (orientation: UIImage.Orientation, isPortrait: Bool) {
var assetOrientation = UIImage.Orientation.up
var isPortrait = false
switch [transform.a, transform.b, transform.c, transform.d] {
case [0.0, 1.0, -1.0, 0.0]:
assetOrientation = .right
isPortrait = true
case [0.0, -1.0, 1.0, 0.0]:
assetOrientation = .left
isPortrait = true
case [1.0, 0.0, 0.0, 1.0]:
assetOrientation = .up
case [-1.0, 0.0, 0.0, -1.0]:
assetOrientation = .down
default:
break
}
return (assetOrientation, isPortrait)
}
}
After calling this function, if the 'urls' array has only 1 item then I get an error saying "The File (insert filename) couldn't be opened because there is no such file." Otherwise, if the array has more than 1 item than the app crashes due to a force unwrapping optional value found nil. This is how I'm formatting the url and saving them to the app directory
func tempURL() -> URL? {
let directory = NSTemporaryDirectory() as NSString
if directory != "" {
let path = directory.appendingPathComponent(NSUUID().uuidString + ".mp4")
return URL(fileURLWithPath: path)
}
return nil
}
Any ideas on what's the issue or how to fix this?
assetTrack = asset.tracks.first!
.first! is a force-unwrapped optional so try doing something like
assetTrack = asset.tracks.first ?? asset.tracks.someValue

ARKit – How to export OBJ from iPhone/iPad with LiDAR?

How can I export the ARMeshGeometry generated by the new SceneReconstruction API on the latest iPad Pro to an .obj file?
Here's SceneReconstruction documentation.
Starting with Apple's Visualising Scene Scemantics sample app, you can retrieve the ARMeshGeometry object from the first anchor in the frame.
The easiest approach to exporting the data is to first convert it to an MDLMesh:
extension ARMeshGeometry {
func toMDLMesh(device: MTLDevice) -> MDLMesh {
let allocator = MTKMeshBufferAllocator(device: device);
let data = Data.init(bytes: vertices.buffer.contents(), count: vertices.stride * vertices.count);
let vertexBuffer = allocator.newBuffer(with: data, type: .vertex);
let indexData = Data.init(bytes: faces.buffer.contents(), count: faces.bytesPerIndex * faces.count * faces.indexCountPerPrimitive);
let indexBuffer = allocator.newBuffer(with: indexData, type: .index);
let submesh = MDLSubmesh(indexBuffer: indexBuffer,
indexCount: faces.count * faces.indexCountPerPrimitive,
indexType: .uInt32,
geometryType: .triangles,
material: nil);
let vertexDescriptor = MDLVertexDescriptor();
vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition,
format: .float3,
offset: 0,
bufferIndex: 0);
vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: vertices.stride);
return MDLMesh(vertexBuffer: vertexBuffer,
vertexCount: vertices.count,
descriptor: vertexDescriptor,
submeshes: [submesh]);
}
}
Once you have the MDLMesh, exporting to an OBJ file is a breeze:
#IBAction func exportMesh(_ button: UIButton) {
let meshAnchors = arView.session.currentFrame?.anchors.compactMap({ $0 as? ARMeshAnchor });
DispatchQueue.global().async {
let directory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0];
let filename = directory.appendingPathComponent("MyFirstMesh.obj");
guard let device = MTLCreateSystemDefaultDevice() else {
print("metal device could not be created");
return;
};
let asset = MDLAsset();
for anchor in meshAnchors! {
let mdlMesh = anchor.geometry.toMDLMesh(device: device);
asset.add(mdlMesh);
}
do {
try asset.export(to: filename);
} catch {
print("failed to write to file");
}
}
}
The answer of the #swiftcoder works great. But in the case of several anchors you need to convert the vertices coordinates to the world coordinate system based on the anchor transform. In the opposite case all meshes will be placed at zero position and you will have a mess.
The updated code looks like this:
extension ARMeshGeometry {
func toMDLMesh(device: MTLDevice, transform: simd_float4x4) -> MDLMesh {
let allocator = MTKMeshBufferAllocator(device: device)
let data = Data.init(bytes: transformedVertexBuffer(transform), count: vertices.stride * vertices.count)
let vertexBuffer = allocator.newBuffer(with: data, type: .vertex)
let indexData = Data.init(bytes: faces.buffer.contents(), count: faces.bytesPerIndex * faces.count * faces.indexCountPerPrimitive)
let indexBuffer = allocator.newBuffer(with: indexData, type: .index)
let submesh = MDLSubmesh(indexBuffer: indexBuffer,
indexCount: faces.count * faces.indexCountPerPrimitive,
indexType: .uInt32,
geometryType: .triangles,
material: nil)
let vertexDescriptor = MDLVertexDescriptor()
vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition,
format: .float3,
offset: 0,
bufferIndex: 0)
vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: vertices.stride)
return MDLMesh(vertexBuffer: vertexBuffer,
vertexCount: vertices.count,
descriptor: vertexDescriptor,
submeshes: [submesh])
}
func transformedVertexBuffer(_ transform: simd_float4x4) -> [Float] {
var result = [Float]()
for index in 0..<vertices.count {
let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
var vertextTransform = matrix_identity_float4x4
vertextTransform.columns.3 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1)
let position = (transform * vertextTransform).position
result.append(position.x)
result.append(position.y)
result.append(position.z)
}
return result
}
}
extension simd_float4x4 {
var position: SIMD3<Float> {
return SIMD3<Float>(columns.3.x, columns.3.y, columns.3.z)
}
}
extension Array where Element == ARMeshAnchor {
func save(to fileURL: URL, device: MTLDevice) throws {
let asset = MDLAsset()
self.forEach {
let mesh = $0.geometry.toMDLMesh(device: device, transform: $0.transform)
asset.add(mesh)
}
try asset.export(to: fileURL)
}
}
I am not an expert in ModelIO and maybe there is more simple way to transform vertex buffer :) But this code works for me.
Exporting LiDAR-reconstructed geometry
This code allows you save LiDAR's geometry as USD and send it to Mac computer via AirDrop. You can export not only .usd but also .usda, .usdc, .obj, .stl, .abc, and .ply file formats.
Additionally you can use SceneKit's write(to:options:delegate:progressHandler:) method to save a .usdz version of file.
import RealityKit
import ARKit
import MetalKit
import ModelIO
#IBOutlet var arView: ARView!
var saveButton: UIButton!
let rect = CGRect(x: 50, y: 50, width: 100, height: 50)
override func viewDidLoad() {
super.viewDidLoad()
let tui = UIControl.Event.touchUpInside
saveButton = UIButton(frame: rect)
saveButton.setTitle("Save", for: [])
saveButton.addTarget(self, action: #selector(saveButtonTapped), for: tui)
self.view.addSubview(saveButton)
}
#objc func saveButtonTapped(sender: UIButton) {
print("Saving is executing...")
guard let frame = arView.session.currentFrame
else { fatalError("Can't get ARFrame") }
guard let device = MTLCreateSystemDefaultDevice()
else { fatalError("Can't create MTLDevice") }
let allocator = MTKMeshBufferAllocator(device: device)
let asset = MDLAsset(bufferAllocator: allocator)
let meshAnchors = frame.anchors.compactMap { $0 as? ARMeshAnchor }
for ma in meshAnchors {
let geometry = ma.geometry
let vertices = geometry.vertices
let faces = geometry.faces
let vertexPointer = vertices.buffer.contents()
let facePointer = faces.buffer.contents()
for vtxIndex in 0 ..< vertices.count {
let vertex = geometry.vertex(at: UInt32(vtxIndex))
var vertexLocalTransform = matrix_identity_float4x4
vertexLocalTransform.columns.3 = SIMD4<Float>(x: vertex.0,
y: vertex.1,
z: vertex.2,
w: 1.0)
let vertexWorldTransform = (ma.transform * vertexLocalTransform).position
let vertexOffset = vertices.offset + vertices.stride * vtxIndex
let componentStride = vertices.stride / 3
vertexPointer.storeBytes(of: vertexWorldTransform.x,
toByteOffset: vertexOffset,
as: Float.self)
vertexPointer.storeBytes(of: vertexWorldTransform.y,
toByteOffset: vertexOffset + componentStride,
as: Float.self)
vertexPointer.storeBytes(of: vertexWorldTransform.z,
toByteOffset: vertexOffset + (2 * componentStride),
as: Float.self)
}
let byteCountVertices = vertices.count * vertices.stride
let byteCountFaces = faces.count * faces.indexCountPerPrimitive * faces.bytesPerIndex
let vertexBuffer = allocator.newBuffer(with: Data(bytesNoCopy: vertexPointer,
count: byteCountVertices,
deallocator: .none), type: .vertex)
let indexBuffer = allocator.newBuffer(with: Data(bytesNoCopy: facePointer,
count: byteCountFaces,
deallocator: .none), type: .index)
let indexCount = faces.count * faces.indexCountPerPrimitive
let material = MDLMaterial(name: "material",
scatteringFunction: MDLPhysicallyPlausibleScatteringFunction())
let submesh = MDLSubmesh(indexBuffer: indexBuffer,
indexCount: indexCount,
indexType: .uInt32,
geometryType: .triangles,
material: material)
let vertexFormat = MTKModelIOVertexFormatFromMetal(vertices.format)
let vertexDescriptor = MDLVertexDescriptor()
vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition,
format: vertexFormat,
offset: 0,
bufferIndex: 0)
vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: ma.geometry.vertices.stride)
let mesh = MDLMesh(vertexBuffer: vertexBuffer,
vertexCount: ma.geometry.vertices.count,
descriptor: vertexDescriptor,
submeshes: [submesh])
asset.add(mesh)
}
let filePath = FileManager.default.urls(for: .documentDirectory,
in: .userDomainMask).first!
let usd: URL = filePath.appendingPathComponent("model.usd")
if MDLAsset.canExportFileExtension("usd") {
do {
try asset.export(to: usd)
let controller = UIActivityViewController(activityItems: [usd],
applicationActivities: nil)
controller.popoverPresentationController?.sourceView = sender
self.present(controller, animated: true, completion: nil)
} catch let error {
fatalError(error.localizedDescription)
}
} else {
fatalError("Can't export USD")
}
}
Tap Save button, and in Activity View Controller choose More and send ready-to-use model to Mac's Downloads folder via AirDrop.
P.S.
Here you can find an extra info on capturing real-world texture.
I'm using Lidar and beginner in scan 3d, how to export to files with swift code:
GLTF (Share in AR for Android devices)
GLB
STL (Un-textured file used in 3d printing)
Point Cloud (PCD PLY PTS XYZ LAS e57)
All Data (Includes captured images)
DAE (Compatible with Sketchfab)
FBX
thanks you

How can I record AVDepthData video and save in the gallery?

I am developing an application to record RGB-D sequences with the iPhone by using the DualRearCamera or the TrueDepthCamera. I can capture and visualize the RGB frame and depth frames and I developed a version where I can compress this data and save in the internal files of the iPhone. Nevertheless, my idea is to save both sequences (RGB and depth map sequences) in the gallery, but I am having problems to use AVAssetWritter and create a depth map video.
I am using the iPhone X, Xcode 10.2.1 and swift 5
import UIKit
import AVFoundation
import AssetsLibrary
var noMoreSpace = false
class ViewController: UIViewController{
#IBOutlet weak var previewView: UIImageView!
#IBOutlet weak var timeLabel: UILabel!
#IBOutlet weak var previewModeControl: UISegmentedControl!
let session = AVCaptureSession()
let dataOutputQueue = DispatchQueue(label: "video data queue")
let videoOutput = AVCaptureVideoDataOutput()
let movieOutput = AVCaptureMovieFileOutput()
let depthOutput = AVCaptureDepthDataOutput()
let depthCapture = DepthCapture()
var previewLayer = AVCaptureVideoPreviewLayer()
var inputDevice: AVCaptureDeviceInput!
let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInDualCamera, .builtInTrueDepthCamera], mediaType: .video, position: .unspecified)
var Timestamp: String {
let currentDate = NSDate()
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "ddMM_HHmmss"
return "\(dateFormatter.string(from: currentDate as Date))"
}
var isRecording = false
var time = 0
var timer = Timer()
enum PreviewMode: Int {
case original
case depth
}
var previewMode = PreviewMode.original
var depthMap: CIImage?
var scale: CGFloat = 0.0
//let sessionQueue = DispatchQueue(label: "session queue")
override func viewDidLoad() {
super.viewDidLoad()
timeLabel.isHidden = true //TODO: Disable the rest of the UI
previewMode = PreviewMode(rawValue: previewModeControl.selectedSegmentIndex) ?? .original
configureCaptureSession()
session.startRunning()
}
func configureCaptureSession() {
session.beginConfiguration()
let camera = AVCaptureDevice.default(.builtInTrueDepthCamera, for: .video, position: .unspecified)!
do {
let cameraInput = try AVCaptureDeviceInput(device: camera)
if session.canAddInput(cameraInput){
session.sessionPreset = .vga640x480
session.addInput(cameraInput)
self.inputDevice = cameraInput
}
if session.canAddOutput(videoOutput){
videoOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
session.addOutput(videoOutput)
let videoConnection = videoOutput.connection(with: .video)
videoConnection?.videoOrientation = .portrait
//previewLayer = AVCaptureVideoPreviewLayer(session: session)
//previewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill
//previewLayer.connection?.videoOrientation = AVCaptureVideoOrientation.portrait
//previewView.layer.addSublayer(previewLayer)
//previewLayer.position = CGPoint(x: self.previewView.frame.width / 2, y: self.previewView.frame.height / 2)
//previewLayer.bounds = previewView.frame
}
//Add Depth output to the session
if session.canAddOutput(depthOutput){
session.addOutput(depthOutput)
depthOutput.setDelegate(self, callbackQueue: dataOutputQueue)
depthOutput.isFilteringEnabled = true
let depthConnection = depthOutput.connection(with: .depthData)
depthConnection?.videoOrientation = .portrait
}
/*if session.canAddOutput(movieOutput){
session.addOutput(movieOutput)
}*/
} catch {
print("Error")
}
let outputRect = CGRect(x: 0, y: 0, width: 1, height: 1)
let videoRect = videoOutput.outputRectConverted(fromMetadataOutputRect: outputRect)
let depthRect = depthOutput.outputRectConverted(fromMetadataOutputRect: outputRect)
// Calculate the scaling factor between videoRect and depthRect
scale = max(videoRect.width, videoRect.height) / max(depthRect.width, depthRect.height)
// Change the AVCaptureDevice configuration, so you need to lock it
do{
try camera.lockForConfiguration()
// Set the AVCaptureDevice‘s minimum frame duration (which is the inverse of the maximum frame rate) to be equal to the supported frame rate of the depth data
if let frameDuration = camera.activeDepthDataFormat?.videoSupportedFrameRateRanges.first?.minFrameDuration{
camera.activeVideoMinFrameDuration = frameDuration
}
// Unlock the configuration you locked
camera.unlockForConfiguration()
}catch{
fatalError(error.localizedDescription)
}
session.commitConfiguration()
}
#IBAction func startStopRecording(_ sender: Any) {
if isRecording{
stopRecording()
} else {
startRecording()
}
}
func startRecording(){
timeLabel.isHidden = false
timer = Timer.scheduledTimer(timeInterval: 1, target: self, selector: #selector(ViewController.timerAction), userInfo: nil, repeats: true)
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let flagTime = Timestamp
let auxStr = flagTime+"_output.mp4"
let fileUrl = paths[0].appendingPathComponent(auxStr)
depthCapture.prepareForRecording(timeFlag: flagTime)
movieOutput.startRecording(to: fileUrl, recordingDelegate: self)
print(fileUrl.absoluteString)
print("Recording started")
self.isRecording = true
}
func stopRecording(){
timeLabel.isHidden = true
timer.invalidate()
time = 0
timeLabel.text = "0"
movieOutput.stopRecording()
print("Stopped recording!")
self.isRecording = false
do {
try depthCapture.finishRecording(success: { (url: URL) -> Void in
print(url.absoluteString)
})
} catch {
print("Error while finishing depth capture.")
}
}
#objc func timerAction() {
time += 1
timeLabel.text = String(time)
}
#IBAction func previeModeChanged(_ sender: UISegmentedControl) {
previewMode = PreviewMode(rawValue: previewModeControl.selectedSegmentIndex) ?? .original
}
#IBAction func switchCamera(_ sender: Any) {
let currentDevice = self.inputDevice.device
let currentPosition = currentDevice.position
let preferredPosition: AVCaptureDevice.Position
let preferredDeviceType: AVCaptureDevice.DeviceType
let devices = self.videoDeviceDiscoverySession.devices
var newVideoDevice: AVCaptureDevice? = nil
switch currentPosition {
case .unspecified, .front:
preferredPosition = .back
preferredDeviceType = .builtInDualCamera
case .back:
preferredPosition = .front
preferredDeviceType = .builtInTrueDepthCamera
#unknown default:
preferredPosition = .back
preferredDeviceType = .builtInDualCamera
}
// First, seek a device with both the preferred position and device type. Otherwise, seek a device with only the preferred position. ENTENDER MEJOR LQS CONDICIONES
if let device = devices.first(where: { $0.position == preferredPosition && $0.deviceType == preferredDeviceType }) {
newVideoDevice = device
} else if let device = devices.first(where: { $0.position == preferredPosition }) {
newVideoDevice = device
}
if let videoDevice = newVideoDevice {
do {
let cameraInput = try AVCaptureDeviceInput(device: videoDevice)
self.session.beginConfiguration()
self.session.removeInput(self.inputDevice)
if self.session.canAddInput(cameraInput) {
session.sessionPreset = .vga640x480
self.session.addInput(cameraInput)
self.inputDevice = cameraInput
}else {
self.session.addInput(self.inputDevice)
}
self.session.commitConfiguration()
} catch{
print("Error occurred while creating video device input: \(error)")
}
}
}
}
extension ViewController: AVCaptureVideoDataOutputSampleBufferDelegate{
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
let image = CIImage(cvPixelBuffer: pixelBuffer!)
let previewImage: CIImage
switch previewMode {
case .original:
previewImage = image
case .depth:
previewImage = depthMap ?? image
}
let displayImage = UIImage(ciImage: previewImage)
DispatchQueue.main.async {[weak self] in self?.previewView.image = displayImage}
}
}
extension ViewController: AVCaptureDepthDataOutputDelegate{
func depthDataOutput(_ output: AVCaptureDepthDataOutput, didOutput depthData: AVDepthData, timestamp: CMTime, connection: AVCaptureConnection) {
var convertedDepth: AVDepthData
// Ensure the depth data is the format you need: 32 bit FP disparity.???
if depthData.depthDataType != kCVPixelFormatType_DepthFloat16{
convertedDepth = depthData.converting(toDepthDataType: kCVPixelFormatType_DepthFloat32)
}else{
convertedDepth = depthData
}
// You save the depth data map from the AVDepthData object as a CVPixelBuffer
let pixelBuffer = convertedDepth.depthDataMap
//Using an extension, you then clamp the pixels in the pixel buffer to keep them between 0.0 and 1.0.
pixelBuffer.clamp()
// Convert the pixel buffer into a CIImage
let depthMap = CIImage(cvPixelBuffer: pixelBuffer)
// You store depthMap in a class variable for later use
DispatchQueue.main.async {
[weak self] in self?.depthMap = depthMap
}
}
}

Converting AVAudioPCMBuffer to another AVAudioPCMBuffer

I am trying to convert a determined AVAudioPCMBuffer (44.1khz, 1ch, float32, not interleaved) to another AVAudioPCMBuffer (16khz, 1ch, int16, not interleaved) using AVAudioConverter and write it using AVAudioFile.
My code uses the library AudioKit together with the tap AKLazyTap to get a buffer each determined time, based on this source:
https://github.com/AudioKit/AudioKit/tree/master/AudioKit/Common/Taps/Lazy%20Tap
Here is my implementation:
lazy var downAudioFormat: AVAudioFormat = {
let avAudioChannelLayout = AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_Mono)!
return AVAudioFormat(
commonFormat: .pcmFormatInt16,
sampleRate: 16000,
interleaved: false,
channelLayout: avAudioChannelLayout)
}()
//...
AKSettings.sampleRate = 44100
AKSettings.numberOfChannels = AVAudioChannelCount(1)
AKSettings.ioBufferDuration = 0.002
AKSettings.defaultToSpeaker = true
//...
let mic = AKMicrophone()
let originalAudioFormat: AVAudioFormat = mic.avAudioNode.outputFormat(forBus: 0) //41.100, 1ch, float32...
let inputFrameCapacity = AVAudioFrameCount(1024)
//I don't think this is correct, the audio is getting chopped...
//How to calculate it correctly?
let outputFrameCapacity = AVAudioFrameCount(512)
guard let inputBuffer = AVAudioPCMBuffer(
pcmFormat: originalAudioFormat,
frameCapacity: inputFrameCapacity) else {
fatalError()
}
// Your timer should fire equal to or faster than your buffer duration
bufferTimer = Timer.scheduledTimer(
withTimeInterval: AKSettings.ioBufferDuration/2,
repeats: true) { [weak self] _ in
guard let unwrappedSelf = self else {
return
}
unwrappedSelf.lazyTap?.fillNextBuffer(inputBuffer, timeStamp: nil)
// This is important, since we're polling for samples, sometimes
//it's empty, and sometimes it will be double what it was the last call.
if inputBuffer.frameLength == 0 {
return
}
//This converter is only create once, as the AVAudioFile. Ignore this code I call a function instead.
let converter = AVAudioConverter(from: originalAudioFormat, to: downAudioFormat)
converter.sampleRateConverterAlgorithm = AVSampleRateConverterAlgorithm_Normal
converter.sampleRateConverterQuality = .min
converter.bitRateStrategy = AVAudioBitRateStrategy_Constant
guard let outputBuffer = AVAudioPCMBuffer(
pcmFormat: converter.outputFormat,
frameCapacity: outputFrameCapacity) else {
print("Failed to create new buffer")
return
}
let inputBlock: AVAudioConverterInputBlock = { inNumPackets, outStatus in
outStatus.pointee = AVAudioConverterInputStatus.haveData
return inputBuffer
}
var error: NSError?
let status: AVAudioConverterOutputStatus = converter.convert(
to: outputBuffer,
error: &error,
withInputFrom: inputBlock)
switch status {
case .error:
if let unwrappedError: NSError = error {
print(unwrappedError)
}
return
default: break
}
//Only created once, instead of this code my code uses a function to verify if the AVAudioFile has been created, ignore it.
outputAVAudioFile = try AVAudioFile(
forWriting: unwrappedCacheFilePath,
settings: format.settings,
commonFormat: format.commonFormat,
interleaved: false)
do {
try outputAVAudioFile?.write(from: avAudioPCMBuffer)
} catch {
print(error)
}
}
(Please note that AVAudioConverter and AVAudioFile are being reused, the initialization there doesn't represent the real implementation on my code, just to simplify and make it more simple to understand.)
With frameCapacity on the outputBuffer: AVAudioPCMBuffer set to 512, the audio get chopped. Is there any way to discovery the correct frameCapacity for this buffer?
Written using Swift 4 and AudioKit 4.1.
Many thanks!
I managed to solve this problem installing a Tap on the inputNode like this:
lazy var downAudioFormat: AVAudioFormat = {
let avAudioChannelLayout = AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_Mono)!
return AVAudioFormat(
commonFormat: .pcmFormatInt16,
sampleRate: SAMPLE_RATE,
interleaved: true,
channelLayout: avAudioChannelLayout)
}()
private func addBufferListener(_ avAudioNode: AVAudioNode) {
let originalAudioFormat: AVAudioFormat = avAudioNode.inputFormat(forBus: 0)
let downSampleRate: Double = downAudioFormat.sampleRate
let ratio: Float = Float(originalAudioFormat.sampleRate)/Float(downSampleRate)
let converter: AVAudioConverter = buildConverter(originalAudioFormat)
avAudioNode.installTap(
onBus: 0,
bufferSize: AVAudioFrameCount(downSampleRate * 2),
format: originalAudioFormat,
block: { (buffer: AVAudioPCMBuffer!, _ : AVAudioTime!) -> Void in
let capacity = UInt32(Float(buffer.frameCapacity)/ratio)
guard let outputBuffer = AVAudioPCMBuffer(
pcmFormat: self.downAudioFormat,
frameCapacity: capacity) else {
print("Failed to create new buffer")
return
}
let inputBlock: AVAudioConverterInputBlock = { inNumPackets, outStatus in
outStatus.pointee = AVAudioConverterInputStatus.haveData
return buffer
}
var error: NSError?
let status: AVAudioConverterOutputStatus = converter.convert(
to: outputBuffer,
error: &error,
withInputFrom: inputBlock)
switch status {
case .error:
if let unwrappedError: NSError = error {
print("Error \(unwrappedError)"))
}
return
default: break
}
self.delegate?.flushAudioBuffer(outputBuffer)
})
}