I am developing an Audio effects application on OSX using Swift, and I'm interested in integrating a pitch-shift effect.
I would like in real-time, to change the tone up or down an octave. Currently I am only getting a dry signal.
I am not sure if this is possible at all and would like to know if this is even possible or any help or suggestions anyone may have.
Current code relevant to the problem is as follows:
import Cocoa
import AVFoundation
class ViewController: NSViewController {
var engine = AVAudioEngine()
var timePitch = AVAudioUnitTimePitch()
override func viewDidLoad() {
timePitch.pitch = 1200
// Setup engine and node instances
var mixer = engine.mainMixerNode
var input = engine.inputNode
var output = engine.outputNode
var format = input.inputFormatForBus(0)
var error:NSError?
engine.attachNode(timePitch)
engine.connect(input, to: timePitch, format: format)
engine.connect(timePitch, to: output, format: format)
engine.startAndReturnError(&error)
super.viewDidLoad()
}
override var representedObject: AnyObject? {
didSet {
// Update the view, if already loaded.
}
}
}
timePitch.pitch = -500 //Rude man voice
timePitch.rate = 1.5 //In 1.5 times faster
Check this tutorial. And direct link to example from tutorial for more info.
Example for Swift 2.0:
import UIKit
import AVFoundation
class ViewController: UIViewController {
var engine: AVAudioEngine!
var player: AVAudioPlayerNode!
var file = AVAudioFile()
override func viewDidLoad() {
super.viewDidLoad()
engine = AVAudioEngine()
player = AVAudioPlayerNode()
player.volume = 1.0
let path = NSBundle.mainBundle().pathForResource("in", ofType: "caf")!
let url = NSURL.fileURLWithPath(path)
let file = try? AVAudioFile(forReading: url)
let buffer = AVAudioPCMBuffer(PCMFormat: file!.processingFormat, frameCapacity: AVAudioFrameCount(file!.length))
do {
try file!.readIntoBuffer(buffer)
} catch _ {
}
let pitch = AVAudioUnitTimePitch()
//
pitch.pitch = -500 //Distortion
pitch.rate = 1.5 //Voice speed
//
engine.attachNode(player)
engine.attachNode(pitch)
engine.connect(player, to: pitch, format: buffer.format)
engine.connect(pitch, to: engine.mainMixerNode, format: buffer.format)
player.scheduleBuffer(buffer, atTime: nil, options: AVAudioPlayerNodeBufferOptions.Loops, completionHandler: nil)
engine.prepare()
do {
try engine.start()
} catch _ {
}
player.play()
}
}
timePitch.pitch = 1000 //Filtered Voice
timePitch.rate = 1 //Normal rate
Related
I loaded into SceneKit a .usdz file which has an animation attached to it. I want to stop this animation to play.. but I can't find the right way.
I load the .usdz asset file with the following method:
func loadIdle() {
let urlfile = Bundle.main.path(forResource: "armi_idle",
ofType: "usdz",
inDirectory: "Asset.scnassets")!
let scene = try! SCNScene(url: URL(string: urlfile)!)
guard let findNode = scene.rootNode.childNode(withName: "armi_idle",
recursively: true)
else {
print("err finde idle")
return
}
// try to pause, but not work
findNode.isPaused = true
// add to main scene
self.scene.rootNode.addChildNode(findNode)
}
A picture of the asset:
Declare the following properties:
import SceneKit
#IBOutlet var sceneView: SCNView!
var notWalking: Bool = true
var animations = [String: CAAnimation]()
var node = SCNNode()
override func viewDidLoad() {
super.viewDidLoad()
sceneView.scene = SCNScene()
self.animation()
}
Then try the following logic (you need Idle and Walking files) :
fileprivate func loadAnimation(withKey: String, scene: String, id: String) {
let url = Bundle.main.url(forResource: scene, withExtension: "dae")!
let source = SCNSceneSource(url: url, options: nil)
guard let character = source?.entryWithIdentifier(id,
withClass: CAAnimation.self) else { return }
character.fadeInDuration = 0.5
character.fadeOutDuration = 0.5
self.animations[withKey] = character
}
then:
fileprivate func animation() {
let standStill = SCNScene(named: "art.scnassets/Idle")!
for childNode in standStill.rootNode.childNodes {
self.node.addChildNode(childNode)
}
sceneView.scene.rootNode.addChildNode(self.node)
self.loadAnimation(withKey: "walking", scene: "art.scnassets/Walking",
id: "Walking-1")
}
and then:
func playWalking(key: String) {
sceneView.scene.rootNode.addAnimation(animations[key]!, forKey: key)
}
func stopWalking(key: String) {
sceneView.scene.rootNode.removeAnimation(forKey: key,
blendOutDuration: 0.75)
}
and at last:
#IBAction func pressed(_ sender: UIButton) {
notWalking ? playWalking(key: "walking") : stopWalking(key: "walking")
notWalking.toggle()
}
Been using the PlayerItem.timedMetadata (pasted below) for quite a while and has worked very well. However, it seems that Apple has marked this method as 'Deprecated in iOS 13' and might (or will) be removed.
Xcode does inform me that I have to use another method called "AVPlayerItemMetadataOutput" to which I've never tried. So, looking on the internet (google) I found nothing at all apart from the apple docs (https://developer.apple.com/documentation/avfoundation/avplayeritemmetadataoutput).
override open func observeValue(forKeyPath: String?, of: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
guard forKeyPath == "timedMetadata" else { return }
guard let meta = PlayerItem.timedMetadata else { return }
for metadata in meta {
if let songName = metadata.value(forKey: "value") as? String {
Variables.MediaInfo = (songName)
self.MediaBox.text = ("Now Playing \n \(songName)")
setupNowPlaying()
}
}
}
UPDATE!
I have banged my head in Apple Docs for a day or so then it hits me like a brick in the face.
class ViewController: UIViewController,AVPlayerItemMetadataOutputPushDelegate {
weak var myDelegate: AVPlayerItemMetadataOutputPushDelegate?
var playerItem: AVPlayerItem?
var player = AVPlayer()
override func viewDidLoad() {
super.viewDidLoad()
print("lets go!")
let url = URL(string: "<URLREMOVED>")!
let asset = AVAsset(url: url)
let playerItem = AVPlayerItem(asset: asset)
let metadataOutput = AVPlayerItemMetadataOutput(identifiers: nil)
metadataOutput.setDelegate(self, queue: DispatchQueue.main)
playerItem.add(metadataOutput)
player = AVPlayer(playerItem: playerItem)
player.play()
print("END")
print(playerItem.automaticallyLoadedAssetKeys.description)
}
func metadataOutput(_ output: AVPlayerItemMetadataOutput, didOutputTimedMetadataGroups groups: [AVTimedMetadataGroup], from track: AVPlayerItemTrack?) {
}
}
SO, this reads the stream for timed Metadata changes and then displays this output:
AVMutableMetadataItem: 0x600002064020, identifier=icy/StreamTitle, keySpace=icy, key class = __NSCFConstantString, key=StreamTitle, commonKey=title, extendedLanguageTag=(null), dataType=(null), time={102328704/44100 = 2320.379}, duration={1/44100 = 0.000}, startDate=(null), extras={\n}, value class=__NSCFString, value=Tina Turner - Way Of The World>
Neat right? SO now all I have to to is filter the icy/StreamTitle into a string and I'm golden! :D
func metadataOutput(_ output: AVPlayerItemMetadataOutput, didOutputTimedMetadataGroups groups: [AVTimedMetadataGroup], from track: AVPlayerItemTrack?) {
if let item = groups.first?.items.first // make this an AVMetadata item
{
item.value(forKeyPath: "value") // looking for that key bro
let Song = (item.value(forKeyPath: "value")!)
MetaData = "Now Playing: \n \(Song)" // print the results
} else {
MetaData = "MetaData Error" // No Metadata or Could not read
}
I am developing an application to record RGB-D sequences with the iPhone by using the DualRearCamera or the TrueDepthCamera. I can capture and visualize the RGB frame and depth frames and I developed a version where I can compress this data and save in the internal files of the iPhone. Nevertheless, my idea is to save both sequences (RGB and depth map sequences) in the gallery, but I am having problems to use AVAssetWritter and create a depth map video.
I am using the iPhone X, Xcode 10.2.1 and swift 5
import UIKit
import AVFoundation
import AssetsLibrary
var noMoreSpace = false
class ViewController: UIViewController{
#IBOutlet weak var previewView: UIImageView!
#IBOutlet weak var timeLabel: UILabel!
#IBOutlet weak var previewModeControl: UISegmentedControl!
let session = AVCaptureSession()
let dataOutputQueue = DispatchQueue(label: "video data queue")
let videoOutput = AVCaptureVideoDataOutput()
let movieOutput = AVCaptureMovieFileOutput()
let depthOutput = AVCaptureDepthDataOutput()
let depthCapture = DepthCapture()
var previewLayer = AVCaptureVideoPreviewLayer()
var inputDevice: AVCaptureDeviceInput!
let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInDualCamera, .builtInTrueDepthCamera], mediaType: .video, position: .unspecified)
var Timestamp: String {
let currentDate = NSDate()
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "ddMM_HHmmss"
return "\(dateFormatter.string(from: currentDate as Date))"
}
var isRecording = false
var time = 0
var timer = Timer()
enum PreviewMode: Int {
case original
case depth
}
var previewMode = PreviewMode.original
var depthMap: CIImage?
var scale: CGFloat = 0.0
//let sessionQueue = DispatchQueue(label: "session queue")
override func viewDidLoad() {
super.viewDidLoad()
timeLabel.isHidden = true //TODO: Disable the rest of the UI
previewMode = PreviewMode(rawValue: previewModeControl.selectedSegmentIndex) ?? .original
configureCaptureSession()
session.startRunning()
}
func configureCaptureSession() {
session.beginConfiguration()
let camera = AVCaptureDevice.default(.builtInTrueDepthCamera, for: .video, position: .unspecified)!
do {
let cameraInput = try AVCaptureDeviceInput(device: camera)
if session.canAddInput(cameraInput){
session.sessionPreset = .vga640x480
session.addInput(cameraInput)
self.inputDevice = cameraInput
}
if session.canAddOutput(videoOutput){
videoOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
session.addOutput(videoOutput)
let videoConnection = videoOutput.connection(with: .video)
videoConnection?.videoOrientation = .portrait
//previewLayer = AVCaptureVideoPreviewLayer(session: session)
//previewLayer.videoGravity = AVLayerVideoGravity.resizeAspectFill
//previewLayer.connection?.videoOrientation = AVCaptureVideoOrientation.portrait
//previewView.layer.addSublayer(previewLayer)
//previewLayer.position = CGPoint(x: self.previewView.frame.width / 2, y: self.previewView.frame.height / 2)
//previewLayer.bounds = previewView.frame
}
//Add Depth output to the session
if session.canAddOutput(depthOutput){
session.addOutput(depthOutput)
depthOutput.setDelegate(self, callbackQueue: dataOutputQueue)
depthOutput.isFilteringEnabled = true
let depthConnection = depthOutput.connection(with: .depthData)
depthConnection?.videoOrientation = .portrait
}
/*if session.canAddOutput(movieOutput){
session.addOutput(movieOutput)
}*/
} catch {
print("Error")
}
let outputRect = CGRect(x: 0, y: 0, width: 1, height: 1)
let videoRect = videoOutput.outputRectConverted(fromMetadataOutputRect: outputRect)
let depthRect = depthOutput.outputRectConverted(fromMetadataOutputRect: outputRect)
// Calculate the scaling factor between videoRect and depthRect
scale = max(videoRect.width, videoRect.height) / max(depthRect.width, depthRect.height)
// Change the AVCaptureDevice configuration, so you need to lock it
do{
try camera.lockForConfiguration()
// Set the AVCaptureDeviceās minimum frame duration (which is the inverse of the maximum frame rate) to be equal to the supported frame rate of the depth data
if let frameDuration = camera.activeDepthDataFormat?.videoSupportedFrameRateRanges.first?.minFrameDuration{
camera.activeVideoMinFrameDuration = frameDuration
}
// Unlock the configuration you locked
camera.unlockForConfiguration()
}catch{
fatalError(error.localizedDescription)
}
session.commitConfiguration()
}
#IBAction func startStopRecording(_ sender: Any) {
if isRecording{
stopRecording()
} else {
startRecording()
}
}
func startRecording(){
timeLabel.isHidden = false
timer = Timer.scheduledTimer(timeInterval: 1, target: self, selector: #selector(ViewController.timerAction), userInfo: nil, repeats: true)
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let flagTime = Timestamp
let auxStr = flagTime+"_output.mp4"
let fileUrl = paths[0].appendingPathComponent(auxStr)
depthCapture.prepareForRecording(timeFlag: flagTime)
movieOutput.startRecording(to: fileUrl, recordingDelegate: self)
print(fileUrl.absoluteString)
print("Recording started")
self.isRecording = true
}
func stopRecording(){
timeLabel.isHidden = true
timer.invalidate()
time = 0
timeLabel.text = "0"
movieOutput.stopRecording()
print("Stopped recording!")
self.isRecording = false
do {
try depthCapture.finishRecording(success: { (url: URL) -> Void in
print(url.absoluteString)
})
} catch {
print("Error while finishing depth capture.")
}
}
#objc func timerAction() {
time += 1
timeLabel.text = String(time)
}
#IBAction func previeModeChanged(_ sender: UISegmentedControl) {
previewMode = PreviewMode(rawValue: previewModeControl.selectedSegmentIndex) ?? .original
}
#IBAction func switchCamera(_ sender: Any) {
let currentDevice = self.inputDevice.device
let currentPosition = currentDevice.position
let preferredPosition: AVCaptureDevice.Position
let preferredDeviceType: AVCaptureDevice.DeviceType
let devices = self.videoDeviceDiscoverySession.devices
var newVideoDevice: AVCaptureDevice? = nil
switch currentPosition {
case .unspecified, .front:
preferredPosition = .back
preferredDeviceType = .builtInDualCamera
case .back:
preferredPosition = .front
preferredDeviceType = .builtInTrueDepthCamera
#unknown default:
preferredPosition = .back
preferredDeviceType = .builtInDualCamera
}
// First, seek a device with both the preferred position and device type. Otherwise, seek a device with only the preferred position. ENTENDER MEJOR LQS CONDICIONES
if let device = devices.first(where: { $0.position == preferredPosition && $0.deviceType == preferredDeviceType }) {
newVideoDevice = device
} else if let device = devices.first(where: { $0.position == preferredPosition }) {
newVideoDevice = device
}
if let videoDevice = newVideoDevice {
do {
let cameraInput = try AVCaptureDeviceInput(device: videoDevice)
self.session.beginConfiguration()
self.session.removeInput(self.inputDevice)
if self.session.canAddInput(cameraInput) {
session.sessionPreset = .vga640x480
self.session.addInput(cameraInput)
self.inputDevice = cameraInput
}else {
self.session.addInput(self.inputDevice)
}
self.session.commitConfiguration()
} catch{
print("Error occurred while creating video device input: \(error)")
}
}
}
}
extension ViewController: AVCaptureVideoDataOutputSampleBufferDelegate{
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
let image = CIImage(cvPixelBuffer: pixelBuffer!)
let previewImage: CIImage
switch previewMode {
case .original:
previewImage = image
case .depth:
previewImage = depthMap ?? image
}
let displayImage = UIImage(ciImage: previewImage)
DispatchQueue.main.async {[weak self] in self?.previewView.image = displayImage}
}
}
extension ViewController: AVCaptureDepthDataOutputDelegate{
func depthDataOutput(_ output: AVCaptureDepthDataOutput, didOutput depthData: AVDepthData, timestamp: CMTime, connection: AVCaptureConnection) {
var convertedDepth: AVDepthData
// Ensure the depth data is the format you need: 32 bit FP disparity.???
if depthData.depthDataType != kCVPixelFormatType_DepthFloat16{
convertedDepth = depthData.converting(toDepthDataType: kCVPixelFormatType_DepthFloat32)
}else{
convertedDepth = depthData
}
// You save the depth data map from the AVDepthData object as a CVPixelBuffer
let pixelBuffer = convertedDepth.depthDataMap
//Using an extension, you then clamp the pixels in the pixel buffer to keep them between 0.0 and 1.0.
pixelBuffer.clamp()
// Convert the pixel buffer into a CIImage
let depthMap = CIImage(cvPixelBuffer: pixelBuffer)
// You store depthMap in a class variable for later use
DispatchQueue.main.async {
[weak self] in self?.depthMap = depthMap
}
}
}
I want to detect object and track that object using vision framework. I am successfully done with detect objects and little bit with tracking also but I don't get so much accuracy with tracking.
I want much more accuracy while converting frames as its frequently lost the accuracy while track the objects.
Please check the below code for detect and track the objects:
import UIKit
import AVFoundation
import Vision
class ViewController: UIViewController {
private lazy var captureSession: AVCaptureSession = {
let session = AVCaptureSession()
session.sessionPreset = AVCaptureSession.Preset.photo
guard let backCamera = AVCaptureDevice.default(for: .video),
let input = try? AVCaptureDeviceInput(device: backCamera) else
{
return session
}
session.addInput(input)
return session
}()
private lazy var cameraLayer: AVCaptureVideoPreviewLayer =
AVCaptureVideoPreviewLayer(session: self.captureSession)
private let handler = VNSequenceRequestHandler()
fileprivate var lastObservation: VNDetectedObjectObservation?
lazy var highlightView: UIView = {
let view = UIView()
view.layer.borderColor = UIColor.red.cgColor
view.layer.borderWidth = 4
view.backgroundColor = .clear
return view
}()
override func viewDidLoad() {
super.viewDidLoad()
view.layer.addSublayer(cameraLayer)
view.addSubview(highlightView)
let output = AVCaptureVideoDataOutput()
output.setSampleBufferDelegate(self, queue: DispatchQueue(label:
"queue"))
captureSession.addOutput(output)
captureSession.startRunning()
let tapGestureRecognizer = UITapGestureRecognizer(target: self,
action: #selector(tapAction))
view.addGestureRecognizer(tapGestureRecognizer)
}
override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
cameraLayer.frame = view.bounds
}
// MARK: - Actions
#objc private func tapAction(recognizer: UITapGestureRecognizer) {
highlightView.frame.size = CGSize(width: 120, height: 120)
highlightView.center = recognizer.location(in: view)
let originalRect = highlightView.frame
var convertedRect =
cameraLayer.metadataOutputRectConverted(fromLayerRect:
originalRect)
convertedRect.origin.y = 1 - convertedRect.origin.y
lastObservation = VNDetectedObjectObservation(boundingBox:
convertedRect)
}
fileprivate func handle(_ request: VNRequest, error: Error?) {
DispatchQueue.main.async {
guard let newObservation = request.results?.first as?
VNDetectedObjectObservation else {
return
}
self.lastObservation = newObservation
var transformedRect = newObservation.boundingBox
transformedRect.origin.y = 1 - transformedRect.origin.y
let convertedRect =
self.cameraLayer.layerRectConverted(fromMetadataOutputRect:
transformedRect)
self.highlightView.frame = convertedRect
}
}
}
extension ViewController:
AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer:
CMSampleBuffer, from connection: AVCaptureConnection) {
guard let pixelBuffer =
CMSampleBufferGetImageBuffer(sampleBuffer),
let observation = lastObservation else {
return
}
let request = VNTrackObjectRequest(detectedObjectObservation:
observation) { [unowned self] request, error in
self.handle(request, error: error)
}
request.trackingLevel = .accurate
do {
try handler.perform([request], on: pixelBuffer)
}
catch {
print(error)
}
}
}
Any help will be appreciated!!
Thanks.
I am not so good at vision and core ml, but apparently your code looks fine. One thing you can do is check when vision does not get any tracking in the buffer, you have to mark its property isLastFrame true if tracking request confidence value falls to 0.
if !trackingRequest.isLastFrame {
if observation.confidence > 0.7 {
trackingRequest.inputObservation = observation
} else {
trackingRequest.isLastFrame = true
}
newTrackingRequests.append(trackingRequest)
}
This way its easy to find out whether vision tracking request lost tracking object or it just tracking the wrong object.
I am trying to make a selfie app. I am unable to access the front camera. When I run the below code, the app still picks only the back camera.I am working on Xcode 7.1.1. I have also tried it on Xcode 7.2 Beta. Can someone help ? Tried solutions given in the other question here, but that did not solve the problem.
import UIKit
import AVFoundation
class ViewController: UIViewController {
#IBOutlet weak var previewView: UIView!
#IBOutlet weak var capturedImage: UIImageView!
var captureSession: AVCaptureSession?
var stillImageOutput: AVCaptureStillImageOutput?
var previewLayer: AVCaptureVideoPreviewLayer?
enum AVCaptureDevicePosition : Int {
case Unspecified
case Front
case Back
}
func viewDidAppear(animated: Bool){
super.viewDidAppear(animated)
previewLayer!.frame = previewView.bounds
reloadCamera()
}
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view, typically from a nib.
}
func reloadCamera() {
captureSession = AVCaptureSession()
captureSession!.sessionPreset = AVCaptureSessionPresetPhoto
//let backCamera = AVCaptureDevice.defaultDeviceWithMediaType(AVMediaTypeVideo)
// access the front camera
let videoDevices = AVCaptureDevice.devicesWithMediaType(AVMediaTypeVideo)
var captureDevice:AVCaptureDevice
for device in videoDevices{
let device = device as AVCaptureDevice
if device.position == AVCaptureDevicePosition.Front {
captureDevice = device
}
}
var error: NSError?
var input: AVCaptureDeviceInput!
do {
input = try AVCaptureDeviceInput(device : captureDevice)
} catch let error1 as NSError {
error = error1
input = nil
}
if error == nil && captureSession!.canAddInput(input) {
captureSession!.addInput(input)
stillImageOutput = AVCaptureStillImageOutput()
stillImageOutput!.outputSettings = [AVVideoCodecKey :AVVideoCodecJPEG]
if captureSession!.canAddOutput(stillImageOutput){
captureSession!.addOutput(stillImageOutput)
previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer!.videoGravity = AVLayerVideoGravityResizeAspect
previewLayer!.connection?.videoOrientation = AVCaptureVideoOrientation.Portrait
previewView.layer.addSublayer(previewLayer!)
captureSession!.startRunning()
}
}
}
#IBAction func didPressTakePhoto(sender: UIButton) {
if let videoConnection = stillImageOutput!.connectionWithMediaType(AVMediaTypeVideo){
videoConnection.videoOrientation = AVCaptureVideoOrientation.Portrait
stillImageOutput?.captureStillImageAsynchronouslyFromConnection(videoConnection, completionHandler: {(sampleBuffer, error) in
if (sampleBuffer != nil){
let imageData = AVCaptureStillImageOutput.jpegStillImageNSDataRepresentation(sampleBuffer)
let dataProvider = CGDataProviderCreateWithCFData(imageData)
let cgImageRef = CGImageCreateWithJPEGDataProvider(dataProvider,nil,true,CGColorRenderingIntent.RenderingIntentDefault)
let image = UIImage(CGImage: cgImageRef!, scale: 1.0, orientation: UIImageOrientation.Right)
self.capturedImage.image = image
UIImageWriteToSavedPhotosAlbum(image, nil, nil, nil)
}
})
}
}
#IBAction func didPressTakeAnother(sender : AnyObject){
captureSession!.startRunning()
}
}