MTAudioProcessingTap - prepare{} and process{} callbacks are not executing - swift

Question

I am working on loading a (local) movie into AVPlayer and applying processing to the audio track with an audioTapProcessor. So far I've found great GitHub examples here, here, and here. I'm using the "tap cookie" approach used in the last link and in an answer to this previous question.
Audio & video playback are working fine. However, my tapPrepare and tapProcess callbacks are not being called, but Init and Finalize are. So I'm doing something both right and wrong. relevant code attached -- Any help appreciated!
import Foundation
import AVFoundation
import AudioToolbox
import MediaToolbox
import CoreAudioTypes
class PlayerViewController: UIViewController {
class TapCookie {
weak var content: PlayerViewController?
deinit {
print("TapCookie deinit") // appears after tapFinalize
}
}
// MARK: Properties
var playerAsset: AVURLAsset?
var playerItem: AVPlayerItem! = nil
var audioProcessingFormat: AudioStreamBasicDescription?
private var tracksObserver: NSKeyValueObservation?
// MARK: Button to trigger actions
#IBAction func selectVideo(_ sender: Any) {
// starts doing stuff:
// - select a video file from device, extract movieURL string ...
playerAsset = AVURLAsset(url: movieURL)
playerItem = AVPlayerItem(url: movieURL)
//... then send asset to AVPlayer (not shown)
// set up audioProcessingTap
tracksObserver = playerItem.observe(\AVPlayerItem.tracks, options: [.initial, .new]) {
[unowned self] item, change in
installTap(playerItem: playerItem)
}
}
func installTap(playerItem: AVPlayerItem) {
let cookie = TapCookie()
cookie.content = self
var callbacks = MTAudioProcessingTapCallbacks(
version: kMTAudioProcessingTapCallbacksVersion_0,
clientInfo: UnsafeMutableRawPointer(Unmanaged.passRetained(cookie).toOpaque()),
init: tapInit,
finalize: tapFinalize,
prepare: tapPrepare,
unprepare: tapUnprepare,
process: tapProcess)
var tap: Unmanaged<MTAudioProcessingTap>?
let err = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap)
assert(noErr == err);
// tapInit successfully called after MTAudioProcessingTapCreate
let audioMix = AVMutableAudioMix()
let audioTrack = playerItem.asset.tracks(withMediaType: AVMediaType.audio).first! //use first audio track
let inputParams = AVMutableAudioMixInputParameters(track: audioTrack)
inputParams.audioTapProcessor = tap?.takeRetainedValue()
audioMix.inputParameters = [inputParams]
playerItem.audioMix = audioMix
}
// MARK: install tap callbacks
let tapInit: MTAudioProcessingTapInitCallback = {
(tap, clientInfo, tapStorageOut) in
tapStorageOut.pointee = clientInfo
print("tapInit tap: \(tap)\n clientInfo: \(String(describing: clientInfo))\n tapStorageOut: \(tapStorageOut)\n")
}
// tapPrepare not called !!
let tapPrepare: MTAudioProcessingTapPrepareCallback = {
(tap, maxFrames, processingFormat) in
print("tapPrepare tap: \(tap), maxFrames: \(maxFrames)\n processingFormat: \(processingFormat)")
let cookie = Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
cookie.content!.audioProcessingFormat = AudioStreamBasicDescription(mSampleRate: processingFormat.pointee.mSampleRate,
mFormatID: processingFormat.pointee.mFormatID,
mFormatFlags: processingFormat.pointee.mFormatFlags,
mBytesPerPacket: processingFormat.pointee.mBytesPerPacket,
mFramesPerPacket: processingFormat.pointee.mFramesPerPacket,
mBytesPerFrame: processingFormat.pointee.mBytesPerFrame,
mChannelsPerFrame: processingFormat.pointee.mChannelsPerFrame,
mBitsPerChannel: processingFormat.pointee.mBitsPerChannel,
mReserved: processingFormat.pointee.mReserved)
}
let tapUnprepare: MTAudioProcessingTapUnprepareCallback = {
(tap) in
print("tapUnprepare \(tap)")
}
// tapProcess not called !!
let tapProcess: MTAudioProcessingTapProcessCallback = {
(tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in
print("tapProcess \(tap)\n \(numberFrames)\n \(flags)\n \(bufferListInOut)\n \(numberFramesOut)\n \(flagsOut)\n")
let status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut)
if noErr != status {
print("get audio: \(status)")
}
let cookie = Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
guard let cookieContent = cookie.content else {
print("Tap callback: cookie content was deallocated!")
return
}
// process audio here...
}
let tapFinalize: MTAudioProcessingTapFinalizeCallback = {
(tap) in
print("tapFinalize \(tap)")
// release cookie
Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).release()
}
}

Answer 1

You need to create an AVPlayer
player = AVPlayer(playerItem: playerItem)
and then at some point start it playing:
player.play()
Then the prepare and process callbacks will be called.