After compressing my audio file, why can I not play the file? - swift

Audio file will not play after reducing it using AVAssetReader/ AVAssetWriter
At the moment, the whole function is being executed fine, with no errors thrown.
For some reason, when I go inside the document directory of the simulator via terminal, the audio file will not play through iTunes and comes up with error when trying to open with quicktime "QuickTime Player can't open "test1.m4a"
Does anyone specialise in this area and understand why this isn't working?
protocol FileConverterDelegate {
func fileConversionCompleted()
}
class WKAudioTools: NSObject {
var delegate: FileConverterDelegate?
var url: URL?
var assetReader: AVAssetReader?
var assetWriter: AVAssetWriter?
func convertAudio() {
let documentDirectory = try! FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
let exportURL = documentDirectory.appendingPathComponent(Assets.soundName1).appendingPathExtension("m4a")
url = Bundle.main.url(forResource: Assets.soundName1, withExtension: Assets.mp3)
guard let assetURL = url else { return }
let asset = AVAsset(url: assetURL)
//reader
do {
assetReader = try AVAssetReader(asset: asset)
} catch let error {
print("Error with reading >> \(error.localizedDescription)")
}
let assetReaderOutput = AVAssetReaderAudioMixOutput(audioTracks: asset.tracks, audioSettings: nil)
//let assetReaderOutput = AVAssetReaderTrackOutput(track: track!, outputSettings: nil)
guard let assetReader = assetReader else {
print("reader is nil")
return
}
if assetReader.canAdd(assetReaderOutput) == false {
print("Can't add output to the reader ☹️")
return
}
assetReader.add(assetReaderOutput)
// writer
do {
assetWriter = try AVAssetWriter(outputURL: exportURL, fileType: .m4a)
} catch let error {
print("Error with writing >> \(error.localizedDescription)")
}
var channelLayout = AudioChannelLayout()
memset(&channelLayout, 0, MemoryLayout.size(ofValue: channelLayout))
channelLayout.mChannelLayoutTag = kAudioChannelLayoutTag_Stereo
// use different values to affect the downsampling/compression
let outputSettings: [String: Any] = [AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100.0,
AVNumberOfChannelsKey: 2,
AVEncoderBitRateKey: 128000,
AVChannelLayoutKey: NSData(bytes: &channelLayout, length: MemoryLayout.size(ofValue: channelLayout))]
let assetWriterInput = AVAssetWriterInput(mediaType: .audio, outputSettings: outputSettings)
guard let assetWriter = assetWriter else { return }
if assetWriter.canAdd(assetWriterInput) == false {
print("Can't add asset writer input ☹️")
return
}
assetWriter.add(assetWriterInput)
assetWriterInput.expectsMediaDataInRealTime = false
// MARK: - File conversion
assetWriter.startWriting()
assetReader.startReading()
let audioTrack = asset.tracks[0]
let startTime = CMTime(seconds: 0, preferredTimescale: audioTrack.naturalTimeScale)
assetWriter.startSession(atSourceTime: startTime)
// We need to do this on another thread, so let's set up a dispatch group...
var convertedByteCount = 0
let dispatchGroup = DispatchGroup()
let mediaInputQueue = DispatchQueue(label: "mediaInputQueue")
//... and go
dispatchGroup.enter()
assetWriterInput.requestMediaDataWhenReady(on: mediaInputQueue) {
while assetWriterInput.isReadyForMoreMediaData {
let nextBuffer = assetReaderOutput.copyNextSampleBuffer()
if nextBuffer != nil {
assetWriterInput.append(nextBuffer!) // FIXME: Handle this safely
convertedByteCount += CMSampleBufferGetTotalSampleSize(nextBuffer!)
} else {
// done!
assetWriterInput.markAsFinished()
assetReader.cancelReading()
dispatchGroup.leave()
DispatchQueue.main.async {
// Notify delegate that conversion is complete
self.delegate?.fileConversionCompleted()
print("Process complete 🎧")
if assetWriter.status == .failed {
print("Writing asset failed ☹️ Error: ", assetWriter.error)
}
}
break
}
}
}
}
}

You need to call finishWriting on your AVAssetWriter to get the output completely written:
assetWriter.finishWriting {
DispatchQueue.main.async {
// Notify delegate that conversion is complete
self.delegate?.fileConversionCompleted()
print("Process complete 🎧")
if assetWriter.status == .failed {
print("Writing asset failed ☹️ Error: ", assetWriter.error)
}
}
}
If exportURL exists before you start the conversion, you should remove it, otherwise the conversion will fail:
try! FileManager.default.removeItem(at: exportURL)
As #matt points out, why the buffer stuff when you could do the conversion more simply with an AVAssetExportSession, and also why convert one of your own assets when you could distribute it already in the desired format?

Related

copyNextSampleBuffer hanging for AVAssetReaderTrackOutput

I am trying to create a video compression tool which simply takes in a video file URL, compresses it, and returns the new URL in swift 5 for ios 16.
I have been following a few tutorials online, but they all use functions which have since been deprecated, so I have put together this code here which is a refactoring from various sources:
import Foundation
import AVFoundation
class VideoCompressorModel: ObservableObject {
let bitrate = 2_500_000 // MBPs
func compressFile(urlToCompress: URL, outputURL: URL, completionHandler:#escaping (URL?)->Void) async {
let asset = AVAsset(url: urlToCompress);
//create asset reader
var assetReader: AVAssetReader?
do{
assetReader = try AVAssetReader(asset: asset)
} catch{
assetReader = nil
}
guard let reader = assetReader else{
completionHandler(nil)
return
}
var videoTrack: AVAssetTrack?
var audioTrack: AVAssetTrack?
do {
videoTrack = try await asset.loadTracks(withMediaType: AVMediaType.video).first
audioTrack = try await asset.loadTracks(withMediaType: AVMediaType.audio).first
} catch {
completionHandler(nil)
return
}
guard let videoTrack, let audioTrack else {
completionHandler(nil)
return
}
let videoReaderSettings: [String:Any] = [kCVPixelBufferPixelFormatTypeKey as String:kCVPixelFormatType_32ARGB ]
// ADJUST BIT RATE OF VIDEO HERE
var videoSettings: [String:Any]?
do {
videoSettings = await [
AVVideoCompressionPropertiesKey: [AVVideoAverageBitRateKey:self.bitrate],
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoHeightKey: try videoTrack.load(.naturalSize).height,
AVVideoWidthKey: try videoTrack.load(.naturalSize).width
]
} catch {
completionHandler(nil)
return
}
guard let videoSettings else {
completionHandler(nil)
return
}
let audioSettings = [
AVSampleRateKey: 44100,
AVFormatIDKey: kAudioFormatLinearPCM
]
let assetReaderVideoOutput = AVAssetReaderTrackOutput(track: videoTrack, outputSettings: videoReaderSettings)
let assetReaderAudioOutput = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: audioSettings)
if reader.canAdd(assetReaderVideoOutput){
reader.add(assetReaderVideoOutput)
}else{
completionHandler(nil)
return
}
if reader.canAdd(assetReaderAudioOutput){
reader.add(assetReaderAudioOutput)
} else{
completionHandler(nil)
return
}
let audioInput = AVAssetWriterInput(mediaType: AVMediaType.audio, outputSettings: nil)
let videoInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: videoSettings)
do {
videoInput.transform = try await videoTrack.load(.preferredTransform)
} catch{
completionHandler(nil)
return
}
//we need to add samples to the video input
let videoInputQueue = DispatchQueue(label: "videoQueue")
let audioInputQueue = DispatchQueue(label: "audioQueue")
var assetWriter: AVAssetWriter?
do{
assetWriter = try AVAssetWriter(outputURL: outputURL, fileType: AVFileType.mov)
} catch {
assetWriter = nil
}
guard let assetWriter else{
completionHandler(nil)
return
}
assetWriter.shouldOptimizeForNetworkUse = true
assetWriter.add(videoInput)
assetWriter.add(audioInput)
assetWriter.startWriting()
reader.startReading()
assetWriter.startSession(atSourceTime: CMTime.zero)
let closeWriterBothDone:()->Void = {
assetWriter.finishWriting(completionHandler: {
completionHandler((assetWriter.outputURL))
})
reader.cancelReading()
}
print("STARTING ")
let closeWriterAudioDone:()->Void = {
print("FINISHED AUDIO")
videoInput.requestMediaDataWhenReady(on: videoInputQueue) {
while(videoInput.isReadyForMoreMediaData){
let sample = assetReaderVideoOutput.copyNextSampleBuffer()
if (sample != nil){
videoInput.append(sample!)
} else{
videoInput.markAsFinished()
DispatchQueue.main.async {
closeWriterBothDone()
}
break;
}
}
}
}
audioInput.requestMediaDataWhenReady(on: audioInputQueue) {
while(audioInput.isReadyForMoreMediaData){
let sample = assetReaderAudioOutput.copyNextSampleBuffer()
print("hi")
if (sample != nil){
audioInput.append(sample!)
} else{
audioInput.markAsFinished()
DispatchQueue.main.async {
closeWriterAudioDone()
}
break;
}
}
}
}
}
The issue that occurs is that copyNextSampleBuffer for the audio track output hangs after a few calls. I test this simply by having a print("hi") which only gets called a handful of times before the code blocks.
I have tried changing the audio options provided to AVAssetReaderTrackOutput, which I originally had simply as nil, and this only had the effect of increasing the number of times copyNextSampleBuffer gets called before it freezes.
Perhaps there is a better way overall to compress video in newer versions of swift, as this seems somewhat unelegant? If not, is there any bugs in my code that is causing it to hang?

AVAudioEngine doesn't playback a sound

I am trying to play with AVAudioEngine to playback the wav file. I tried to do it in a few different ways, but nothing work.
Try 1
...
private var audioEngine: AVAudioEngine = AVAudioEngine()
private var mixer: AVAudioMixerNode = AVAudioMixerNode()
private var audioFilePlayer: AVAudioPlayerNode = AVAudioPlayerNode()
func Play1() {
guard let filePath = Bundle.main.url(forResource: "testwav", withExtension: "wav", subdirectory: "res") else {
print("file not found")
return
}
print("\(filePath)")
guard let audioFile = try? AVAudioFile(forReading: filePath) else{ return }
let audioFormat = audioFile.processingFormat
let audioFrameCount = UInt32(audioFile.length)
guard let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount) else{ return }
do{
try audioFile.read(into: audioFileBuffer)
} catch{
print("over")
}
let mainMixer = audioEngine.mainMixerNode
audioEngine.attach(audioFilePlayer)
audioEngine.connect(audioFilePlayer, to:mainMixer, format: audioFileBuffer.format)
audioEngine.connect(mainMixer, to:audioEngine.outputNode, format: audioFileBuffer.format)
try? audioEngine.start()
audioFilePlayer.play()
audioFilePlayer.scheduleBuffer(audioFileBuffer, at: nil, options:AVAudioPlayerNodeBufferOptions.loops)
}
...
Try 2
...
private var audioEngine: AVAudioEngine = AVAudioEngine()
private var mixer: AVAudioMixerNode = AVAudioMixerNode()
private var audioFilePlayer: AVAudioPlayerNode = AVAudioPlayerNode()
func Play2() {
DispatchQueue.global(qos: .background).async {
self.audioEngine.attach(self.mixer)
self.audioEngine.connect(self.mixer, to: self.audioEngine.outputNode, format: nil)
// !important - start the engine *before* setting up the player nodes
try! self.audioEngine.start()
let audioPlayer = AVAudioPlayerNode()
self.audioEngine.attach(audioPlayer)
// Notice the output is the mixer in this case
self.audioEngine.connect(audioPlayer, to: self.mixer, format: nil)
guard let fileUrl = Bundle.main.url(forResource: "testwav", withExtension: "wav", subdirectory: "res") else {
// guard let url = Bundle.main.url(forResource: "audiotest", withExtension: "mp3", subdirectory: "res") else {
print("mp3 not found")
return
}
do {
let file = try AVAudioFile(forReading: fileUrl)
audioPlayer.scheduleFile(file, at: nil, completionHandler: nil)
audioPlayer.play(at: nil)
} catch let error {
print(error.localizedDescription)
}
}
}
...
...
private var audioEngine: AVAudioEngine = AVAudioEngine()
private var mixer: AVAudioMixerNode = AVAudioMixerNode()
private var audioFilePlayer: AVAudioPlayerNode = AVAudioPlayerNode()
func Play3() {
DispatchQueue.global(qos: .background).async {
self.audioEngine = AVAudioEngine()
_ = self.audioEngine.mainMixerNode
self.audioEngine.prepare()
do {
try self.audioEngine.start()
} catch {
print(error)
}
guard let url = Bundle.main.url(forResource: "testwav", withExtension: "wav", subdirectory: "res") else {
// guard let url = Bundle.main.url(forResource: "audiotest", withExtension: "mp3", subdirectory: "res") else {
print("mp3 not found")
return
}
let player = AVAudioPlayerNode()
player.volume = 1.0
do {
let audioFile = try AVAudioFile(forReading: url)
let format = audioFile.processingFormat
print(format)
self.audioEngine.attach(player)
self.audioEngine.connect(player, to: self.audioEngine.mainMixerNode, format: format)
player.scheduleFile(audioFile, at: nil, completionHandler: nil)
} catch let error {
print(error.localizedDescription)
}
player.play()
}
}
...
Also should be mentioned that there are no errors, while debugging I see that all the methods are executed and everything is ok, but I don't hear sound playback...
What am I doing wrong?
Try to activate your audio session with the following method:
func setActive(_ active: Bool, options: AVAudioSession.SetActiveOptions = []) throws.
Please note that if another active audio session has higher priority than yours (for example, a phone call), and neither audio session allows mixing, attempting to activate your audio session fails. Deactivating an audio session that has running audio objects stops them, deactivates the session, and return an AVAudioSession.ErrorCode.isBusy error.

Video data hash changes everytime I use `AVMutableComposition` & `AVAssetExportSession` to export even if goes through same steps with no changes

Minimum snippet to regenerate. Exports a new exportURL from given videoURL.
While exporting I am
Adding the video track
Adding the audio track
Keeping the complete time range for export
let asset = AVURLAsset(url: videoURL)
let mixComposition = AVMutableComposition()
guard let compositionTrack = mixComposition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid),
let assetTrack = asset.tracks(withMediaType: .video).first
else {
print("Something is wrong with the asset.")
return
}
do {
let startTime = .zero
let endTime = asset.duration
let timeRange = CMTimeRangeFromTimeToTime(start: startTime, end: endTime)
try compositionTrack.insertTimeRange(timeRange, of: assetTrack, at: .zero)
// if audio isn't muted in the editor, add audio track from the asset
if let audioAssetTrack = asset.tracks(withMediaType: .audio).first,
let compositionAudioTrack = mixComposition.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) {
try compositionAudioTrack.insertTimeRange(timeRange, of: audioAssetTrack, at: .zero)
}
} catch {
print(error)
return
}
guard let export = AVAssetExportSession(asset: mixComposition, presetName: videoQuality.exportPreset)
else {
print("Cannot create export session.")
onComplete(nil)
return
}
let videoName = UUID().uuidString
let exportURL = URL(fileURLWithPath: NSTemporaryDirectory())
.appendingPathComponent(videoName)
.appendingPathExtension("mp4")
export.outputFileType = .mp4
export.outputURL = exportURL
export.exportAsynchronously {
DispatchQueue.main.async {
switch export.status {
case .completed:
onComplete(exportURL)
default:
print("Something went wrong during export.")
print(export.error ?? "unknown error")
break
}
}
}
Now after I have this exportURL can get data from it. And that data is hashed. But even if the same video goes through this same process which does not apply any edit to the video why the hash is changed?
In case the function is needed which I am using to generate the hash.
extension Data {
var digest: Data {
let digestLength = Int(CC_SHA256_DIGEST_LENGTH)
var hash = [UInt8](repeating: 0, count: digestLength)
CC_SHA256(bytes, UInt32(count), &hash)
return Data(bytes: hash, count: digestLength)
}
var hexString: String {
let hexString = map { String(format: "%02.2hhx", $0) }.joined()
return hexString
}
}

AVAssetWriter recorded audio no sound

My app can record the audio from the chat and save it in file. I recorded some music on the app screen but when I playback the audio.m4a file there is no sound coming out. The file show as "Apple MPEG-4 audio" and has 12KB size. Did I config the setting wrong? Thanks in advence.
edit: I added the stop recording function.
var assetWriter: AVAssetWriter?
var input: AVAssetWriterInput?
var channelLayout = AudioChannelLayout()
func record() {
guard let doc = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else {
return
}
let inputURL = docURL.appendingPathComponent("audio.m4a")
do {
try assetWriter = AVAssetWriter(outputURL: inputURL, fileType: .m4a)
} catch {
print("error: \(error)")
assetWriter = nil
return
}
guard let assetWriter = assetWriter else {
return
}
channelLayout.mChannelLayoutTag = kAudioChannelLayoutTag_MPEG_5_1_D
let audioSettings: [String : Any] = [
AVNumberOfChannelsKey: 6,
AVFormatIDKey: kAudioFormatMPEG4AAC_HE,
AVSampleRateKey: 44100,
AVEncoderBitRateKey: 128000,
AVChannelLayoutKey: NSData(bytes: &channelLayout, length: MemoryLayout.size(ofValue: channelLayout)),
]
input = AVAssetWriterInput(mediaType: .audio, outputSettings: settings)
guard let audioInput = input else {
print("Failed to find input.")
return
}
audioInput.expectsMediaDataInRealTime = true
if ((assetWriter.canAdd(audioInput)) != nil) {
assetWriter.add(audioInput)
}
RPScreenRecorder.shared().startCapture(handler: { (sample, bufferType, error) in
guard error == nil else {
print("Failed to capture with error: \(String(describing: error))")
return
}
if bufferType == .audioApp {
if assetWriter.status == AVAssetWriter.Status.unknown {
if ((assetWriter.startWriting()) != nil) {
assetWriter.startSession(atSourceTime: CMSampleBufferGetPresentationTimeStamp(sample))
}
}
if assetWriter.status == AVAssetWriter.Status.writing {
if audioInput.isReadyForMoreMediaData == true {
if audioInput.append(sample) == false {
}
}
}
}
})
}
func stopRecord() {
RPScreenRecorder.shared().stopCapture{ (error) in
self.audioInput.markAsFinished()
if error == nil{
self.assetWriter.finishWriting {
print("finish writing")
}
} else {
print(error as Any)
}
}
}
In light of your comments, you definitely don't need six channel audio. Try these simpler mono audio settings.
let audioSettings: [String : Any] = [
AVNumberOfChannelsKey: 1,
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100,
AVEncoderBitRateKey: 128000,
]
You don't say whether this is on iOS or macOS. You have a problem on macOS because as of 11.2.1 no .audioApp buffers are captured. If you still want microphone, you can configure that:
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = true
recorder.startCapture(handler: { (sample, bufferType, error) in
if bufferType == .audioMic {
// etc
}
})
Don't bother checking for writer status, just append buffers when you can
if audioInput.isReadyForMoreMediaData {
if !audioInput.append(sample) {
// do something
}
}
PREVIOUSLY
You need to call assetWriter.finishWriting at some point.
It's interesting that you have 6 channel input. Are you using a special device or some kind of virtual device?

Is it possible to load images recovered from a server in runtime into a plane object?

I have been asked to build an app that shows a catalog with AR, so what I need to do is pretty simple: when an user chooses a product I must load the image recovered in base64 from the server into a plane object. Is this possible with swift - arkit ? Or are all the sprites/images/textures required to be previously loaded into the assets folder?
You can definitely download resources from a server, save them to the device (e.g in NSDocumentsDirectory), and then load with the file URL. I do it for a similar use case as yours -at least it sounds so, per the description you gave-
EDIT
Here's the relevant code. I use Alamofire to download from the server and ZIPFoundation for unzipping. I believe that if you just need to download an image, it'll be a bit simpler, probably not needing the unzip part.
let modelsDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
func loadNodeWithID(_ id: String, completion: #escaping (SCNNode?) -> Void) {
// Check that assets for that model are not already downloaded
let fileManager = FileManager.default
let dirForModel = modelsDirectory.appendingPathComponent(id)
let dirExists = fileManager.fileExists(atPath: dirForModel.path)
if dirExists {
completion(loadNodeWithIdFromDisk(id))
} else {
let dumbURL = "http://yourserver/yourfile.zip"
downloadZip(from: dumbURL, at: id) {
if let url = $0 {
print("Downloaded and unzipped at: \(url.absoluteString)")
completion(self.loadNodeWithIdFromDisk(id))
} else {
print("Something went wrong!")
completion(nil)
}
}
}
}
func loadNodeWithIdFromDisk(_ id: String) -> SCNNode? {
let fileManager = FileManager.default
let dirForModel = modelsDirectory.appendingPathComponent(id)
do {
let files = try fileManager.contentsOfDirectory(atPath: dirForModel.path)
if let objFile = files.first(where: { $0.hasSuffix(".obj") }) {
let objScene = try? SCNScene(url: dirForModel.appendingPathComponent(objFile), options: nil)
let objNode = objScene?.rootNode.firstChild()
return objNode
} else {
print("No obj file in directory: \(dirForModel.path)")
return nil
}
} catch {
print("Could not enumarate files or load scene: \(error)")
return nil
}
}
func downloadZip(from urlString: String, at destFileName: String, completion: ((URL?) -> Void)?) {
print("Downloading \(urlString)")
let fullDestName = destFileName + ".zip"
let destination: DownloadRequest.DownloadFileDestination = { _, _ in
let fileURL = modelsDirectory.appendingPathComponent(fullDestName)
return (fileURL, [.removePreviousFile, .createIntermediateDirectories])
}
Alamofire.download(urlString, to: destination).response { response in
let error = response.error
if error == nil {
if let filePath = response.destinationURL?.path {
let nStr = NSString(string: filePath)
let id = NSString(string: nStr.lastPathComponent).deletingPathExtension
print(response)
print("file downloaded at: \(filePath)")
let fileManager = FileManager()
let sourceURL = URL(fileURLWithPath: filePath)
var destinationURL = modelsDirectory
destinationURL.appendPathComponent(id)
do {
try fileManager.createDirectory(at: destinationURL, withIntermediateDirectories: true, attributes: nil)
try fileManager.unzipItem(at: sourceURL, to: destinationURL)
completion?(destinationURL)
} catch {
completion?(nil)
print("Extraction of ZIP archive failed with error: \(error)")
}
} else {
completion?(nil)
print("File path not found")
}
} else {
// Handle error
completion?(nil)
}
}
}