How to achieve better performance in converting UIView to CVPixelBuffer? - swift

Question

I'm wondering if it's possible to achieve a better performance in converting the UIView into CVPixelBuffer.
My app converts a sequence of UIViews first into UIImages and then into CVPixelBuffers as shown below. In the end, I record all these images/frames into an AVAssetWriterInput and save the result as a movie file.
Thank you in advance!
Best, Aibek
func viewToImage(view: UIView) -> CGImage {
let rect: CGRect = container.frame
UIGraphicsBeginImageContextWithOptions(rect.size, true, 1)
let context: CGContext = UIGraphicsGetCurrentContext()!
view.layer.render(in: context)
let img = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return img!.cgImage
}
func imageToBuffer(image: CGImage) -> CVPixelBuffer? {
let frameSize = CGSize(width: image.width, height: image.height)
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(frameSize.width), Int(frameSize.height), kCVPixelFormatType_32BGRA, nil, &pixelBuffer)
if status != kCVReturnSuccess {
return nil
}
CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
let data = CVPixelBufferGetBaseAddress(pixelBuffer!)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue)
let context = CGContext(data: data, width: Int(frameSize.width), height: Int(frameSize.height), bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: rgbColorSpace, bitmapInfo: bitmapInfo.rawValue)
context?.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
return pixelBuffer
}

Answer 1

You'd better to see this. https://stackoverflow.com/a/61862728/13680955
In short, this sample converts UIView to MTLTexture in 12ms.
Sure you can use CVPixelBuffer directly, but I used MTLTexture to make video and no issue was on it.
If you are struggling with the performance, too slow or weird to use, try to do this.
With MTLTexture
import AVFoundation
import MetalKit
class VideoRecorder {
let assetWriter: AVAssetWriter
let assetWriterVideoInput: AVAssetWriterInput
let assetWriterInputPixelBufferAdapter: AVAssetWriterInputPixelBufferAdaptor
var recordingStartTime = TimeInterval(0)
var recordingElapsedTime = TimeInterval(0)
let url: URL = {
let fileName = "exported_video.mp4"
return FileManager.default.temporaryDirectory.appendingPathComponent(fileName)
}()
init(outputSize: CGSize) throws {
if FileManager.default.fileExists(atPath: url.path) {
try FileManager.default.removeItem(at: url)
}
let fileType: AVFileType = .mov
assetWriter = try AVAssetWriter(outputURL: url, fileType: fileType)
let mediaType: AVMediaType = .video
let outputSettings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: outputSize.width,
AVVideoHeightKey: outputSize.height
]
assetWriterVideoInput = AVAssetWriterInput(mediaType: mediaType, outputSettings: outputSettings)
assetWriterVideoInput.expectsMediaDataInRealTime = false
let sourcePixelBufferAttributes: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey as String: outputSize.width,
kCVPixelBufferHeightKey as String: outputSize.height
]
assetWriterInputPixelBufferAdapter = AVAssetWriterInputPixelBufferAdaptor(
assetWriterInput: assetWriterVideoInput, sourcePixelBufferAttributes: sourcePixelBufferAttributes)
assetWriter.add(assetWriterVideoInput)
}
private static func currentTimestampString() -> String {
let date = Date()
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "yyyy-MM-dd HH:mm:ss"
return dateFormatter.string(from: date)
}
public func start() {
print("videoRecorder.start")
assetWriter.startWriting()
assetWriter.startSession(atSourceTime: .zero)
recordingStartTime = CACurrentMediaTime()
}
public func cancel() {
#if DEBUG
print("videoRecorder.cancel")
#endif
assetWriterVideoInput.markAsFinished()
assetWriter.cancelWriting()
}
public func finish(_ callback: #escaping () -> Void) {
print("videoRecorder.finish")
assetWriterVideoInput.markAsFinished()
assetWriter.finishWriting {
self.recordingElapsedTime = CACurrentMediaTime() - self.recordingStartTime
print("videoRecorder.finish elapsedTime: \(self.recordingElapsedTime)")
callback()
}
}
private var pixelBuffer: CVPixelBuffer?
public func writeFrame(texture: MTLTexture, at presentationTime: CMTime) {
print("videoRecorder.writeFrame: \(presentationTime)")
if pixelBuffer == nil {
guard let pixelBufferPool = assetWriterInputPixelBufferAdapter.pixelBufferPool else {
print("Pixel buffer asset writer input did not have a pixel buffer pool available;")
print("cannot retrieve frame")
return
}
var maybePixelBuffer: CVPixelBuffer?
let status = CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &maybePixelBuffer)
if status != kCVReturnSuccess {
print("Could not get pixel buffer from asset writer input; dropping frame...")
return
}
pixelBuffer = maybePixelBuffer
print("videoRecorder.writeFrame: pixelBuffer was created: \(String(describing: pixelBuffer))")
}
guard let pixelBuffer = pixelBuffer else {
print("videoRecorder.writeFrame: NO pixelBuffer")
return
}
writeFrame(texture: texture, at: presentationTime, with: pixelBuffer)
}
private func writeFrame(texture: MTLTexture, at presentationTime: CMTime, with pixelBuffer: CVPixelBuffer) {
while !assetWriterVideoInput.isReadyForMoreMediaData {
//
print("NOT ready for more media data at: \(presentationTime)")
}
CVPixelBufferLockBaseAddress(pixelBuffer, [])
let pixelBufferBytes = CVPixelBufferGetBaseAddress(pixelBuffer)!
// Use the bytes per row value from the pixel buffer since its stride may be rounded up to be 16-byte aligned
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let region = MTLRegionMake2D(0, 0, texture.width, texture.height)
texture.getBytes(pixelBufferBytes, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)
assetWriterInputPixelBufferAdapter.append(pixelBuffer, withPresentationTime: presentationTime)
CVPixelBufferUnlockBaseAddress(pixelBuffer, [])
}
}

Answer 2

Converting the UIViews into MTLTextures and recording them into a video file using the Recorder provided by Mayo didn't increase the performance actually.
However, the recorder is able to write MTLTextures in real-time. That meant for me that I can re-write all the animations using Metal and use the recorder.