What's the correct way to generate a MTLTexture backed by a CVPixelBuffer?
I have the following code, but it seems to leak:
func PixelBufferToMTLTexture(pixelBuffer:CVPixelBuffer) -> MTLTexture
{
var texture:MTLTexture!
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let format:MTLPixelFormat = .BGRA8Unorm
var textureRef : Unmanaged<CVMetalTextureRef>?
let status = CVMetalTextureCacheCreateTextureFromImage(nil,
videoTextureCache!.takeUnretainedValue(),
pixelBuffer,
nil,
format,
width,
height,
0,
&textureRef)
if(status == kCVReturnSuccess)
{
texture = CVMetalTextureGetTexture(textureRef!.takeUnretainedValue())
}
return texture
}
Ah, I was missing: textureRef?.release()
Related
I really need help. I'm creating a DataMatrix reader, and part of codes are with white background only and causes any problem with AVFoundation, but another part has grey with shimmer background (see image below), and this driving me crazy.
What I've tried:
1) AVFoundation with its metaDataOutput works perfect only with white background, and there was no success with shimmer grey
2)zxing - actually can't find any working example for swift, and their sample from GitHub find no Datamatrix on grey too (with datamatrix as qr code type), will be thankful for tutorial or smth like this (zxingObjC for Swift)
3)about 20 libs from cocoapods/github - nothing with grey back again
4) then I found that Vision perfectly detect Datamatrix on white from photo, so I decided to work with this lib and changed the way: no more catching a video output, only UIImages, then handle them and detect DataMatrix using Vision framework.
And to convert colors I've tried:
CIFilters (ColorsControls, NoirEffect), GPU filters (monochrome, luminance, averageLuminance,adaptiveTreshold) playing with params
In the end I have no solution that will work 10 from 10 with my DataMatrix stickers. sometimes it works with GPUImageAverageLuminanceThresholdFilter and GPUImageAdaptiveThresholdFilter, but about 20% luck.
And this 20% luck only at daylight, with electric light comes shimmer-glitter, I think.
Any advice will be helpful for me! Maybe there is nice solution with Zxing for Swift, which I can't find. Or there is no need to use Vision and get frames from AVFoundation, but how?
I-nigma etc. catch my stickers perfectly, from live video, so there should be the way. Android version of my scanner use Zxing, and I guess that Zxing do the job..
My scanning scheme:
fileprivate func createSession(input:AVCaptureDeviceInput) -> Bool {
let session = AVCaptureSession()
if session.canAddInput(input) {
session.addInput(input)
} else {
return false
}
let output = AVCaptureVideoDataOutput()
if session.canAddOutput(output) {
output.setSampleBufferDelegate(self, queue: DispatchQueue(label: "com.output"))
output.videoSettings = [String(kCVPixelBufferPixelFormatTypeKey):kCVPixelFormatType_32BGRA]
output.alwaysDiscardsLateVideoFrames = true
session.addOutput(output)
}
self.videoSession = session
return true
}
extension ViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
func convert(cmage:CIImage) -> UIImage
{
let context:CIContext = CIContext.init(options: nil)
let cgImage:CGImage = context.createCGImage(cmage, from: cmage.extent)!
let image:UIImage = UIImage.init(cgImage: cgImage)
return image
}
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
let threshold:Double = 1.0 / 3
let timeStamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
currentTime = Double(timeStamp.value) / Double(timeStamp.timescale)
if (currentTime - lastTime > threshold) {
if let image = imageFromSampleBuffer(sampleBuffer: sampleBuffer),
let cgImage = image.cgImage {
let ciImage = CIImage(cgImage: cgImage)
// with CIFilter
// let blackAndWhiteImage = ciImage.applyingFilter("CIColorControls", parameters: [kCIInputContrastKey: 2.5,
// kCIInputSaturationKey: 0,
// kCIInputBrightnessKey: 0.5])
// let imageToScan = convert(cmage: blackAndWhiteImage) //UIImage(ciImage: blackAndWhiteImage)
// resImage = imageToScan
// scanBarcode(cgImage: imageToScan.cgImage!)
let filter = GPUImageAverageLuminanceThresholdFilter()
filter.thresholdMultiplier = 0.7
let imageToScan = filter.image(byFilteringImage: image)
resImage = imageToScan!
scanBarcode(cgImage: imageToScan!.cgImage!)
}
}
}
fileprivate func imageFromSampleBuffer(sampleBuffer : CMSampleBuffer) -> UIImage? {
guard let imgBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return nil
}
// Lock the base address of the pixel buffer
CVPixelBufferLockBaseAddress(imgBuffer, CVPixelBufferLockFlags.readOnly)
// Get the number of bytes per row for the pixel buffer
let baseAddress = CVPixelBufferGetBaseAddress(imgBuffer)
// Get the number of bytes per row for the pixel buffer
let bytesPerRow = CVPixelBufferGetBytesPerRow(imgBuffer)
// Get the pixel buffer width and height
let width = CVPixelBufferGetWidth(imgBuffer)
let height = CVPixelBufferGetHeight(imgBuffer)
// Create a device-dependent RGB color space
let colorSpace = CGColorSpaceCreateDeviceRGB()
// Create a bitmap graphics context with the sample buffer data
var bitmapInfo: UInt32 = CGBitmapInfo.byteOrder32Little.rawValue
bitmapInfo |= CGImageAlphaInfo.premultipliedFirst.rawValue & CGBitmapInfo.alphaInfoMask.rawValue
//let bitmapInfo: UInt32 = CGBitmapInfo.alphaInfoMask.rawValue
let context = CGContext.init(data: baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo)
// Create a Quartz image from the pixel data in the bitmap graphics context
let quartzImage = context?.makeImage()
// Unlock the pixel buffer
CVPixelBufferUnlockBaseAddress(imgBuffer, CVPixelBufferLockFlags.readOnly)
CVPixelBufferLockBaseAddress(imgBuffer, .readOnly)
if var image = quartzImage {
if shouldInvert, let inverted = invertImage(image) {
image = inverted
}
let output = UIImage(cgImage: image)
return output
}
return nil
}
fileprivate func scanBarcode(cgImage: CGImage) {
let barcodeRequest = VNDetectBarcodesRequest(completionHandler: { request, _ in
self.parseResults(results: request.results)
})
let handler = VNImageRequestHandler(cgImage: cgImage, options: [.properties : ""])
guard let _ = try? handler.perform([barcodeRequest]) else {
return print("Could not scan")
}
}
fileprivate func parseResults(results: [Any]?) {
guard let results = results else {
return print("No results")
}
print("GOT results - ", results.count)
for result in results {
if let barcode = result as? VNBarcodeObservation {
if let code = barcode.payloadStringValue {
DispatchQueue.main.async {
self.videoSession?.stopRunning()
self.resultLabel.text = code
self.blackWhiteImageView.image = self.resImage //just to check from what image code scanned
}
} else {
print("No results 2")
}
} else {
print("No results 1")
}
}
}
}
I have a UIImage which I've previously created from a png file:
let strokeUIImage = UIImage(data: pngData)
I want to convert strokeImage (which has opacity) to an MTLTexture for display in an MTKView, but doing the conversion seems to perform an unwanted premultiplication, which darkens all the semitransparent edges.
My blending settings are as follows:
pipelineDescriptor.colorAttachments[0].isBlendingEnabled = true
pipelineDescriptor.colorAttachments[0].rgbBlendOperation = .add
pipelineDescriptor.colorAttachments[0].alphaBlendOperation = .add
pipelineDescriptor.colorAttachments[0].sourceRGBBlendFactor = .one
pipelineDescriptor.colorAttachments[0].sourceAlphaBlendFactor = .one
pipelineDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha
pipelineDescriptor.colorAttachments[0].destinationAlphaBlendFactor = .oneMinusSourceAlpha
I've tried two methods of conversion:
let stampTexture = try! MTKTextureLoader(device: self.device!).newTexture(cgImage: strokeUIImage.cgImage!, options: nil)
and the more elaborate dataProvider-driven method:
let image = strokeUIImage.cgImage!
let imageWidth = image.width
let imageHeight = image.height
let bytesPerPixel:Int! = 4
let rowBytes = imageWidth * bytesPerPixel
let texDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm_srgb,
width: imageWidth,
height: imageHeight,
mipmapped: false)
guard let stampTexture = device!.makeTexture(descriptor: texDescriptor) else { return }
let srcData: CFData! = image.dataProvider?.data
let pixelData = CFDataGetBytePtr(srcData)
let region = MTLRegionMake2D(0, 0, imageWidth, imageHeight)
stampTexture.replace(region: region, mipmapLevel: 0, withBytes: pixelData!, bytesPerRow: Int(rowBytes))
both of which yield the same unwanted premultiplied result.
The latter I tried, as there were some posts suggesting that the old swift3 method CGDataProviderCopyData() extracts raw pixel data from the image which is not premultiplied. Sadly, the equivalent:
let srcData: CFData! = image.dataProvider?.data
does not seem to do the trick. Am I missing something?
Any pointers would be appreciated.
After much experimenting, I've come to a solution which addresses the pre-multiplication issue inherent in CoreGraphics images. Thanks to Warren's tip regarding using an Accelerate function (vImageUnpremultiplyData_ARGB8888 in particular), I thought, why not build a CGImage using vImage_CGImageFormat which will allow me to play with the bitmapInfo setting that specifies how to interpret alpha...The result is not perfect, as demonstrated by the image attachment below:
Somehow, in the translation the alpha values are getting punched up slightly, (possibly the rgb as well, but not significantly). By the way, I should point out that the png pixel format is sRGB, and the MTKView I'm using is set to MTLPixelFormat.rgba16Float (app requirement)
Below is the full metalDrawStrokeUIImage routine I implemented. Of particular note is the line:
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.last.rawValue)
which essentially unassociates the alpha (I think) without calling vImageUnpremultiplyData_ARGB8888. Looking at the resulting image certainly looks like an un-premultiplied image...
Lastly, to get back a premultiplied texture on the MTKView side, I let the fragment shader handle the pre-multiplication:
fragment float4 premult_fragment(VertexOut interpolated [[stage_in]],
texture2d<float> texture [[texture(0)]],
sampler sampler2D [[sampler(0)]]) {
float4 sampled = texture.sample(sampler2D, interpolated.texCoord);
// this fragment shader premultiplies incoming rgb with texture's alpha
return float4(sampled.r * sampled.a,
sampled.g * sampled.a,
sampled.b * sampled.a,
sampled.a );
} // end of premult_fragment
The result is pretty close to the input source, but the image is maybe 5% more opaque than the incoming png. Again, png pixel format is sRGB, and the MTKView I'm using to display is set to MTLPixelFormat.rgba16Float . So, I'm sure something is getting mushed somewhere. If anyone has any pointers, I'd sure appreciate it.
Below is the rest of the relevant code:
func metalDrawStrokeUIImage (strokeUIImage: UIImage, strokeBbox: CGRect) {
self.metalSetupRenderPipeline(compStyle: compMode.strokeCopy) // needed so stampTexture is not modified by fragmentFunction
let bytesPerPixel = 4
let bitsPerComponent = 8
let width = Int(strokeUIImage.size.width)
let height = Int(strokeUIImage.size.height)
let rowBytes = width * bytesPerPixel
//
let texDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .rgba8Unorm_srgb,
width: width,
height: height,
mipmapped: false)
guard let stampTexture = device!.makeTexture(descriptor: texDescriptor) else { return }
//let cgImage: CGImage = strokeUIImage.cgImage!
//let sourceColorSpace = cgImage.colorSpace else {
guard
let cgImage = strokeUIImage.cgImage,
let sourceColorSpace = cgImage.colorSpace else {
print("Unable to initialize cgImage or colorSpace.")
return
}
var format = vImage_CGImageFormat(
bitsPerComponent: UInt32(cgImage.bitsPerComponent),
bitsPerPixel: UInt32(cgImage.bitsPerPixel),
colorSpace: Unmanaged.passRetained(sourceColorSpace),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.last.rawValue),
version: 0, decode: nil,
renderingIntent: CGColorRenderingIntent.defaultIntent)
var sourceBuffer = vImage_Buffer()
defer {
free(sourceBuffer.data)
}
var error = vImageBuffer_InitWithCGImage(&sourceBuffer, &format, nil, cgImage, numericCast(kvImageNoFlags))
guard error == kvImageNoError else {
print ("[MetalBrushStrokeView]: can't vImageBuffer_InitWithCGImage")
return
}
//vImagePremultiplyData_RGBA8888(&sourceBuffer, &sourceBuffer, numericCast(kvImageNoFlags))
// create a CGImage from vImage_Buffer
var destCGImage = vImageCreateCGImageFromBuffer(&sourceBuffer, &format, nil, nil, numericCast(kvImageNoFlags), &error)?.takeRetainedValue()
guard error == kvImageNoError else {
print ("[MetalBrushStrokeView]: can't vImageCreateCGImageFromBuffer")
return
}
let dstData: CFData = (destCGImage!.dataProvider!.data)!
let pixelData = CFDataGetBytePtr(dstData)
destCGImage = nil
let region = MTLRegionMake2D(0, 0, Int(width), Int(height))
stampTexture.replace(region: region, mipmapLevel: 0, withBytes: pixelData!, bytesPerRow: Int(rowBytes))
let stampColor = UIColor.white
let stampCorners = self.stampSetVerticesFromBbox(bbox: strokeBbox)
self.stampAppendToVertexBuffer(stampLayer: stampLayerMode.stampLayerFG, stampCorners: stampCorners, stampColor: stampColor)
self.metalRenderStampSingle(stampTexture: stampTexture)
self.initializeStampArray() // clears out the stamp array so we always draw 1 stamp at a time
} // end of func metalDrawStrokeUIImage (strokeUIImage: UIImage, strokeBbox: CGRect)
I have an CGImage which is constructed out of a CVPixelbuffer (ARGB). I want to convert that CGImage into a MTLTexture. I use:
let texture: MTLTexture = try m_textureLoader.newTexture(with: cgImage, options: [MTKTextureLoaderOptionSRGB : NSNumber(value: true)] )
Later I want to use the texture in an MPSImage having 3 channels:
let sid = MPSImageDescriptor(channelFormat: MPSImageFeatureChannelFormat.float16, width: 40, height: 40, featureChannels: 3)
preImage = MPSTemporaryImage(commandBuffer: commandBuffer, imageDescriptor: sid)
lanczos.encode(commandBuffer: commandBuffer, sourceTexture: texture!, destinationTexture: preImage.texture)
scale.encode (commandBuffer: commandBuffer, sourceImage: preImage, destinationImage: srcImage)
Now my questions:
How does textureLoader.newTexture(...) map the four ARGB channels to the 3 channels specified in the MPSImageDescriptor ?
How can I ensure that the RGB components are used and not e.g. ARG ?
Is there a way to specify that channel mapping ?
Thanks, Chris
Why not construct the MTLTexture from the CVPixelBuffer directly? Is much quicker!
Do this once at the beginning of your program:
// declare this somewhere, so we can re-use it
var textureCache: CVMetalTextureCache?
// create the texture cache object
guard CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &textureCache) == kCVReturnSuccess else {
print("Error: could not create a texture cache")
return false
}
Do this once your have your CVPixelBuffer:
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
var texture: CVMetalTexture?
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache,
pixelBuffer, nil, .bgra8Unorm, width, height, 0, &texture)
if let texture = texture {
metalTexture = CVMetalTextureGetTexture(texture)
}
Now metalTexture contains an MTLTexture object with the contents of the CVPixelBuffer.
I try to change the size and the resolution of an image programmatically, afterwards I save this image.
The imagesize in the imageView is changing, but when I look at my file "file3.png" it always has the original resolution of 640x1142.
I googled around but can't find a solution. I try to redraw the image. But maybe it's the wrong strategy.
thanks
#IBAction func pickOneImageBtn(sender: AnyObject) {
//load image from path
pickedImage.image = loadImageFromPath(fileInDocumentsDirectory("Angebote.png"))
let newSize = NSSize(width: 10, height: 10)
if let image = pickedImage.image {
print("found image")
//cast to CGImage
var imageRect:CGRect = CGRectMake(0, 0, image.size.width, image.size.height)
let imageRef = image.CGImageForProposedRect(&imageRect, context: nil, hints: nil)
if let imageRefExists = imageRef {
print("Cast to CGImage worked \(imageRefExists)")
}
//redraw to NSImage with new size
let imageWithNewSize = NSImage(CGImage: imageRef!, size: newSize)
//save on disk
let imgData: NSData! = imageWithNewSize.TIFFRepresentation!
let bitmap: NSBitmapImageRep! = NSBitmapImageRep(data: imgData!)
if let pngCoverImage = bitmap!.representationUsingType(NSBitmapImageFileType.NSPNGFileType, properties: [:]) {
pngCoverImage.writeToFile("/...correctpath.../imageSourceForResize/file3.png", atomically: false)
print("saved new image")
}
//the size is smaller
pickedImage.image = imageWithNewSize
}
}
Change
let imgData: NSData! = pickedImage.image!.TIFFRepresentation!
to
let imgData: NSData! = imageWithNewSize.TIFFRepresentation!
I tried to change the size of a NSImage for Mac application and here is the working function to resize an image written in swift.
func resize(image: NSImage, w: Int, h: Int) -> NSImage
{
let destSize = NSMakeSize(CGFloat(w), CGFloat(h))
let newImage = NSImage(size: destSize)
newImage.lockFocus()
image.drawInRect(NSMakeRect(0, 0, destSize.width, destSize.height), fromRect: NSZeroRect, operation: NSCompositingOperation.CompositeCopy, fraction: 1.0)
newImage.unlockFocus()
newImage.size = destSize
return NSImage(data: newImage.TIFFRepresentation!)!
}
You need to pass 3 parameters to call this function i.e NSImage, width, height and this function will return resized image.
targetimage = resize(source, w: Int(targetwidth), h: Int(targetheight))
As you know the iphone guidelines discourage loading uiimages that are greater than 1024x1024.
The size of the images that i would have to load varies, and i would like to check the size of the image i am about to load; however using the .size property of uiimage requires the image to be laoded... which is exactly what i am trying to avoid.
Is there something wrong in my reasoning or is there a solution to that?
thank you all
As of iOS 4.0, the iOS SDK includes the CGImageSource... functions (in the ImageIO framework). It's a very flexible API to query metadata without loading the image into memory. Getting the pixel dimensions of an image should work like this (make sure to include the ImageIO.framework in your target):
#import <ImageIO/ImageIO.h>
NSURL *imageFileURL = [NSURL fileURLWithPath:...];
CGImageSourceRef imageSource = CGImageSourceCreateWithURL((CFURLRef)imageFileURL, NULL);
if (imageSource == NULL) {
// Error loading image
...
return;
}
CGFloat width = 0.0f, height = 0.0f;
CFDictionaryRef imageProperties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, NULL);
CFRelease(imageSource);
if (imageProperties != NULL) {
CFNumberRef widthNum = CFDictionaryGetValue(imageProperties, kCGImagePropertyPixelWidth);
if (widthNum != NULL) {
CFNumberGetValue(widthNum, kCFNumberCGFloatType, &width);
}
CFNumberRef heightNum = CFDictionaryGetValue(imageProperties, kCGImagePropertyPixelHeight);
if (heightNum != NULL) {
CFNumberGetValue(heightNum, kCFNumberCGFloatType, &height);
}
// Check orientation and flip size if required
CFNumberRef orientationNum = CFDictionaryGetValue(imageProperties, kCGImagePropertyOrientation);
if (orientationNum != NULL) {
int orientation;
CFNumberGetValue(orientationNum, kCFNumberIntType, &orientation);
if (orientation > 4) {
CGFloat temp = width;
width = height;
height = temp;
}
}
CFRelease(imageProperties);
}
NSLog(#"Image dimensions: %.0f x %.0f px", width, height);
(adapted from "Programming with Quartz" by Gelphman and Laden, listing 9.5, page 228)
Swift 3 version of the answer:
import Foundation
import ImageIO
func sizeForImage(at url: URL) -> CGSize? {
guard let imageSource = CGImageSourceCreateWithURL(url as CFURL, nil)
, let imageProperties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil) as? [AnyHashable: Any]
, let pixelWidth = imageProperties[kCGImagePropertyPixelWidth as String]
, let pixelHeight = imageProperties[kCGImagePropertyPixelHeight as String]
, let orientationNumber = imageProperties[kCGImagePropertyOrientation as String]
else {
return nil
}
var width: CGFloat = 0, height: CGFloat = 0, orientation: Int = 0
CFNumberGetValue(pixelWidth as! CFNumber, .cgFloatType, &width)
CFNumberGetValue(pixelHeight as! CFNumber, .cgFloatType, &height)
CFNumberGetValue(orientationNumber as! CFNumber, .intType, &orientation)
// Check orientation and flip size if required
if orientation > 4 { let temp = width; width = height; height = temp }
return CGSize(width: width, height: height)
}
In Swift 5, with ImageIO,
extension URL{
var sizeOfImage: CGSize?{
guard let imageSource = CGImageSourceCreateWithURL(self as CFURL, nil)
, let imageProperties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil) as? [AnyHashable: Any]
, let pixelWidth = imageProperties[kCGImagePropertyPixelWidth as String] as! CFNumber?
, let pixelHeight = imageProperties[kCGImagePropertyPixelHeight as String] as! CFNumber?
else {
return nil
}
var width: CGFloat = 0, height: CGFloat = 0
CFNumberGetValue(pixelWidth, .cgFloatType, &width)
CFNumberGetValue(pixelHeight, .cgFloatType, &height)
}
return CGSize(width: width, height: height)
}
}
imageProperties[kCGImagePropertyOrientation as String] may be nil.
It is nil, I tested with png image file
as Apple says
kCGImagePropertyOrientation
The numeric value for this key encodes the intended display orientation for the image according to the TIFF and Exif specifications.