I have tried the following code:
windowNumber is CGWindowID
func screenshot(imgOption:CGWindowImageOption = .bestResolution ) -> NSImage {
guard let winNumber = self.windowNumber else { return NSImage( size: NSSize(width: 1, height: 1)) }
let inf = CGFloat(FP_INFINITE)
let null = CGRect(x: inf, y: inf, width: 0, height: 0)
let cgImage = CGWindowListCreateImage(null, .optionIncludingWindow, CGWindowID(winNumber), imgOption)
//cgImage == nil
guard let cgImageUnwrapped = cgImage else { return NSImage() }
let image = NSImage(cgImage: cgImageUnwrapped, size: self.frame.value.size )
return image
}
but with the following code I have a nil result in case of window is hidden.
Related
I am doing some task to apply filter effect in to my WebRTC call, follow this tutorial:
https://developer.apple.com/documentation/vision/applying_matte_effects_to_people_in_images_and_video
Here is my code to convert:
func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) {
let pixelBufferr = frame.buffer as! RTCCVPixelBuffer
let pixelBufferRef = pixelBufferr.pixelBuffer
if #available(iOS 15.0, *) {
DispatchQueue.global().async {
if let output = GreetingProcessor.shared.processVideoFrame(
foreground: pixelBufferRef,
background: self.vbImage) {
print("new output: \(output) => \(output.pixelBuffer) + \(self.buffer(from: output))")
guard let px = output.pixelBuffer else { return }
let rtcPixelBuffer = RTCCVPixelBuffer(pixelBuffer: px)
let i420buffer = rtcPixelBuffer.toI420()
let newFrame = RTCVideoFrame(buffer: i420buffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs)
self.videoSource.capturer(capturer, didCapture: newFrame)
}
}
}
}
THen here is how I apply effect:
func blendImages(
background: CIImage,
foreground: CIImage,
mask: CIImage,
isRedMask: Bool = false
) -> CIImage? {
// scale mask
let maskScaleX = foreground.extent.width / mask.extent.width
let maskScaleY = foreground.extent.height / mask.extent.height
let maskScaled = mask.transformed(by: __CGAffineTransformMake(maskScaleX, 0, 0, maskScaleY, 0, 0))
// scale background
let backgroundScaleX = (foreground.extent.width / background.extent.width)
let backgroundScaleY = (foreground.extent.height / background.extent.height)
let backgroundScaled = background.transformed(
by: __CGAffineTransformMake(backgroundScaleX, 0, 0, backgroundScaleY, 0, 0))
let blendFilter = isRedMask ? CIFilter.blendWithRedMask() : CIFilter.blendWithMask()
blendFilter.inputImage = foreground
blendFilter.backgroundImage = backgroundScaled
blendFilter.maskImage = maskScaled
return blendFilter.outputImage
}
The problem is output.pixelBuffer always nil, so I can not create RTCFrame to pass it again to delegate
Can someone help?
Trying to add a Aztec barcode to a PDF using UIGraphicsPDFRenderer my issue is that the result is blurry thought the fix was setting interpolationQuality, Thanks for your help.
let renderer = UIGraphicsPDFRenderer(bounds: CGRect(x: 0, y: 0, width: 70.8661, height: 70.8661))
return renderer.pdfData{ ctx in
ctx.beginPage()
ctx.cgContext.interpolationQuality = .none //Doesn't do anything
let barcode = generateQRCode(from: UUID().description)
barcode.draw(in: CGRect(x: 0, y: 0, width: 70.8661, height: 70.8661))
}
func generateQRCode(from string: String) -> UIImage {
filter.message = Data(string.utf8)
if let outputImage = filter.outputImage {
if let cgimg = context.createCGImage(outputImage, from: outputImage.extent) {
return UIImage(cgImage: cgimg)
}
}
return UIImage()
}
Essentially I have the following QR Code function that successfully creates a QR code based on a given string - how can add a square image to the center of this QR code that is static no matter what string the code represents?
The following is the function I use to generate:
func generateQRCode(from string: String) -> UIImage? {
let data = string.data(using: String.Encoding.ascii)
if let filter = CIFilter(name: "CIQRCodeGenerator") {
filter.setValue(data, forKey: "inputMessage")
let transform = CGAffineTransform(scaleX: 3, y: 3)
if let output = filter.outputImage?.transformed(by: transform) {
return UIImage(ciImage: output)
}
}
return nil
}
Sample code from one of my apps, only slightly commented.
The size calculations maybe won't be required for you app.
func generateImage(code: String, size pointSize: CGSize, logo: UIImage? = nil) -> UIImage? {
let pixelScale = UIScreen.main.scale
let pixelSize = CGSize(width: pointSize.width * pixelScale, height: pointSize.height * pixelScale)
guard
let codeData = code.data(using: .isoLatin1),
let generator = CIFilter(name: "CIQRCodeGenerator")
else {
return nil
}
generator.setValue(codeData, forKey: "inputMessage")
// set higher self-correction level
generator.setValue("Q", forKey: "inputCorrectionLevel")
guard let codeImage = generator.outputImage else {
return nil
}
// calculate transform depending on required size
let transform = CGAffineTransform(
scaleX: pixelSize.width / codeImage.extent.width,
y: pixelSize.height / codeImage.extent.height
)
let scaledCodeImage = UIImage(ciImage: codeImage.transformed(by: transform), scale: 0, orientation: .up)
guard let logo = logo else {
return scaledCodeImage
}
// create a drawing buffer
UIGraphicsBeginImageContextWithOptions(pointSize, false, 0)
defer {
UIGraphicsEndImageContext()
}
// draw QR code into the buffer
scaledCodeImage.draw(in: CGRect(origin: .zero, size: pointSize))
// calculate scale to cover the central 25% of the image
let logoScaleFactor: CGFloat = 0.25
// update depending on logo width/height ratio
let logoScale = min(
pointSize.width * logoScaleFactor / logo.size.width,
pointSize.height * logoScaleFactor / logo.size.height
)
// size of the logo
let logoSize = CGSize(width: logoScale * logo.size.width, height: logoScale * logo.size.height)
// draw the logo
logo.draw(in: CGRect(
x: (pointSize.width - logoSize.width) / 2,
y: (pointSize.height - logoSize.height) / 2,
width: logoSize.width,
height: logoSize.height
))
return UIGraphicsGetImageFromCurrentImageContext()!
}
I want to generate pdf from textview after user enters the data. The textview contains paragraphs and attributed texts.
I have Followed
this procedure
and my code is:
func createPdf() {
createPDFNamed("test")
}
func getPDFPath(_ name: String) -> String {
let newPDFName = "\(name).pdf"
let paths = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)
let documentsDirectory = paths[0]
let pdfPath: String = (documentsDirectory as String).appending(newPDFName);
print(pdfPath)
return pdfPath
}
func createPDFNamed(_ name: String) {
let text = myTextView.text
let currentText: CFAttributedString = CFAttributedStringCreate(nil, (text as CFString?), nil)
if currentText != nil {
let framesetter: CTFramesetter = CTFramesetterCreateWithAttributedString(currentText)
if framesetter != nil {
let pdfFileName: String = getPDFPath(name)
UIGraphicsBeginPDFContextToFile(pdfFileName, CGRect.zero, nil)
var currentRange: CFRange! = CFRangeMake(0, 0)
var currentPage: Int = 0
var done = false
repeat {
UIGraphicsBeginPDFPageWithInfo(CGRect(x: 0, y: 0, width: 612, height: 792), nil)
currentPage += 1
drawPageNbr(currentPage)
currentRange = renderPagewithTextRange(currentRange, andFramesetter: framesetter)
if currentRange.location == CFAttributedStringGetLength((currentText as? CFAttributedString)) {
done = true
}
} while !done
UIGraphicsEndPDFContext()
}
else {
print("Could not create the framesetter needed to lay out the atrributed string.")
}
}
else {
print("Could not create the attributed string for the framesetter")
}
}
func renderPagewithTextRange(_ currentRange: CFRange, andFramesetter framesetter: CTFramesetter) -> CFRange {
var currentRange: CFRange! = CFRangeMake(0, 0)
let currentContext: CGContext? = UIGraphicsGetCurrentContext()
currentContext?.textMatrix = .identity
let frameRect = CGRect(x: 72, y: 72, width: 468, height: 648)
let framePath: CGMutablePath = CGMutablePath()
framePath.addRect(frameRect, transform: .identity)
let frameRef: CTFrame = CTFramesetterCreateFrame(framesetter, currentRange, framePath, nil)
currentContext?.translateBy(x: 0, y: 792)
currentContext?.scaleBy(x: 1.0, y: -1.0)
CTFrameDraw(frameRef, currentContext!)
currentRange = CTFrameGetVisibleStringRange(frameRef)
currentRange.location += currentRange.length
currentRange.length = 0 as? CFIndex ?? CFIndex()
return currentRange
}
func drawPageNumber(_ pageNum: Int) {
let pageString = "Page \(Int(pageNum))"
let theFont = UIFont.systemFont(ofSize: 12)
let pageStringSize: CGSize = pageString.size(withAttributes: [.font: UIFont.systemFont(ofSize: 17.0)])
let stringRect = CGRect(x: ((612.0 - pageStringSize.width) / 2.0), y: 720.0 + ((72.0 - pageStringSize.height) / 2.0), width: pageStringSize.width, height: pageStringSize.height)
let paragraphStyle = NSParagraphStyle.default.mutableCopy() as? NSMutableParagraphStyle
paragraphStyle?.lineBreakMode = .byTruncatingTail
paragraphStyle?.alignment = .right
let attributes = [NSAttributedStringKey.font: theFont, .paragraphStyle: paragraphStyle as Any] as [NSAttributedStringKey : Any]
pageString.draw(in: stringRect, withAttributes: attributes)
}
The pdf generates and never ends even it went upto 2Gb file size. But not opening. Could you please advice where I am doing mistake to finish the pdf generation? Also after pdf creation, would like to send by mail
Thanks in advance
EDIT:
If I change the code to while done it stops but prints one page only. If I change to while !done it goes on generating pdf. Past two days searching and not end up with any solution. Also the pdf prints black only. I want to reflect the attributes to pdf. Any suggestions.
if (currentRange.location == CFAttributedStringGetLength(currentText)){
done = true
}
} while done
UIGraphicsEndPDFContext()
}
else {
print("Could not create the framesetter needed to lay out the atrributed string.")
}
}
else {
print("Could not create the attributed string for the framesetter")
}
}
I am trying to get Apple's sample Core ML Models that were demoed at the 2017 WWDC to function correctly. I am using the GoogLeNet to try and classify images (see the Apple Machine Learning Page). The model takes a CVPixelBuffer as an input. I have an image called imageSample.jpg that I'm using for this demo. My code is below:
var sample = UIImage(named: "imageSample")?.cgImage
let bufferThree = getCVPixelBuffer(sample!)
let model = GoogLeNetPlaces()
guard let output = try? model.prediction(input: GoogLeNetPlacesInput.init(sceneImage: bufferThree!)) else {
fatalError("Unexpected runtime error.")
}
print(output.sceneLabel)
I am always getting the unexpected runtime error in the output rather than an image classification. My code to convert the image is below:
func getCVPixelBuffer(_ image: CGImage) -> CVPixelBuffer? {
let imageWidth = Int(image.width)
let imageHeight = Int(image.height)
let attributes : [NSObject:AnyObject] = [
kCVPixelBufferCGImageCompatibilityKey : true as AnyObject,
kCVPixelBufferCGBitmapContextCompatibilityKey : true as AnyObject
]
var pxbuffer: CVPixelBuffer? = nil
CVPixelBufferCreate(kCFAllocatorDefault,
imageWidth,
imageHeight,
kCVPixelFormatType_32ARGB,
attributes as CFDictionary?,
&pxbuffer)
if let _pxbuffer = pxbuffer {
let flags = CVPixelBufferLockFlags(rawValue: 0)
CVPixelBufferLockBaseAddress(_pxbuffer, flags)
let pxdata = CVPixelBufferGetBaseAddress(_pxbuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB();
let context = CGContext(data: pxdata,
width: imageWidth,
height: imageHeight,
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(_pxbuffer),
space: rgbColorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue)
if let _context = context {
_context.draw(image, in: CGRect.init(x: 0, y: 0, width: imageWidth, height: imageHeight))
}
else {
CVPixelBufferUnlockBaseAddress(_pxbuffer, flags);
return nil
}
CVPixelBufferUnlockBaseAddress(_pxbuffer, flags);
return _pxbuffer;
}
return nil
}
I got this code from a previous StackOverflow post (last answer here). I recognize that the code may not be correct, but I have no idea of how to do this myself. I believe that this is the section that contains the error. The model calls for the following type of input: Image<RGB,224,224>
You don't need to do a bunch of image mangling yourself to use a Core ML model with an image — the new Vision framework can do that for you.
import Vision
import CoreML
let model = try VNCoreMLModel(for: MyCoreMLGeneratedModelClass().model)
let request = VNCoreMLRequest(model: model, completionHandler: myResultsMethod)
let handler = VNImageRequestHandler(url: myImageURL)
handler.perform([request])
func myResultsMethod(request: VNRequest, error: Error?) {
guard let results = request.results as? [VNClassificationObservation]
else { fatalError("huh") }
for classification in results {
print(classification.identifier, // the scene label
classification.confidence)
}
}
The WWDC17 session on Vision should have a bit more info — it's tomorrow afternoon.
You can use a pure CoreML, but you should resize an image to (224,224)
DispatchQueue.global(qos: .userInitiated).async {
// Resnet50 expects an image 224 x 224, so we should resize and crop the source image
let inputImageSize: CGFloat = 224.0
let minLen = min(image.size.width, image.size.height)
let resizedImage = image.resize(to: CGSize(width: inputImageSize * image.size.width / minLen, height: inputImageSize * image.size.height / minLen))
let cropedToSquareImage = resizedImage.cropToSquare()
guard let pixelBuffer = cropedToSquareImage?.pixelBuffer() else {
fatalError()
}
guard let classifierOutput = try? self.classifier.prediction(image: pixelBuffer) else {
fatalError()
}
DispatchQueue.main.async {
self.title = classifierOutput.classLabel
}
}
// ...
extension UIImage {
func resize(to newSize: CGSize) -> UIImage {
UIGraphicsBeginImageContextWithOptions(CGSize(width: newSize.width, height: newSize.height), true, 1.0)
self.draw(in: CGRect(x: 0, y: 0, width: newSize.width, height: newSize.height))
let resizedImage = UIGraphicsGetImageFromCurrentImageContext()!
UIGraphicsEndImageContext()
return resizedImage
}
func cropToSquare() -> UIImage? {
guard let cgImage = self.cgImage else {
return nil
}
var imageHeight = self.size.height
var imageWidth = self.size.width
if imageHeight > imageWidth {
imageHeight = imageWidth
}
else {
imageWidth = imageHeight
}
let size = CGSize(width: imageWidth, height: imageHeight)
let x = ((CGFloat(cgImage.width) - size.width) / 2).rounded()
let y = ((CGFloat(cgImage.height) - size.height) / 2).rounded()
let cropRect = CGRect(x: x, y: y, width: size.height, height: size.width)
if let croppedCgImage = cgImage.cropping(to: cropRect) {
return UIImage(cgImage: croppedCgImage, scale: 0, orientation: self.imageOrientation)
}
return nil
}
func pixelBuffer() -> CVPixelBuffer? {
let width = self.size.width
let height = self.size.height
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue,
kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
var pixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(kCFAllocatorDefault,
Int(width),
Int(height),
kCVPixelFormatType_32ARGB,
attrs,
&pixelBuffer)
guard let resultPixelBuffer = pixelBuffer, status == kCVReturnSuccess else {
return nil
}
CVPixelBufferLockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let pixelData = CVPixelBufferGetBaseAddress(resultPixelBuffer)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(data: pixelData,
width: Int(width),
height: Int(height),
bitsPerComponent: 8,
bytesPerRow: CVPixelBufferGetBytesPerRow(resultPixelBuffer),
space: rgbColorSpace,
bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue) else {
return nil
}
context.translateBy(x: 0, y: height)
context.scaleBy(x: 1.0, y: -1.0)
UIGraphicsPushContext(context)
self.draw(in: CGRect(x: 0, y: 0, width: width, height: height))
UIGraphicsPopContext()
CVPixelBufferUnlockBaseAddress(resultPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return resultPixelBuffer
}
}
The expected image size for inputs you can find in the mimodel file:
A demo project that uses both pure CoreML and Vision variants you can find here: https://github.com/handsomecode/iOS11-Demos/tree/coreml_vision/CoreML/CoreMLDemo
If the input is UIImage, rather than an URL, and you want to use VNImageRequestHandler, you can use CIImage.
func updateClassifications(for image: UIImage) {
let orientation = CGImagePropertyOrientation(image.imageOrientation)
guard let ciImage = CIImage(image: image) else { return }
let handler = VNImageRequestHandler(ciImage: ciImage, orientation: orientation)
}
From Classifying Images with Vision and Core ML