Vision – Face recognition performed but correct coordinates cannot be obtained - swift

Swift 5, Xcode 11, iOS 13.0.
In the code below, we will get the face from the image 'test', recognize the left eye, and display 'num' at its coordinates.
However, the coordinates of the nose are not displayed at the correct position.
I'm in trouble because I don't know the solution. I would be grateful if you could tell me.
import Vision
import Foundation
import SwiftUI
struct ContentView: View {
#ObservedObject var faceGet = test()
#State var uiimage : UIImage? = nil
var body: some View {
VStack{
if uiimage != nil {
Image(uiImage: uiimage!).resizable().scaledToFit()
}
Divider()
Button(action: {
self.uiimage = self.faceGet.faceCheck()
}){
Text("Tap image to see result")
}
}
}
}
class test :ObservableObject{
private var originalImage = UIImage(named: "test3")
func faceCheck() -> UIImage?{
var drawnImage : UIImage? = originalImage
let request = VNDetectFaceLandmarksRequest { (request, error) in
for observation in request.results as! [VNFaceObservation] {
if let landmark = observation.landmarks?.nose{
for i in 0...landmark.pointCount - 1 {
drawnImage = self.drawText(
image: drawnImage!,
point: landmark.pointsInImage(imageSize: self.originalImage!.size) [i] ,
num: i)
}
}
}
}
if let cgImage = self.originalImage?.cgImage {
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try? handler.perform([request])
}
return drawnImage
}
func drawText(image :UIImage , point:CGPoint , num:Int ) ->UIImage
{
let text = num.description
var newImage : UIImage? = nil
let fontSize = image.size.height / image.size.width * 10
let font = UIFont.boldSystemFont(ofSize: fontSize)
let textWidth = CGFloat(round(text.widthOfString(usingFont: font)))
let textHeight = CGFloat(round(text.heightOfString(usingFont: font)))
let imageRect = CGRect(origin: .zero, size: image.size)
UIGraphicsBeginImageContext(image.size);
image.draw(in: imageRect)
let rePoint :CGPoint = CGPoint(x:imageRect.maxX - CGFloat(round(point.x)),
y:imageRect.maxY - CGFloat(round(point.y)))
let textRect = CGRect(origin: rePoint, size: CGSize(width: textWidth , height: textHeight ))
let textStyle = NSMutableParagraphStyle.default.mutableCopy() as! NSMutableParagraphStyle
let textFontAttributes = [
NSAttributedString.Key.font: font,
NSAttributedString.Key.foregroundColor: UIColor.red,
NSAttributedString.Key.paragraphStyle: textStyle
]
text.draw(in: textRect, withAttributes: textFontAttributes)
newImage = UIGraphicsGetImageFromCurrentImageContext();
UIGraphicsEndImageContext()
return newImage!
}
}
extension String {
public func widthOfString(usingFont font: UIFont) -> CGFloat {
let attributes = [NSAttributedString.Key.font: font]
let size = self.size(withAttributes: attributes)
return size.width
}
public func heightOfString(usingFont font: UIFont) -> CGFloat {
let attributes = [NSAttributedString.Key.font: font]
let size = self.size(withAttributes: attributes)
return size.height
}
}

Related

CIImage pixelBuffer always return nil

I am doing some task to apply filter effect in to my WebRTC call, follow this tutorial:
https://developer.apple.com/documentation/vision/applying_matte_effects_to_people_in_images_and_video
Here is my code to convert:
func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) {
let pixelBufferr = frame.buffer as! RTCCVPixelBuffer
let pixelBufferRef = pixelBufferr.pixelBuffer
if #available(iOS 15.0, *) {
DispatchQueue.global().async {
if let output = GreetingProcessor.shared.processVideoFrame(
foreground: pixelBufferRef,
background: self.vbImage) {
print("new output: \(output) => \(output.pixelBuffer) + \(self.buffer(from: output))")
guard let px = output.pixelBuffer else { return }
let rtcPixelBuffer = RTCCVPixelBuffer(pixelBuffer: px)
let i420buffer = rtcPixelBuffer.toI420()
let newFrame = RTCVideoFrame(buffer: i420buffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs)
self.videoSource.capturer(capturer, didCapture: newFrame)
}
}
}
}
THen here is how I apply effect:
func blendImages(
background: CIImage,
foreground: CIImage,
mask: CIImage,
isRedMask: Bool = false
) -> CIImage? {
// scale mask
let maskScaleX = foreground.extent.width / mask.extent.width
let maskScaleY = foreground.extent.height / mask.extent.height
let maskScaled = mask.transformed(by: __CGAffineTransformMake(maskScaleX, 0, 0, maskScaleY, 0, 0))
// scale background
let backgroundScaleX = (foreground.extent.width / background.extent.width)
let backgroundScaleY = (foreground.extent.height / background.extent.height)
let backgroundScaled = background.transformed(
by: __CGAffineTransformMake(backgroundScaleX, 0, 0, backgroundScaleY, 0, 0))
let blendFilter = isRedMask ? CIFilter.blendWithRedMask() : CIFilter.blendWithMask()
blendFilter.inputImage = foreground
blendFilter.backgroundImage = backgroundScaled
blendFilter.maskImage = maskScaled
return blendFilter.outputImage
}
The problem is output.pixelBuffer always nil, so I can not create RTCFrame to pass it again to delegate
Can someone help?

Take snapshot from UIView with lower resolution

I'm taking snapshot from a PDFView in PDFKit for streaming (20 times per sec), and I use this extesnsion
extension UIView {
func asImageBackground(viewLayer: CALayer, viewBounds: CGRect) -> UIImage {
let renderer = UIGraphicsImageRenderer(bounds: viewBounds)
return renderer.image { rendererContext in
viewLayer.render(in: rendererContext.cgContext)
}
}
}
But the output UIImage from this extension has a high resolution which make it difficult to stream. I can reduce it by this extension
extension UIImage {
func resize(_ max_size: CGFloat) -> UIImage {
// adjust for device pixel density
let max_size_pixels = max_size / UIScreen.main.scale
// work out aspect ratio
let aspectRatio = size.width/size.height
// variables for storing calculated data
var width: CGFloat
var height: CGFloat
var newImage: UIImage
if aspectRatio > 1 {
// landscape
width = max_size_pixels
height = max_size_pixels / aspectRatio
} else {
// portrait
height = max_size_pixels
width = max_size_pixels * aspectRatio
}
// create an image renderer of the correct size
let renderer = UIGraphicsImageRenderer(size: CGSize(width: width, height: height), format: UIGraphicsImageRendererFormat.default())
// render the image
newImage = renderer.image {
(context) in
self.draw(in: CGRect(x: 0, y: 0, width: width, height: height))
}
// return the image
return newImage
}
}
but it add an additional workload which make the process even worse. Is there any better way?
Thanks
You can downsample it using ImageIO which is recommended by Apple:
extension UIImage {
func downsample(to resolution: CGSize) -> UIImage? {
let imageSourceOptions = [kCGImageSourceShouldCache: false] as CFDictionary
guard let data = self.jpegData(compressionQuality: 0.75) as? CFData, let imageSource = CGImageSourceCreateWithData(data, imageSourceOptions) else {
return nil
}
let maxDimensionInPixels = Swift.max(resolution.width, resolution.height) * 3
let downsampleOptions = [
kCGImageSourceCreateThumbnailFromImageAlways: true,
kCGImageSourceShouldCacheImmediately: true,
kCGImageSourceCreateThumbnailWithTransform: true,
kCGImageSourceThumbnailMaxPixelSize: maxDimensionInPixels
] as CFDictionary
guard let downsampledImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, downsampleOptions) else {
return nil
}
return UIImage(cgImage: downsampledImage)
}
}

How to stroke on UIImage?

I would like to save the colored shapes along with strokes but the code below (getImage()) generates unstroked shapes:
func getImage() -> UIImage {
let renderer = UIGraphicsImageRenderer(size: CGSize(width: 1024, height: 1024))
let image = renderer.image { (context) in
for key in shapeItemKeys {
let currentShape = shapeItemsByKey[key]!
UIColor.black.setStroke()
context.stroke(renderer.format.bounds)
currentShape.color.setFill()
context.cgContext.addPath(currentShape.path.bezierPath.cgPath)
context.cgContext.fillPath()
context.cgContext.strokePath()
}
}
return image
}
struct ShapeItem: Identifiable {
let id = UUID()
var color: UIColor = UIColor.white
var path: ScaledBezier
init(path: ScaledBezier) {
self.path = path
}
}
struct ScaledBezier: Shape {
let bezierPath: UIBezierPath
let sourceWidth: CGFloat
let sourceHeight: CGFloat
func path(in rect: CGRect) -> Path {
var path = Path(bezierPath.cgPath)
// Figure out how much bigger we need to make our path in order for it to fill the available space without clipping.
let multiplier = min(rect.width/sourceWidth, rect.height/sourceHeight)
// Create an affine transform that uses the multiplier for both dimensions equally.
let transform = CGAffineTransform(scaleX: multiplier, y: multiplier)
// Apply that scale and send back the result.
path.closeSubpath()
return path.applying(transform)
}
}
Does anyone know how to stroke shapes in order to be visible on UIImage?
If you want and to stroke and to fill then you need to use path for each as follows
func getImage() -> UIImage {
let renderer = UIGraphicsImageRenderer(size: CGSize(width: 1024, height: 1024))
let image = renderer.image { (context) in
for key in shapeItemKeys {
let currentShape = shapeItemsByKey[key]!
UIColor.black.setStroke()
context.stroke(renderer.format.bounds)
currentShape.color.setFill()
context.cgContext.addPath(currentShape.path.bezierPath.cgPath)
context.cgContext.strokePath()
context.cgContext.addPath(currentShape.path.bezierPath.cgPath)
context.cgContext.fillPath()
}
}
return image
}
Demo code:
struct DemoView: View {
let shapeItemKeys = [1]
let shapeItemsByKey = [1: ShapeItem(path: ScaledBezier(bezierPath: UIBezierPath(roundedRect: CGRect(x: 10, y: 10, width: 100, height: 200), cornerRadius: 20), sourceWidth: 100, sourceHeight: 200))]
var body: some View {
VStack(spacing: 0) {
Image(uiImage: getImage())
}
}
func getImage() -> UIImage {
let renderer = UIGraphicsImageRenderer(size: CGSize(width: 300, height: 300))
let image = renderer.image { (context) in
for key in shapeItemKeys {
let currentShape = shapeItemsByKey[key]!
UIColor.black.setStroke()
context.stroke(renderer.format.bounds)
currentShape.color.setFill()
context.cgContext.addPath(currentShape.path.bezierPath.cgPath)
context.cgContext.strokePath()
context.cgContext.addPath(currentShape.path.bezierPath.cgPath)
context.cgContext.fillPath()
}
}
return image
}
}

How to convert pixel dimension to CG Size in Swift?

I have large images uploaded by users in Swift and I need to resize them all to 100x100px to create thumbnails to store in my server. So far I have found that this resizes an image given a CGSize:
func resizedImage(image: UIImage, size: CGSize) -> UIImage? {
let renderer = UIGraphicsImageRenderer(size: size)
return renderer.image { (context) in
image.draw(in: CGRect(origin: .zero, size: size))
}
}
Is there any way to create a CGSize knowing that my target size is strictly 100x100px?
Got this to work:
extension UIImage {
func resizedImage(pixelSize: (width: Int, height: Int)) -> UIImage? {
var size = CGSize(width: CGFloat(pixelSize.width) / UIScreen.main.scale, height: CGFloat(pixelSize.height) / UIScreen.main.scale)
let rect = AVMakeRect(aspectRatio: self.size, insideRect: CGRect(x:0, y:0, width: size.width, height: size.height))
let renderer = UIGraphicsImageRenderer(size: size)
return renderer.image { (context) in
self.draw(in: rect)
}
}
}
You should initialize your render based on the user device scale and multiply its width and height instead of dividing it:
extension UIImage {
func aspectFitScaled(to size: CGSize) -> UIImage {
let format = imageRendererFormat
format.opaque = false
format.scale = UIScreen.main.scale
let isLandscape = self.size.width > self.size.height
let ratio = isLandscape ? size.width / self.size.width : size.height / self.size.height
let drawSize = self.size.scaled(by: ratio)
let x = (size.width - drawSize.width) / 2
let y = (size.height - drawSize.height) / 2
let origin = CGPoint(x: x, y: y)
return UIGraphicsImageRenderer(size: size, format: format).image { _ in
draw(in: CGRect(origin: origin, size: drawSize))
}
}
}
usage:
class ViewController: UIViewController {
// imageView frame is 200 x 200
#IBOutlet weak var imageView: UIImageView!
override func viewDidLoad() {
super.viewDidLoad()
// original image size is (719.0, 808.0)
let image = UIImage(data: try! Data(contentsOf: URL(string: "https://i.stack.imgur.com/Xs4RX.jpg")!))!
imageView.backgroundColor = .gray
let ivImage = image.aspectFitScaled(to: imageView.frame.size)
imageView.image = ivImage
print("ivImage.size", ivImage.size) // (200.0, 200.0)
print("ivImage.scale", ivImage.scale) // screen scale 3.0 iPhone 8 Plus
// lets check the real image dimension
let data = ivImage.jpegData(compressionQuality: 1)!
let savedSize = UIImage(data: data)!.size
print("savedSize", savedSize) // savedSize (600.0, 600.0)
}
}

NSImage Getting Resized when I draw Text on it

I have the following code to draw a text over an NSImage.
But the resulting image is getting resized to smaller one when I save it to disk.
What i'm i doing wrong? Please advice
func drawText(image :NSImage) ->NSImage
{
let text = "Sample Text"
let font = NSFont.boldSystemFont(ofSize: 18)
let imageRect = CGRect(x: 0, y: 0, width: image.size.width, height: image.size.height)
let textRect = CGRect(x: 5, y: 5, width: image.size.width - 5, height: image.size.height - 5)
let textStyle = NSMutableParagraphStyle.default().mutableCopy() as! NSMutableParagraphStyle
let textFontAttributes = [
NSFontAttributeName: font,
NSForegroundColorAttributeName: NSColor.white,
NSParagraphStyleAttributeName: textStyle
]
let im:NSImage = NSImage(size: image.size)
let rep:NSBitmapImageRep = NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: Int(image.size.width), pixelsHigh: Int(image.size.height), bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, colorSpaceName: NSCalibratedRGBColorSpace, bytesPerRow: 0, bitsPerPixel: 0)!
im.addRepresentation(rep)
im.lockFocus()
image.draw(in: imageRect)
text.draw(in: textRect, withAttributes: textFontAttributes)
im.unlockFocus()
return im
}
This is a different approach using a temporary NSView to draw the image and the text and cache the result in a new image (code is Swift 4). The benefit it you don't need to deal with pixels
class ImageView : NSView {
var image : NSImage
var text : String
init(image: NSImage, text: String)
{
self.image = image
self.text = text
super.init(frame: NSRect(origin: NSZeroPoint, size: image.size))
}
required init?(coder decoder: NSCoder) { fatalError() }
override func draw(_ dirtyRect: NSRect) {
let font = NSFont.boldSystemFont(ofSize: 18)
let textRect = CGRect(x: 5, y: 5, width: image.size.width - 5, height: image.size.height - 5)
image.draw(in: dirtyRect)
text.draw(in: textRect, withAttributes: [.font: font, .foregroundColor: NSColor.white])
}
var outputImage : NSImage {
let imageRep = bitmapImageRepForCachingDisplay(in: frame)!
cacheDisplay(in: frame, to:imageRep)
let tiffData = imageRep.tiffRepresentation!
return NSImage(data : tiffData)!
}
}
To use it, initialize a view
let image = ... // get some image
let view = ImageView(image: image, text: "Sample Text")
and get the new image
let imageWithText = view.outputImage
Note:
The paragraph style is not used at all, but if you want to create a mutable paragraph style just write
let textStyle = NSMutableParagraphStyle()
Mixed pixel vs point?
Depending on your screen 2x or 3x image is smaller 2 times or 3 times?
Here is more detailed info (scroll down to "Converting between pixels and points")
http://blog.fluidui.com/designing-for-mobile-101-pixels-points-and-resolutions/
But keep in mind that:
NSImage is resolution aware and uses a HiDPI graphics context when you lockFocus on a system with retina screen.
The image dimensions you pass to your NSBitmapImageRep initializer are in points (not pixels). An 150.0 point-wide image therefore uses 300 horizontal pixels in a #2x context.
Source:
How to save PNG file from NSImage (retina issues)
Following simple app works for me. Enjoy ;)
import Cocoa
class ViewController: NSViewController {
func save(image:NSImage, imageURL:String, format:String) -> Bool
{
let bMImg = NSBitmapImageRep(data: (image.tiffRepresentation)!)
switch format {
case ".png":
let filepath = URL(fileURLWithPath: imageURL+".png")
let dataToSave = bMImg?.representation(using: NSBitmapImageRep.FileType.png, properties: [NSBitmapImageRep.PropertyKey.compressionFactor : 1])
do
{
try dataToSave?.write(to: filepath)
return true
} catch {
return false
}
default:
return false
}
}
func draw(text:String, image:NSImage) -> NSImage
{
let font = NSFont.boldSystemFont(ofSize: 18)
let imageRect = CGRect(x: 0, y: 0, width: image.size.width, height: image.size.height)
let textRect = CGRect(x: 5, y: 5, width: image.size.width - 5, height: image.size.height - 5)
let textStyle = NSMutableParagraphStyle.default.mutableCopy() as! NSMutableParagraphStyle
let textFontAttributes = [
NSAttributedStringKey.font: font,
NSAttributedStringKey.foregroundColor: NSColor.white,
NSAttributedStringKey.paragraphStyle: textStyle
]
let im:NSImage = NSImage(size: image.size)
let rep:NSBitmapImageRep = NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: Int(image.size.width), pixelsHigh: Int(image.size.height), bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, colorSpaceName: NSColorSpaceName.calibratedRGB, bytesPerRow: 0, bitsPerPixel: 0)!
im.addRepresentation(rep)
im.lockFocus()
image.draw(in: imageRect)
text.draw(in: textRect, withAttributes: textFontAttributes)
im.unlockFocus()
return im
}
#IBAction func action(_ sender: NSButton) {
let dialog = NSOpenPanel();
dialog.title = "Choose a image...";
dialog.showsResizeIndicator = true;
dialog.showsHiddenFiles = false;
dialog.canChooseDirectories = true;
dialog.canCreateDirectories = true;
dialog.allowsMultipleSelection = false;
dialog.allowedFileTypes = ["png", "jpg"];
if (dialog.runModal() == NSApplication.ModalResponse.OK) {
guard let url = dialog.url,
let imageCIImage = CIImage(contentsOf: url) else {
return
}
let rep: NSCIImageRep = NSCIImageRep(ciImage: imageCIImage)
let nsImage = NSImage(size: rep.size)
nsImage.addRepresentation(rep)
let imageWithText = draw(text:"ABC", image: nsImage)
if (save(image: imageWithText, imageURL: "imageWithText", format: ".png")) {
print("Success")
} else {
print("ERROR:Failed to save image")
}
} else {
// User clicked on "Cancel"
return
}
}
}