AVCaptureSession With STT(Speech to text) - iphone

I am using AVCapture Session to record video same time i am using STT(google Speech to text api) to convert voice into text. I have facing a problem when I click on the speak button then camera get freezes. Any correct answer will be acceptable. Thanks in advance .
To start camera in
-(void)viewDidLoad;
if ([[self captureManager] setupSession]) {
// Create video preview layer and add it to the UI
AVCaptureVideoPreviewLayer *newCaptureVideoPreviewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:[[self captureManager] session]];
UIView *view = [self videoPreviewView];
CALayer *viewLayer = [view layer];
[viewLayer setMasksToBounds:YES];
CGRect bounds = [view bounds];
[newCaptureVideoPreviewLayer setFrame:bounds];
if ([newCaptureVideoPreviewLayer isOrientationSupported]) {
[newCaptureVideoPreviewLayer setOrientation:AVCaptureVideoOrientationLandscapeLeft|AVCaptureVideoOrientationLandscapeRight];
}
[newCaptureVideoPreviewLayer setVideoGravity:AVLayerVideoGravityResizeAspectFill];
[viewLayer insertSublayer:newCaptureVideoPreviewLayer below:[[viewLayer sublayers] objectAtIndex:0]];
[self setCaptureVideoPreviewLayer:newCaptureVideoPreviewLayer];
[newCaptureVideoPreviewLayer release];
// Start the session. This is done asychronously since -startRunning doesn't return until the session is running.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[[[self captureManager] session] startRunning];
});
[self updateButtonStates];
}
- (BOOL) setupSession
{
BOOL success = NO;
// Set torch and flash mode to auto
if ([[self backFacingCamera] hasFlash]) {
if ([[self backFacingCamera] lockForConfiguration:nil]) {
if ([[self backFacingCamera] isFlashModeSupported:AVCaptureFlashModeAuto]) {
[[self backFacingCamera] setFlashMode:AVCaptureFlashModeAuto];
}
[[self backFacingCamera] unlockForConfiguration];
}
}
if ([[self backFacingCamera] hasTorch]) {
if ([[self backFacingCamera] lockForConfiguration:nil]) {
if ([[self backFacingCamera] isTorchModeSupported:AVCaptureTorchModeAuto]) {
[[self backFacingCamera] setTorchMode:AVCaptureTorchModeAuto];
}
[[self backFacingCamera] unlockForConfiguration];
}
}
// Init the device inputs
AVCaptureDeviceInput *newVideoInput = [[AVCaptureDeviceInput alloc] initWithDevice:[self frontFacingCamera] error:nil];
AVCaptureDeviceInput *newAudioInput = [[AVCaptureDeviceInput alloc] initWithDevice:[self audioDevice] error:nil];
// Setup the still image file output
AVCaptureStillImageOutput *newStillImageOutput = [[AVCaptureStillImageOutput alloc] init];
NSDictionary *outputSettings = [[NSDictionary alloc] initWithObjectsAndKeys:
AVVideoCodecJPEG, AVVideoCodecKey,
nil];
[newStillImageOutput setOutputSettings:outputSettings];
[outputSettings release];
// Create session (use default AVCaptureSessionPresetHigh)
AVCaptureSession *newCaptureSession = [[AVCaptureSession alloc] init];
// Add inputs and output to the capture session
if ([newCaptureSession canAddInput:newVideoInput]) {
[newCaptureSession addInput:newVideoInput];
}
if ([newCaptureSession canAddInput:newAudioInput]) {
[newCaptureSession addInput:newAudioInput];
}
if ([newCaptureSession canAddOutput:newStillImageOutput]) {
[newCaptureSession addOutput:newStillImageOutput];
}
[self setStillImageOutput:newStillImageOutput];
[self setVideoInput:newVideoInput];
[self setAudioInput:newAudioInput];
[self setSession:newCaptureSession];
[newStillImageOutput release];
[newVideoInput release];
[newAudioInput release];
[newCaptureSession release];
// Set up the movie file output
NSURL *outputFileURL = [self tempFileURL];
AVCamRecorder *newRecorder = [[AVCamRecorder alloc] initWithSession:[self session] outputFileURL:outputFileURL];
[newRecorder setDelegate:self];
// Send an error to the delegate if video recording is unavailable
if (![newRecorder recordsVideo] && [newRecorder recordsAudio]) {
NSString *localizedDescription = NSLocalizedString(#"Video recording unavailable", #"Video recording unavailable description");
NSString *localizedFailureReason = NSLocalizedString(#"Movies recorded on this device will only contain audio. They will be accessible through iTunes file sharing.", #"Video recording unavailable failure reason");
NSDictionary *errorDict = [NSDictionary dictionaryWithObjectsAndKeys:
localizedDescription, NSLocalizedDescriptionKey,
localizedFailureReason, NSLocalizedFailureReasonErrorKey,
nil];
NSError *noVideoError = [NSError errorWithDomain:#"AVCam" code:0 userInfo:errorDict];
if ([[self delegate] respondsToSelector:#selector(captureManager:didFailWithError:)]) {
[[self delegate] captureManager:self didFailWithError:noVideoError];
}
}
[self setRecorder:newRecorder];
[newRecorder release];
success = YES;
return success;
}

Related

cannot change AVCaptureConnection orientation

I want to change the video orientation from front camera,
So I choose the previewLayer.orientation,and it indeed work .
Since the method is deprecated from IOS6 and I get a warning to use previewLayer.connection.videoOrientation
but I cannot access the connection property from previewLayer
- (void)addDeviceInput {
[session beginConfiguration];
[session removeInput:videoInput];
[self setVideoInput:nil];
AVCaptureDeviceInput *newDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:[self frontFacingCamera] error:nil];
if ([session canAddInput:newDeviceInput]) {
[session addInput:newDeviceInput];
[self setVideoInput:newDeviceInput];
}
newDeviceInput = nil;
if (previewLayer) {
[previewLayer removeFromSuperlayer];
[previewLayer release];
previewLayer = nil;
}
[session commitConfiguration];
}
- (id)setupCameraSession {
if (!(self = [super init])) {
return nil;
}
self.frameRate = 0;
session = [[AVCaptureSession alloc] init];
[session beginConfiguration];
[session setSessionPreset:AVCaptureSessionPreset352x288];
[self addDeviceInput]; //invoke previous method
AVCaptureVideoDataOutput * newVideoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
newVideoDataOutput.videoSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithInt:kCVPixelFormatType_32BGRA],
kCVPixelBufferPixelFormatTypeKey,nil];
newVideoDataOutput.alwaysDiscardsLateVideoFrames = YES;
if ([session canAddOutput:newVideoDataOutput]) {
[session addOutput:newVideoDataOutput];
[self setVideoOutput:newVideoDataOutput];
}
[newVideoDataOutput release];
self.frameRate = VIDEO_FRAME_RATE;
[session commitConfiguration];
AVCaptureVideoPreviewLayer * newPreviewLayer =
[[AVCaptureVideoPreviewLayer alloc] initWithSession:session];
//[newPreviewLayer setSession:session]; //it doesn't work? and get error
[self setPreviewLayer:newPreviewLayer];
[newPreviewLayer release];
[session startRunning];
return self;
}
I get the error
error: Execution was interrupted, reason: Attempted to dereference an invalid ObjC Object or send it an unrecognized selector.

Camera freezes upon switching to the front facing camera

My problem
I am using AVFoundation to capture the frames of the video output every 5 seconds. When the user taps a button, the input camera should switch from the front one to the back one and the opposite. Problem is, every time I switch the back camera to the front camera it freezes (oddly enough, it works the other way around - meaning, front camera to back camera)!
The code I use
To switch between the cameras, I use the exact code taken from Apple's AVCam sample code with a slight change of variable names (so it will match my code):
- (BOOL)toggleCamera
{
BOOL success = NO;
if ([self cameraCount] > 1) {
NSError *error;
AVCaptureDeviceInput *newVideoInput;
AVCaptureDevicePosition position = [[self.videoInput device] position];
if (position == AVCaptureDevicePositionBack)
newVideoInput = [[AVCaptureDeviceInput alloc] initWithDevice:[self frontFacingCamera] error:&error];
else if (position == AVCaptureDevicePositionFront)
newVideoInput = [[AVCaptureDeviceInput alloc] initWithDevice:[self backFacingCamera] error:&error];
else
goto bail;
if (newVideoInput != nil) {
// Start configuring the session.
[[self captureSession] beginConfiguration];
// Remove the current video input device.
[[self captureSession] removeInput:[self videoInput]];
if ([[self captureSession] canAddInput:newVideoInput]) {
[[self captureSession] addInput:newVideoInput];
[self setVideoInput:newVideoInput];
}
else {
[[self captureSession] addInput:[self videoInput]];
}
[[self captureSession] commitConfiguration];
success = YES;
[newVideoInput release];
}
else if (error) {
NSLog(#"PICTURE TAKER: Failed to toggle cameras."
#"\nError: %#", error.localizedDescription);
}
}
bail:
return success;
}
And to setup the AVCaptureSession, again, I use the exact same code from the AVCam sample code, with a slight change in the added output, (to match my needs):
- (BOOL) setupSession
{
BOOL success = NO;
// Set torch and flash mode to auto
if ([[self backFacingCamera] hasFlash]) {
if ([[self backFacingCamera] lockForConfiguration:nil]) {
if ([[self backFacingCamera] isFlashModeSupported:AVCaptureFlashModeAuto]) {
[[self backFacingCamera] setFlashMode:AVCaptureFlashModeAuto];
}
[[self backFacingCamera] unlockForConfiguration];
}
}
if ([[self backFacingCamera] hasTorch]) {
if ([[self backFacingCamera] lockForConfiguration:nil]) {
if ([[self backFacingCamera] isTorchModeSupported:AVCaptureTorchModeAuto]) {
[[self backFacingCamera] setTorchMode:AVCaptureTorchModeAuto];
}
[[self backFacingCamera] unlockForConfiguration];
}
}
// Init the device inputs
AVCaptureDeviceInput *newVideoInput = [[AVCaptureDeviceInput alloc] initWithDevice:[self frontFacingCamera] error:nil];
// Create a video output & configure it.
AVCaptureVideoDataOutput *newVideoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
newVideoDataOutput.videoSettings = #{(NSString *)kCVPixelBufferPixelFormatTypeKey : #(kCVPixelFormatType_32BGRA)};
// Set the output's delegate.
dispatch_queue_t queue = dispatch_queue_create("MyQueue", NULL);
[newVideoDataOutput setSampleBufferDelegate:self queue:queue];
dispatch_release(queue);
// Create session (use default AVCaptureSessionPresetHigh)
AVCaptureSession *newCaptureSession = [[AVCaptureSession alloc] init];
// Add inputs and output to the capture session
if ([newCaptureSession canAddInput:newVideoInput]) {
[newCaptureSession addInput:newVideoInput];
}
if ([newCaptureSession canAddOutput:newVideoDataOutput]) {
[newCaptureSession addOutput:newVideoDataOutput];
}
[self setVideoOutput:newVideoDataOutput];
[self setVideoInput:newVideoInput];
[self setCaptureSession:newCaptureSession];
[newVideoDataOutput release];
[newVideoInput release];
[newCaptureSession release];
// Get our view's layer.
CALayer *viewLayer = self.view.layer;
// Add the the session's preview layer.
AVCaptureVideoPreviewLayer *captureVideoPreviewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:self.captureSession];
captureVideoPreviewLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
captureVideoPreviewLayer.frame = self.view.bounds;
[viewLayer insertSublayer:captureVideoPreviewLayer below:self.imageViewOverlay.layer];
[captureVideoPreviewLayer release];
success = YES;
return success;
}
*Important note: the camera freezes right after the call to the output's delegate.
Can anyone please help me solve this? It seems as if I had tried almost everything!
Update #1
As requested, self is referring to a view controllers that presented the session's preview layer and manages all things related to the camera. In addition, I'm posting the code the handles the frontFacingCamera part, which is (just like all of the code that I have posted so far) taken from Apple's AVCam sample code:
// Find a camera with the specificed AVCaptureDevicePosition, returning nil if one is not found
- (AVCaptureDevice *) cameraWithPosition:(AVCaptureDevicePosition) position
{
NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
for (AVCaptureDevice *device in devices) {
if ([device position] == position) {
return device;
}
}
return nil;
}
// Find a front facing camera, returning nil if one is not found
- (AVCaptureDevice *) frontFacingCamera
{
return [self cameraWithPosition:AVCaptureDevicePositionFront];
}
BOOL isUsingFrontFacingCamera;
- (BOOL) swapCameras
{
if ([self cameraCount] > 1) {
AVCaptureDevicePosition desiredPosition;
if (isUsingFrontFacingCamera) {
desiredPosition = AVCaptureDevicePositionBack;
} else {
desiredPosition = AVCaptureDevicePositionFront;
}
for (AVCaptureDevice *d in [AVCaptureDevice devicesWithMediaType: AVMediaTypeVideo]) {
if ([d position] == desiredPosition) {
[[self session] beginConfiguration];
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:d error:nil];
for (AVCaptureInput *oldInput in [[self session] inputs]) {
[[self session] removeInput:oldInput];
}
[[self session] addInput:input];
[[self session] commitConfiguration];
break;
}
}
isUsingFrontFacingCamera = !isUsingFrontFacingCamera;
return YES;
}
return NO;
}
Answer from here
EDITED : This from Square Cam Apple sample code. Works fine. Tested also. http://developer.apple.com/library/ios/#samplecode/SquareCam/Listings/SquareCam_SqareCamViewController_m.html
// use front/back camera
- (IBAction)switchCameras:(id)sender
{
AVCaptureDevicePosition desiredPosition;
if (isUsingFrontFacingCamera)
desiredPosition = AVCaptureDevicePositionBack;
else
desiredPosition = AVCaptureDevicePositionFront;
for (AVCaptureDevice *d in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) {
if ([d position] == desiredPosition) {
[[previewLayer session] beginConfiguration];
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:d error:nil];
for (AVCaptureInput *oldInput in [[previewLayer session] inputs]) {
[[previewLayer session] removeInput:oldInput];
}
[[previewLayer session] addInput:input];
[[previewLayer session] commitConfiguration];
break;
}
}
isUsingFrontFacingCamera = !isUsingFrontFacingCamera;
}

How to capture image without displaying preview in iOS

I want to capture images at specific instances, for example when a button is pushed; but I don't want to show any video preview screen. I guess captureStillImageAsynchronouslyFromConnection is what I need to use for this scenario. Currently, I can capture image if I show a video preview. However, if I remove the code to show the preview, the app crashes with the following output:
2012-04-07 11:25:54.898 imCapWOPreview[748:707] *** Terminating app
due to uncaught exception 'NSInvalidArgumentException', reason: '***
-[AVCaptureStillImageOutput captureStillImageAsynchronouslyFromConnection:completionHandler:] -
inactive/invalid connection passed.'
*** First throw call stack: (0x336ee8bf 0x301e21e5 0x3697c35d 0x34187 0x33648435 0x310949eb 0x310949a7 0x31094985 0x310946f5 0x3109502d
0x3109350f 0x31092f01 0x310794ed 0x31078d2d 0x37db7df3 0x336c2553
0x336c24f5 0x336c1343 0x336444dd 0x336443a5 0x37db6fcd 0x310a7743
0x33887 0x3382c) terminate called throwing an exception(lldb)
So here is my implementation:
BIDViewController.h:
#import <UIKit/UIKit.h>
#import <AVFoundation/AVFoundation.h>
#interface BIDViewController : UIViewController
{
AVCaptureStillImageOutput *stillImageOutput;
}
#property (strong, nonatomic) IBOutlet UIView *videoPreview;
- (IBAction)doCap:(id)sender;
#end
Relevant staff inside BIDViewController.m :
#import "BIDViewController.h"
#interface BIDViewController ()
#end
#implementation BIDViewController
#synthesize capturedIm;
#synthesize videoPreview;
- (void)viewDidLoad
{
[super viewDidLoad];
[self setupAVCapture];
}
- (BOOL)setupAVCapture
{
NSError *error = nil;
AVCaptureSession *session = [AVCaptureSession new];
[session setSessionPreset:AVCaptureSessionPresetHigh];
/*
AVCaptureVideoPreviewLayer *captureVideoPreviewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:session];
captureVideoPreviewLayer.frame = self.videoPreview.bounds;
[self.videoPreview.layer addSublayer:captureVideoPreviewLayer];
*/
// Select a video device, make an input
AVCaptureDevice *backCamera = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:backCamera error:&error];
if (error)
return NO;
if ([session canAddInput:input])
[session addInput:input];
// Make a still image output
stillImageOutput = [AVCaptureStillImageOutput new];
NSDictionary *outputSettings = [[NSDictionary alloc] initWithObjectsAndKeys: AVVideoCodecJPEG, AVVideoCodecKey, nil];
[stillImageOutput setOutputSettings:outputSettings];
if ([session canAddOutput:stillImageOutput])
[session addOutput:stillImageOutput];
[session startRunning];
return YES;
}
- (IBAction)doCap:(id)sender {
AVCaptureConnection *videoConnection = nil;
for (AVCaptureConnection *connection in stillImageOutput.connections)
{
for (AVCaptureInputPort *port in [connection inputPorts])
{
if ([[port mediaType] isEqual:AVMediaTypeVideo] )
{
videoConnection = connection;
break;
}
}
if (videoConnection) { break; }
}
[stillImageOutput captureStillImageAsynchronouslyFromConnection:videoConnection
completionHandler:^(CMSampleBufferRef imageDataSampleBuffer, NSError *__strong error) {
// Do something with the captured image
}];
}
With the above code, if doCap is called, then the crash occurs. On the other hand, if I remove the following comments in setupAVCapture function
/*
AVCaptureVideoPreviewLayer *captureVideoPreviewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:session];
captureVideoPreviewLayer.frame = self.videoPreview.bounds;
[self.videoPreview.layer addSublayer:captureVideoPreviewLayer];
*/
then it works without any problem.
In summary, my questions is, How can I capture images at controlled instances without showing preview ?
I use the following code for capturing from front facing camera (if available) or using the back camera. Works well on my iPhone 4S.
-(void)viewDidLoad{
AVCaptureSession *session = [[AVCaptureSession alloc] init];
session.sessionPreset = AVCaptureSessionPresetMedium;
AVCaptureDevice *device = [self frontFacingCameraIfAvailable];
NSError *error = nil;
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
if (!input) {
// Handle the error appropriately.
NSLog(#"ERROR: trying to open camera: %#", error);
}
[session addInput:input];
//stillImageOutput is a global variable in .h file: "AVCaptureStillImageOutput *stillImageOutput;"
stillImageOutput = [[AVCaptureStillImageOutput alloc] init];
NSDictionary *outputSettings = [[NSDictionary alloc] initWithObjectsAndKeys: AVVideoCodecJPEG, AVVideoCodecKey, nil];
[stillImageOutput setOutputSettings:outputSettings];
[session addOutput:stillImageOutput];
[session startRunning];
}
-(AVCaptureDevice *)frontFacingCameraIfAvailable{
NSArray *videoDevices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
AVCaptureDevice *captureDevice = nil;
for (AVCaptureDevice *device in videoDevices){
if (device.position == AVCaptureDevicePositionFront){
captureDevice = device;
break;
}
}
// couldn't find one on the front, so just get the default video device.
if (!captureDevice){
captureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
}
return captureDevice;
}
-(IBAction)captureNow{
AVCaptureConnection *videoConnection = nil;
for (AVCaptureConnection *connection in stillImageOutput.connections){
for (AVCaptureInputPort *port in [connection inputPorts]){
if ([[port mediaType] isEqual:AVMediaTypeVideo]){
videoConnection = connection;
break;
}
}
if (videoConnection) {
break;
}
}
NSLog(#"about to request a capture from: %#", stillImageOutput);
[stillImageOutput captureStillImageAsynchronouslyFromConnection:videoConnection completionHandler: ^(CMSampleBufferRef imageSampleBuffer, NSError *error){
CFDictionaryRef exifAttachments = CMGetAttachment( imageSampleBuffer, kCGImagePropertyExifDictionary, NULL);
if (exifAttachments){
// Do something with the attachments if you want to.
NSLog(#"attachements: %#", exifAttachments);
}
else
NSLog(#"no attachments");
NSData *imageData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageSampleBuffer];
UIImage *image = [[UIImage alloc] initWithData:imageData];
self.vImage.image = image;
}];
}
Well, i was facing a similar issue where by the captureStillImageAsynchronouslyFromConnection:stillImageConnection was raising an exception that the passed connection is invalid. Later on, i figured out that when i made properties for the session and stillImageOutPut to retain values, the issue got resolved.
I am a javascript developer. Wanted to create an iOS native framework for my cross-platform Javascript project
When I started to do the same, I faced many issues with methods deprecated and other runtime errors.
After fixing all the issues, below is the answer which is compliant with iOS 13.5
The code helps you to take a picture on a button click without a preview.
Your .h file
#interface NoPreviewCameraViewController : UIViewController <AVCapturePhotoCaptureDelegate> {
AVCaptureSession *captureSession;
AVCapturePhotoOutput *photoOutput;
AVCapturePhotoSettings *photoSetting;
AVCaptureConnection *captureConnection;
UIImageView *imageView;
}
#end
Your .m file
- (void)viewDidLoad {
[super viewDidLoad];
imageView = [[UIImageView alloc] initWithFrame:CGRectMake(0, 0, self.view.frame.size.width, self.view.frame.size.height - 140)];
[self.view addSubview:imageView];
UIButton *takePicture = [UIButton buttonWithType:UIButtonTypeCustom];
[takePicture addTarget:self action:#selector(takePicture:) forControlEvents:UIControlEventTouchUpInside];
[takePicture setTitle:#"Take Picture" forState:UIControlStateNormal];
takePicture.frame = CGRectMake(40.0, self.view.frame.size.height - 140, self.view.frame.size.width - 40, 40);
[self.view addSubview:takePicture];
[self initCaptureSession];
}
- (void) initCaptureSession {
captureSession = [[AVCaptureSession alloc] init];
if([captureSession canSetSessionPreset: AVCaptureSessionPresetPhoto] ) {
[captureSession setSessionPreset:AVCaptureSessionPresetPhoto];
}
AVCaptureDeviceInput *deviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:[AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo] error:nil];
if ([captureSession canAddInput:deviceInput]) {
[captureSession addInput:deviceInput];
}
photoOutput = [[AVCapturePhotoOutput alloc] init];
if ([captureSession canAddOutput:photoOutput]) {
[captureSession addOutput:photoOutput];
}
[captureSession startRunning];
}
-(void) setNewPhotoSetting {
photoSetting = [AVCapturePhotoSettings photoSettingsWithFormat:#{AVVideoCodecKey : AVVideoCodecTypeJPEG}];
[photoOutput setPhotoSettingsForSceneMonitoring:photoSetting];
}
- (IBAction)takePicture:(id)sender {
captureConnection = nil;
[self setNewPhotoSetting];
for (AVCaptureConnection *connection in photoOutput.connections) {
for (AVCaptureInputPort *port in [connection inputPorts]) {
if ([[port mediaType] isEqual: AVMediaTypeVideo]) {
captureConnection = connection;
NSLog(#"Value of connection = %#", connection);
NSLog(#"Value of captureConnection = %#", captureConnection);
break;
}
}
if (captureConnection) {
break;
}
}
[photoOutput capturePhotoWithSettings:photoSetting delegate:self];
}

How to detect faces within a video session with iOS 5

Within my App I'm using a videosession where I can take pictures with. However I'd like to have face detection within this videosession. I looked at Apple's example "SquareCam" which is exactly what I'm looking for. However implementing their code in my project is driving me bonkers.
#import "CaptureSessionManager.h"
#import <ImageIO/ImageIO.h>
#implementation CaptureSessionManager
#synthesize captureSession;
#synthesize previewLayer;
#synthesize stillImageOutput;
#synthesize stillImage;
#pragma mark Capture Session Configuration
- (id)init {
if ((self = [super init])) {
[self setCaptureSession:[[AVCaptureSession alloc] init]];
}
return self;
}
- (void)didReceiveMemoryWarning
{
// Releases the view if it doesn't have a superview.
NSLog(#"memorywarning");
// Release any cached data, images, etc that aren't in use.
}
- (void)addVideoPreviewLayer {
[self setPreviewLayer:[[[AVCaptureVideoPreviewLayer alloc] initWithSession:[self captureSession]] autorelease]];
[[self previewLayer] setVideoGravity:AVLayerVideoGravityResizeAspectFill];
}
- (void)addVideoInput {
AVCaptureDevice *videoDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
if ([videoDevice isFocusModeSupported:AVCaptureFocusModeLocked]) {
NSError *error = nil;
if ([videoDevice lockForConfiguration:&error]) {
NSLog(#"focus");
videoDevice.focusMode = AVCaptureFocusModeLocked;
videoDevice.focusMode = AVCaptureFocusModeContinuousAutoFocus;
//videoDevice.focusMode = AVCaptureSessionPresetPhoto
videoDevice.whiteBalanceMode = AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance;
[captureSession setSessionPreset:AVCaptureSessionPresetPhoto];
[videoDevice unlockForConfiguration];
}
else {
}
}
// Respond to the failure as appropriate.
if (videoDevice) {
NSError *error;
AVCaptureDeviceInput *videoIn = [AVCaptureDeviceInput deviceInputWithDevice:videoDevice error:&error];
if (!error) {
if ([[self captureSession] canAddInput:videoIn])
[[self captureSession] addInput:videoIn];
else
NSLog(#"Couldn't add video input");
}
else
NSLog(#"Couldn't create video input");
}
else
NSLog(#"Couldn't create video capture device");
}
- (void)addStillImageOutput
{
[self setStillImageOutput:[[[AVCaptureStillImageOutput alloc] init] autorelease]];
NSDictionary *outputSettings = [[NSDictionary alloc] initWithObjectsAndKeys:AVVideoCodecJPEG,AVVideoCodecKey,nil];
[[self stillImageOutput] setOutputSettings:outputSettings];
AVCaptureConnection *videoConnection = nil;
for (AVCaptureConnection *connection in [[self stillImageOutput] connections]) {
for (AVCaptureInputPort *port in [connection inputPorts]) {
if ([[port mediaType] isEqual:AVMediaTypeVideo] ) {
videoConnection = connection;
break;
}
}
if (videoConnection) {
break;
}
}
[[self captureSession] addOutput:[self stillImageOutput]];
[outputSettings release];
}
- (void)captureStillImage
{
AVCaptureConnection *videoConnection = nil;
for (AVCaptureConnection *connection in [[self stillImageOutput] connections]) {
for (AVCaptureInputPort *port in [connection inputPorts]) {
if ([[port mediaType] isEqual:AVMediaTypeVideo]) {
videoConnection = connection;
break;
}
}
if (videoConnection) {
break;
}
}
NSLog(#"about to request a capture from: %#", [self stillImageOutput]);
[[self stillImageOutput] captureStillImageAsynchronouslyFromConnection:videoConnection
completionHandler:^(CMSampleBufferRef imageSampleBuffer, NSError *error) {
CFDictionaryRef exifAttachments = CMGetAttachment(imageSampleBuffer, kCGImagePropertyExifDictionary, NULL);
if (exifAttachments) {
NSLog(#"attachements: %#", exifAttachments);
} else {
NSLog(#"no attachments");
}
NSData *imageData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageSampleBuffer];
UIImage *image = [[UIImage alloc] initWithData:imageData];
[self setStillImage:image];
[image release];
[[NSNotificationCenter defaultCenter] postNotificationName:kImageCapturedSuccessfully object:nil];
}];
}
- (void)dealloc {
[[self captureSession] stopRunning];
[previewLayer release], previewLayer = nil;
[captureSession release], captureSession = nil;
[stillImageOutput release], stillImageOutput = nil;
[stillImage release], stillImage = nil;
[super dealloc];
}
#end
Besides my videosession I did succeeded with recognizing faces within a UIImage which I imported inside my project. I did this with the example of #Abhinav Jha (How to properly instantiate CIDetector class object in iOS 5 face detection API).
CIImage *ciImage = [[CIImage alloc] initWithImage:[UIImage imageNamed:#"Photo.JPG"]];
if (ciImage == nil)
[imageView setImage:[UIImage imageNamed:#"Photo.JPG"]];
NSDictionary *options = [[NSDictionary alloc] initWithObjectsAndKeys:
#"CIDetectorAccuracy", #"CIDetectorAccuracyHigh",nil];
CIDetector *ciDetector = [CIDetector detectorOfType:CIDetectorTypeFace
context:nil
options:options];
NSArray *features = [ciDetector featuresInImage:ciImage];
NSLog(#"no of face detected: %d", [features count]);
Hope someone can point me in the right direction with combining the two examples!

AVAssetWritter does not work with audio

I'm trying to get audio to work with the video for an iOS application. The video is fine. No audio is recorded to the file (My iPhone speaker works.)
Here's the init setup:
session = [[AVCaptureSession alloc] init];
menu->session = session;
menu_open = NO;
session.sessionPreset = AVCaptureSessionPresetMedium;
camera = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
microphone = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
menu->camera = camera;
[session beginConfiguration];
[camera lockForConfiguration:nil];
if([camera isExposureModeSupported:AVCaptureExposureModeContinuousAutoExposure]){
camera.exposureMode = AVCaptureExposureModeContinuousAutoExposure;
}
if([camera isFocusModeSupported:AVCaptureFocusModeContinuousAutoFocus]){
camera.focusMode = AVCaptureFocusModeContinuousAutoFocus;
}
if([camera isWhiteBalanceModeSupported:AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance]){
camera.whiteBalanceMode = AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance;
}
if ([camera hasTorch]) {
if([camera isTorchModeSupported:AVCaptureTorchModeOn]){
[camera setTorchMode:AVCaptureTorchModeOn];
}
}
[camera unlockForConfiguration];
[session commitConfiguration];
AVCaptureDeviceInput * camera_input = [AVCaptureDeviceInput deviceInputWithDevice:camera error:nil];
[session addInput:camera_input];
microphone_input = [[AVCaptureDeviceInput deviceInputWithDevice:microphone error:nil] retain];
AVCaptureVideoDataOutput * output = [[[AVCaptureVideoDataOutput alloc] init] autorelease];
output.videoSettings = [NSDictionary dictionaryWithObject: [NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey];
[session addOutput:output];
output.minFrameDuration = CMTimeMake(1,30);
dispatch_queue_t queue = dispatch_queue_create("MY QUEUE", NULL);
[output setSampleBufferDelegate:self queue:queue];
dispatch_release(queue);
audio_output = [[[AVCaptureAudioDataOutput alloc] init] retain];
queue = dispatch_queue_create("MY QUEUE", NULL);
AudioOutputBufferDelegate * special_delegate = [[[AudioOutputBufferDelegate alloc] init] autorelease];
special_delegate->normal_delegate = self;
[special_delegate retain];
[audio_output setSampleBufferDelegate:special_delegate queue:queue];
dispatch_release(queue);
[session startRunning];
Here is the beginning and end of recording:
if (recording) { //Hence stop recording
[video_button setTitle:#"Video" forState: UIControlStateNormal];
recording = NO;
[writer_input markAsFinished];
[audio_writer_input markAsFinished];
[video_writer endSessionAtSourceTime: CMTimeMakeWithSeconds([[NSDate date] timeIntervalSinceDate: start_time],30)];
[video_writer finishWriting];
UISaveVideoAtPathToSavedPhotosAlbum(temp_url,self,#selector(video:didFinishSavingWithError:contextInfo:),nil);
[start_time release];
[temp_url release];
[av_adaptor release];
[microphone lockForConfiguration:nil];
[session beginConfiguration];
[session removeInput:microphone_input];
[session removeOutput:audio_output];
[session commitConfiguration];
[microphone unlockForConfiguration];
[menu restateConfigiration];
[vid_off play];
}else{ //Start recording
[vid_on play];
[microphone lockForConfiguration:nil];
[session beginConfiguration];
[session addInput:microphone_input];
[session addOutput:audio_output];
[session commitConfiguration];
[microphone unlockForConfiguration];
[menu restateConfigiration];
[video_button setTitle:#"Stop" forState: UIControlStateNormal];
recording = YES;
NSError *error = nil;
NSFileManager * file_manager = [[NSFileManager alloc] init];
temp_url = [[NSString alloc] initWithFormat:#"%#/%#", NSTemporaryDirectory(), #"temp.mp4"];
[file_manager removeItemAtPath: temp_url error:NULL];
[file_manager release];
video_writer = [[AVAssetWriter alloc] initWithURL: [NSURL fileURLWithPath:temp_url] fileType: AVFileTypeMPEG4 error: &error];
NSDictionary *video_settings = [NSDictionary dictionaryWithObjectsAndKeys: AVVideoCodecH264, AVVideoCodecKey,[NSNumber numberWithInt:360], AVVideoWidthKey,[NSNumber numberWithInt:480], AVVideoHeightKey,nil];
writer_input = [[AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:video_settings] retain];
AudioChannelLayout acl;
bzero( &acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
audio_writer_input = [[AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings: [NSDictionary dictionaryWithObjectsAndKeys: [NSNumber numberWithInt: kAudioFormatMPEG4AAC], AVFormatIDKey,[NSNumber numberWithInt: 1], AVNumberOfChannelsKey,[NSNumber numberWithFloat: 44100.0], AVSampleRateKey,[NSNumber numberWithInt: 64000], AVEncoderBitRateKey,[NSData dataWithBytes: &acl length: sizeof(acl) ], AVChannelLayoutKey,nil]] retain];
audio_writer_input.expectsMediaDataInRealTime = YES;
av_adaptor = [[AVAssetWriterInputPixelBufferAdaptor assetWriterInputPixelBufferAdaptorWithAssetWriterInput: writer_input sourcePixelBufferAttributes:NULL] retain];
[video_writer addInput:writer_input];
[video_writer addInput: audio_writer_input];
[video_writer startWriting];
[video_writer startSessionAtSourceTime: CMTimeMake(0,1)];
start_time = [[NSDate alloc] init];
}
Here is the delegate for the audio:
#implementation AudioOutputBufferDelegate
-(void)captureOutput: (AVCaptureOutput *) captureOutput didOutputSampleBuffer: (CMSampleBufferRef) sampleBuffer fromConnection: (AVCaptureConnection *) conenction{
if (normal_delegate->recording) {
CMSampleBufferSetOutputPresentationTimeStamp(sampleBuffer,CMTimeMakeWithSeconds([[NSDate date] timeIntervalSinceDate: normal_delegate->start_time],30));
[normal_delegate->audio_writer_input appendSampleBuffer: sampleBuffer];
}
}
#end
The video method doesn't matter because it works. "restateConfigiration" just sorts out the session configuration otherwise the torch goes off etc:
[session beginConfiguration];
switch (quality) {
case Low:
session.sessionPreset = AVCaptureSessionPresetLow;
break;
case Medium:
session.sessionPreset = AVCaptureSessionPreset640x480;
break;
}
[session commitConfiguration];
[camera lockForConfiguration:nil];
if([camera isExposureModeSupported:AVCaptureExposureModeContinuousAutoExposure]){
camera.exposureMode = AVCaptureExposureModeContinuousAutoExposure;
}
if([camera isFocusModeSupported:AVCaptureFocusModeContinuousAutoFocus]){
camera.focusMode = AVCaptureFocusModeContinuousAutoFocus;
}
if([camera isWhiteBalanceModeSupported:AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance]){
camera.whiteBalanceMode = AVCaptureWhiteBalanceModeContinuousAutoWhiteBalance;
}
if ([camera hasTorch]) {
if (torch) {
if([camera isTorchModeSupported:AVCaptureTorchModeOn]){
[camera setTorchMode:AVCaptureTorchModeOn];
}
}else{
if([camera isTorchModeSupported:AVCaptureTorchModeOff]){
[camera setTorchMode:AVCaptureTorchModeOff];
}
}
}
[camera unlockForConfiguration];
THank you for any help.
AVAssetWriter and Audio
This may be the same issue as mentioned in the linked post. Try commenting out these lines
[writer_input markAsFinished];
[audio_writer_input markAsFinished];
[video_writer endSessionAtSourceTime: CMTimeMakeWithSeconds([[NSDate date] timeIntervalSinceDate: start_time],30)];
Edit
I don't know if the way you are setting the presentation time stamp is necessarily wrong. The way I handle this is with a local variable that is set to 0 on start. Then when my delegate receives the first packet I do:
if (_startTime.value == 0) {
_startTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
}
and then
[bufferWriter->writer startWriting];
[bufferWriter->writer startSessionAtSourceTime:_startTime];
Your code looks valid as you are calculating the time difference for each received packet. However, AVFoundation calculates this for you, and also optimizes the timestamps for placement in the interleaved container. Another thing I am unsure of is each CMSampleBufferRef for audio contains more then 1 data buffer where each data buffer has it's own PTS. I am not sure if setting the PTS automatically adjusts all the other data buffers.
Where my code differs from yours is I use a single dispatch queue for both audio and video. In the callback I use (some code removed).
switch (bufferWriter->writer.status) {
case AVAssetWriterStatusUnknown:
if (_startTime.value == 0) {
_startTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
}
[bufferWriter->writer startWriting];
[bufferWriter->writer startSessionAtSourceTime:_startTime];
//Break if not ready, otherwise fall through.
if (bufferWriter->writer.status != AVAssetWriterStatusWriting) {
break ;
}
case AVAssetWriterStatusWriting:
if( captureOutput == self.captureManager.audioOutput) {
if( !bufferWriter->audioIn.readyForMoreMediaData) {
break;
}
#try {
if( ![bufferWriter->audioIn appendSampleBuffer:sampleBuffer] ) {
[self delegateMessage:#"Audio Writing Error" withType:ERROR];
}
}
#catch (NSException *e) {
NSLog(#"Audio Exception: %#", [e reason]);
}
}
else if( captureOutput == self.captureManager.videoOutput ) {
if( !bufferWriter->videoIn.readyForMoreMediaData) {
break;;
}
#try {
if (!frontCamera) {
if( ![bufferWriter->videoIn appendSampleBuffer:sampleBuffer] ) {
[self delegateMessage:#"Video Writing Error" withType:ERROR];
}
}
else {
CMTime pt = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
flipBuffer(sampleBuffer, pixelBuffer);
if( ![bufferWriter->adaptor appendPixelBuffer:pixelBuffer withPresentationTime:pt] ) {
[self delegateMessage:#"Video Writing Error" withType:ERROR];
}
}
}
#catch (NSException *e) {
NSLog(#"Video Exception Exception: %#", [e reason]);
}
}
break;
case AVAssetWriterStatusCompleted:
return;
case AVAssetWriterStatusFailed:
[self delegateMessage:#"Critical Error Writing Queues" withType:ERROR];
bufferWriter->writer_failed = YES ;
_broadcastError = YES;
[self stopCapture] ;
return;
case AVAssetWriterStatusCancelled:
break;
default:
break;
}