Problems with OpenGL when using Image Processing to filter video - iphone

Question

I am trying to use Brad Larson's Framework as a reference to get some basic image processing tests. So far i have implemented the sepia filter and it kind of works, the problem is that I get a processed Image and a non processed Image every render cycle (like if i set the fps to 1 sec, first second rendered , second not, third yes, etc). And i have spent hours and i cant seem to find out why.
I have 2 windows, the video preview layer and a glkview where i am rendering my buffer.
This is my code, which is basically just an attempt to synthethize the previously mentioned framework:
View did Load method:
- (void)viewDidLoad
{
[super viewDidLoad];
session = [[AVCaptureSession alloc] init];
// Add inputs and outputs.
if ([session canSetSessionPreset:AVCaptureSessionPreset640x480]) {
session.sessionPreset = AVCaptureSessionPreset640x480;
}
else {
// Handle the failure.
NSLog(#"Cannot set session preset to 640x480");
}
AVCaptureDevice *device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
NSError *error = nil;
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
if (!input) {
// Handle the error appropriately.
NSLog(#"Could create input: %#", error);
}
if ([session canAddInput:input]) {
[session addInput:input];
}
else {
// Handle the failure.
NSLog(#"Could not add input");
}
captureVideoPreviewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:session];
[captureVideoPreviewLayer setVideoGravity:AVLayerVideoGravityResizeAspect];
[captureVideoPreviewLayer setBounds:videoLayer.layer.bounds];
[captureVideoPreviewLayer setPosition:videoLayer.layer.position];
[videoLayer.layer addSublayer:captureVideoPreviewLayer];
[session startRunning];
// OPENGL stuff
self.imageView.context = [[GPUImageOpenGLESContext sharedImageProcessingOpenGLESContext] context];
if ([self createFilterProgram]) {
NSLog(#"Filter Program Created");
}
if ([self createOutputProgram]) {
NSLog(#"Output Program Created");
}
if (CVOpenGLESTextureCacheCreate != NULL)
{
[GPUImageOpenGLESContext useImageProcessingContext];
CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, (__bridge void *)[[GPUImageOpenGLESContext sharedImageProcessingOpenGLESContext] context], NULL, &coreVideoTextureCache);
if (err)
{
NSAssert(NO, #"Error at CVOpenGLESTextureCacheCreate %d");
}
else
NSLog(#"Initial CVOpenGLESTextureCache created");
// Need to remove the initially created texture
if (outputTexture)
{
glDeleteTextures(1, &outputTexture);
outputTexture = 0;
}
}
// DATA OUTPUT
dataOutput = [[AVCaptureVideoDataOutput alloc] init];
if ([session canAddOutput:dataOutput]) {
[session addOutput:dataOutput];
dataOutput.videoSettings =
[NSDictionary dictionaryWithObject: [NSNumber numberWithInt:kCVPixelFormatType_32BGRA]
forKey: (id)kCVPixelBufferPixelFormatTypeKey];
AVCaptureConnection *connection = [dataOutput connectionWithMediaType:AVMediaTypeVideo];
[connection setVideoMaxFrameDuration:CMTimeMake(1, 1)];
[connection setVideoMinFrameDuration:CMTimeMake(1, 1)];
}
else {
// Handle the failure.
NSLog(#"Could not add output");
}
// DATA OUTPUT END
//dispatch_queue_t queue = dispatch_queue_create("MyQueue", NULL);
[dataOutput setSampleBufferDelegate:self queue:dispatch_get_main_queue()];
//dispatch_release(queue);
}
and the delegate method from the video capture:
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection
{
NSLog(#"Render");
//CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent();
CVImageBufferRef cameraFrame = CMSampleBufferGetImageBuffer(sampleBuffer);
int bufferWidth = CVPixelBufferGetWidth(cameraFrame);
int bufferHeight = CVPixelBufferGetHeight(cameraFrame);
if (CVOpenGLESTextureCacheCreate != NULL)
{
NSLog(#"Not Null");
CVPixelBufferLockBaseAddress(cameraFrame, 0);
[GPUImageOpenGLESContext useImageProcessingContext];
CVOpenGLESTextureRef texture = NULL;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, coreVideoTextureCache, cameraFrame, NULL, GL_TEXTURE_2D, GL_RGBA, bufferWidth, bufferHeight, GL_BGRA, GL_UNSIGNED_BYTE, 0, &texture);
if (!texture || err) {
NSLog(#"CVOpenGLESTextureCacheCreateTextureFromImage failed (error: %d)", err);
return;
}
outputTexture = CVOpenGLESTextureGetName(texture);
glBindTexture(CVOpenGLESTextureGetTarget(texture), outputTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// "setInputSize" method
CGSize sizeOfFBO = CGSizeMake(bufferWidth, bufferHeight);
// "newFrameReady" method
static const GLfloat squareVertices[] = {
-1.0f, -1.0f,
1.0f, -1.0f,
-1.0f, 1.0f,
1.0f, 1.0f,
};
static const GLfloat squareTextureCoordinates[] = {
0.0f, 0.0f,
1.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f,
};
[GPUImageOpenGLESContext useImageProcessingContext];
// "setFilterFBO" method
if (!filterFramebuffer)
{
NSLog(#"New Filter Frame buffer");
glActiveTexture(GL_TEXTURE1);
glGenFramebuffers(1, &filterFramebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, filterFramebuffer);
NSLog(#"Filter size: %f, %f", sizeOfFBO.width, sizeOfFBO.height);
glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8_OES, (int)sizeOfFBO.width, (int)sizeOfFBO.height);
glBindTexture(GL_TEXTURE_2D, outputTexture);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, (int)sizeOfFBO.width, (int)sizeOfFBO.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, outputTexture, 0);
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
NSAssert(status == GL_FRAMEBUFFER_COMPLETE, #"Incomplete filter FBO: %d", status);
}
glBindFramebuffer(GL_FRAMEBUFFER, filterFramebuffer);
glViewport(0, 0, (int)sizeOfFBO.width, (int)sizeOfFBO.height);
// END "setFilterFBO" method
glUseProgram(filterProgram); // This Program MUST be the filter one
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, outputTexture);
glUniform1i(uniforms[filterInputTextureUniform], 2);
glVertexAttribPointer(position, 2, GL_FLOAT, 0, 0, squareVertices);
glVertexAttribPointer(inputTextureCoordinate, 2, GL_FLOAT, 0, 0, squareTextureCoordinates);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
static const GLfloat squareVertices2[] = {
-1.0f, -1.0f,
1.0f, -1.0f,
-1.0f, 1.0f,
1.0f, 1.0f,
};
static const GLfloat textureCoordinates2[] = {
0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f,
};
[GPUImageOpenGLESContext useImageProcessingContext];
//[self setDisplayFramebuffer];
if (!displayFramebuffer)
{
NSLog(#"New Display Frame buffer");
glGenFramebuffers(1, &displayFramebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, displayFramebuffer);
glGenRenderbuffers(1, &displayRenderbuffer);
glBindRenderbuffer(GL_RENDERBUFFER, displayRenderbuffer);
[[[GPUImageOpenGLESContext sharedImageProcessingOpenGLESContext] context] renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer*)imageView.layer];
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &backingWidth);
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &backingHeight);
NSLog(#"Backing width: %d, height: %d", backingWidth, backingHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, displayRenderbuffer);
GLuint framebufferCreationStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
NSAssert(framebufferCreationStatus == GL_FRAMEBUFFER_COMPLETE, #"Failure with display framebuffer generation");
}
glBindFramebuffer(GL_FRAMEBUFFER, displayFramebuffer);
glViewport(0, 0, backingWidth, backingHeight);
glUseProgram(outputProgram);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glActiveTexture(GL_TEXTURE4);
glBindTexture(GL_TEXTURE_2D, outputTexture);
glUniform1i(uniformsOutput[outputTextureUniform], 4);
glVertexAttribPointer(outputPosition, 2, GL_FLOAT, 0, 0, squareVertices2);
glVertexAttribPointer(outputTextureCoordinate, 2, GL_FLOAT, 0, 0, textureCoordinates2);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glBindRenderbuffer(GL_RENDERBUFFER, displayRenderbuffer);
[[GPUImageOpenGLESContext sharedImageProcessingOpenGLESContext] presentBufferForDisplay];
//glBindRenderbuffer(GL_RENDERBUFFER, 0);
CVPixelBufferUnlockBaseAddress(cameraFrame, 0);
glBindTexture(outputTexture, 0);
// Flush the CVOpenGLESTexture cache and release the texture
CVOpenGLESTextureCacheFlush(coreVideoTextureCache, 0);
CFRelease(texture);
outputTexture = 0;
}
else
NSLog(#"No support for fast CVOpenGLESTextureCacheCreate");
}
This is Brad's original framework (which is pretty cool) http://www.sunsetlakesoftware.com/2012/02/12/introducing-gpuimage-framework
If anyone has any idea why this might be happening i would appreciate the help.
So far the reasons i thought that could cause this are:
Rendering 2 times during the cycle (which i discarded it because it
happens every 1 second, and even if it did happen super fast the last
one would stay which is the processed one)
Enabling or disabling a state of OpenGL (which i think might be the
cause but cant find where)
Not using the buffers correctly (I am using the same buffer variable
for both steps this might cause some trouble)
It might have something to do with the context (I believe the context
is where everything is being drawn, so If i am working on the glkview
context I might be overwriting the image)
Please correct me if some of this ideas are wrong, this is my first attempt on processing using OpenGL
Thank you!
EDIT:
As suggested by Brad, I used the OpenGL profiler and got the following results:
This tool does seem to be amazing for optimizing the OpenGL code.