How to overlay a picture over a video stream with Gstreamer in C? - overlay

Question

I want to overlay a ".png" picture over a stream coming from an IP Camera using Gstreamer.
A working pipeline for my hardware is:
gst-launch-1.0
rtspsrc location=rtsp://user:pass#IP:port/channel latency=400 ! rtph264depay !
vpudec use-vpu-memory=false ! imxvideoconvert_ipu
! video/x-raw,format=I420 ! gdkpixbufoverlay
location=/home/user/folder/image.png offset-x=100 offset-y=100 ! overlaysink
The problem comes when I try to translate this pipeline in C.
The code I wrote for this pipeline runs, but there is no video playback on the display. The player stuck itself before setting the pipeline on "playing" state.
Here, there is a simple version of my C implementation:
#include <gst/gst.h>
#include <glib.h>
#include <iostream>
typedef struct _CustomData {
GstElement *source;
GstElement *rtp;
GstElement *sink;
GstElement *vpudec;
GstElement *converter, *gdkpixbufoverlay, *capsfilter ;
GstBus *bus;
GstElement *pipeline;
GMainLoop *loop;
} CustomData;
static gboolean bus_call (GstBus *bus,
GstMessage *msg,
gpointer data)
{
GMainLoop *loop = (GMainLoop *) data;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS:{
g_print ("End of stream\n");
g_main_loop_quit (loop);
break;
}
case GST_MESSAGE_ERROR: {
gchar *debug;
GError *error;
gst_message_parse_error (msg, &error, &debug);
g_free (debug);
g_printerr ("Error: %s\n", error->message);
g_error_free (error);
g_main_loop_quit (loop);
break;
}
default:
break;
}
return TRUE;
}
static void pad_added_handler (GstElement *src, GstPad *new_pad, CustomData *data) {
GstPad *sink_pad = gst_element_get_static_pad (data->rtp, "sink");
GstPadLinkReturn ret;
GstCaps *new_pad_caps = NULL;
GstStructure *new_pad_struct = NULL;
const gchar *new_pad_type = NULL;
if (gst_pad_is_linked (sink_pad)) {
goto exit;
}
new_pad_caps = gst_pad_query_caps (new_pad, NULL);
new_pad_struct = gst_caps_get_structure (new_pad_caps, 0);
new_pad_type = gst_structure_get_name (new_pad_struct);
if (!g_str_has_prefix (new_pad_type, "application/x-rtp")) {
g_print (" It has type '%s' which is not x-rtp . Ignoring.\n",
new_pad_type);
goto exit;
}
ret = gst_pad_link (new_pad, sink_pad);
if (GST_PAD_LINK_FAILED (ret)) {
g_print(" Type is '%s' but link failed.\n", new_pad_type);
} else {
g_print (" Link succeeded (type '%s').\n", new_pad_type);
}
exit:
if (new_pad_caps != NULL)
gst_caps_unref (new_pad_caps);
gst_object_unref (sink_pad);
}
int main (int argc, char *argv[]){
CustomData data;
gst_init (NULL, NULL);
data.loop = g_main_loop_new (NULL, FALSE);
// Create gstreamer elements
data.pipeline = gst_pipeline_new ("player");
data.source = gst_element_factory_make ("rtspsrc", "source");
data.rtp = gst_element_factory_make ("rtph264depay","rtp");
data.vpudec = gst_element_factory_make ("vpudec","vpudec");
data.converter = gst_element_factory_make
("imxcompositor_ipu","converter");
data.capsfilter = gst_element_factory_make ("capsfilter", "video-
rate");
data.gdkpixbufoverlay = gst_element_factory_make
("gdkpixbufoverlay","overlaytool");
data.sink = gst_element_factory_make ("overlaysink",
"videoSink");
if (!data.pipeline || !data.source || !data.rtp || !data.vpudec ||
!data.converter || !data.capsfilter || !data.gdkpixbufoverlay || !data.sink)
{
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
g_object_set (data.source, "location","rtsp://user:pass#IP:port/channel",
NULL);
g_object_set (data.source,"latency", 400 , NULL);
g_object_set (data.vpudec, "use-vpu-memory", false, NULL);
g_object_set (data.gdkpixbufoverlay,
"location","/home/user/folder/image.png", NULL);
g_object_set (data.gdkpixbufoverlay, "offset-x", 100 , NULL);
g_object_set (data.gdkpixbufoverlay, "offset-y", 100 , NULL);
GstCaps *capsFormat = gst_caps_from_string ("video/x-raw,format=I420");
g_object_set ( data.capsfilter, "caps", capsFormat, NULL);
gst_caps_unref(capsFormat);
//add all elements into the pipeline
gst_bin_add_many (GST_BIN (data.pipeline),
data.source,
data.rtp,
data.vpudec,
data.converter,
data.capsfilter,
data.gdkpixbufoverlay,
data.sink,
NULL);
// link all elements
gst_element_link_many ( data.rtp, data.vpudec , data.converter ,
data.capsfilter, data.gdkpixbufoverlay, data.sink, NULL);
g_signal_connect (data.source, "pad-added", G_CALLBACK (pad_added_handler),
&data);
// Set the pipeline to "playing" state
GstStateChangeReturn ret;
ret = gst_element_set_state (data.pipeline, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE) {
g_printerr("Unable to set the pipeline to the playing state.\n");
gst_object_unref (data.pipeline);
return -1;
}
// Iterate
g_main_loop_run (data.loop);
// Out of the main loop, clean
g_print ("Returned, stopping playback\n");
gst_element_set_state (data.pipeline, GST_STATE_NULL);
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (data.pipeline));
return 0;
}
Does anyone see the problem?
Thank you

Answer 1

After many tries, I did figure out that in the C code I posted I have selected the wrong element, therefore the data.converter element is:
data.converter = gst_element_factory_make("imxvideoconvert_ipu ","converter");
and not imxcompositor_ipu .