Encoding H264 with OMX RTMP/FLV on a RPI issue - encoding

Question

I use an RPI to stream a video sequence through RTMP with H264 encoding, using the OpenMax(OMX) hardware encoder on-board.
What I get out of the stream is an image that seems to be misaligned (without padding) where the YUV planes are not aligned.
Here is an example of the outputenter image description here
I'm using opencv and ffmpeg with libswscale to encode the stream. I think I have narrowed down the issue to the conversion process with the swscale call. Could someone point me if I'm doing something wrong or give me any pointers? The video stream is otherwise completely stable.
The resolution is 1280x720.
Here is my code.
#include <iostream>
#include <vector>
#include <opencv2/highgui.hpp>
#include <opencv2/video.hpp>
#include <opencv2/opencv.hpp>
#include "constants_c.h"
#include <iostream>
extern "C" {
#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>
#include <libavutil/channel_layout.h>
#include <libavutil/common.h>
#include <libavutil/imgutils.h>
#include <libavutil/mathematics.h>
#include <libavutil/samplefmt.h>
#include <libavformat/avformat.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
}
cv::VideoCapture get_device(int camID, double width, double height)
{
cv::VideoCapture cam;
cam.open(camID, CV_CAP_V4L2);
if (!cam.isOpened())
{
std::cout << "Failed to open video capture device!" << std::endl;
exit(1);
}
cam.set(cv::CAP_PROP_FRAME_WIDTH, width);
cam.set(cv::CAP_PROP_FRAME_HEIGHT, height);
return cam;
}
void initialize_avformat_context(AVFormatContext *&fctx, const char *format_name)
{
int ret = avformat_alloc_output_context2(&fctx, nullptr, format_name, nullptr);
if (ret < 0)
{
std::cout << "Could not allocate output format context!" << std::endl;
exit(1);
}
}
void initialize_io_context(AVFormatContext *&fctx, const char *output)
{
if (!(fctx->oformat->flags & AVFMT_NOFILE))
{
int ret = avio_open2(&fctx->pb, output, AVIO_FLAG_WRITE, nullptr, nullptr);
if (ret < 0)
{
std::cout << "Could not open output IO context!" << std::endl;
exit(1);
}
}
}
void set_codec_params(AVFormatContext *&fctx, AVCodecContext *&codec_ctx, double width, double height, int fps)
{
const AVRational dst_fps = {fps, 1};
codec_ctx->codec_tag = 0;
codec_ctx->codec_id = AV_CODEC_ID_H264;
codec_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
codec_ctx->width = width;
codec_ctx->height = height;
codec_ctx->gop_size = 12;
codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
codec_ctx->framerate = dst_fps;
codec_ctx->time_base = av_inv_q(dst_fps);
if (fctx->oformat->flags & AVFMT_GLOBALHEADER)
{
codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
}
void initialize_codec_stream(AVStream *&stream, AVCodecContext *&codec_ctx, AVCodec *&codec)
{
int ret = avcodec_parameters_from_context(stream->codecpar, codec_ctx);
if (ret < 0)
{
std::cout << "Could not initialize stream codec parameters!" << std::endl;
exit(1);
}
AVDictionary *codec_options = nullptr;
av_dict_set(&codec_options, "profile", "high", 0);
av_dict_set(&codec_options, "preset", "superfast", 0);
av_dict_set(&codec_options, "tune", "zerolatency", 0);
// open video encoder
ret = avcodec_open2(codec_ctx, codec, &codec_options);
if (ret < 0)
{
std::cout << "Could not open video encoder!" << std::endl;
exit(1);
}
}
SwsContext *initialize_sample_scaler(AVCodecContext *codec_ctx, double width, double height)
{
SwsContext *swsctx = sws_getContext(width, height, AV_PIX_FMT_BGR24, width, height, codec_ctx->pix_fmt, SWS_BICUBIC, nullptr, nullptr, nullptr);
if (!swsctx)
{
std::cout << "Could not initialize sample scaler!" << std::endl;
exit(1);
}
return swsctx;
}
AVFrame *allocate_frame_buffer(AVCodecContext *codec_ctx, double width, double height)
{
AVFrame *frame = av_frame_alloc();
std::vector<uint8_t> framebuf(av_image_get_buffer_size(codec_ctx->pix_fmt, width, height, 1));
av_image_fill_arrays(frame->data, frame->linesize, framebuf.data(), codec_ctx->pix_fmt, width, height, 1);
frame->width = width;
frame->height = height;
frame->format = static_cast<int>(codec_ctx->pix_fmt);
return frame;
}
void write_frame(AVCodecContext *codec_ctx, AVFormatContext *fmt_ctx, AVFrame *frame)
{
AVPacket pkt = {0};
av_init_packet(&pkt);
int ret = avcodec_send_frame(codec_ctx, frame);
if (ret < 0)
{
std::cout << "Error sending frame to codec context!" << std::endl;
exit(1);
}
do
{
ret = avcodec_receive_packet(codec_ctx, &pkt);
if (ret == 0)
{
break;
}
else if ((ret < 0) && (ret != AVERROR(EAGAIN)))
{
fprintf(stderr, "Error receiving packet from codec context! %i\n",ret );
}
else if (ret == AVERROR(EAGAIN))
{
ret = avcodec_send_frame(codec_ctx, frame);
if (0 > ret)
{
char err[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(err, AV_ERROR_MAX_STRING_SIZE, ret);
}
}
} while (ret == 0);
av_interleaved_write_frame(fmt_ctx, &pkt);
av_packet_unref(&pkt);
}
void stream_video(double width, double height, int fps, int camID)
{
av_log_set_level(AV_LOG_DEBUG);
av_register_all();
avformat_network_init();
const char *output = "rtmp://172.17.xx.x:1935/live/stream";
int ret;
auto cam = get_device(camID, width, height);
std::vector<uint8_t> imgbuf(height * width * 3 + 16);
cv::Mat image(height, width, CV_8UC3, imgbuf.data(), width * 3);
AVFormatContext *ofmt_ctx = nullptr;
AVCodec *out_codec = nullptr;
AVStream *out_stream = nullptr;
AVCodecContext *out_codec_ctx = nullptr;
initialize_avformat_context(ofmt_ctx, "flv");
initialize_io_context(ofmt_ctx, output);
out_codec = avcodec_find_encoder(AV_CODEC_ID_H264);
out_stream = avformat_new_stream(ofmt_ctx, out_codec);
out_codec_ctx = avcodec_alloc_context3(out_codec);
set_codec_params(ofmt_ctx, out_codec_ctx, width, height, fps);
initialize_codec_stream(out_stream, out_codec_ctx, out_codec);
out_stream->codecpar->extradata = out_codec_ctx->extradata;
out_stream->codecpar->extradata_size = out_codec_ctx->extradata_size;
av_dump_format(ofmt_ctx, 0, output, 1);
auto *swsctx = initialize_sample_scaler(out_codec_ctx, width, height);
auto *frame = allocate_frame_buffer(out_codec_ctx, width, height);
av_image_alloc(frame->data, frame->linesize, frame->width, frame->height, AV_PIX_FMT_YUV420P, 32);
int cur_size;
uint8_t *cur_ptr;
ret = avformat_write_header(ofmt_ctx, nullptr);
if (ret < 0)
{
std::cout << "Could not write header!" << std::endl;
exit(1);
}
bool end_of_stream = false;
int64_t frame_pts = 0;
do
{
bool ok = cam.read(image);
const int stride[] = {static_cast<int>(image.step[0])};
std::cout << "sws_scale" << std::endl;
sws_scale(swsctx, &image.data, stride, 0, image.rows, frame->data, frame->linesize);
//frame->pts += av_rescale_q(1, out_codec_ctx->time_base, out_stream->time_base);
frame->pts = frame_pts++;
write_frame(out_codec_ctx, ofmt_ctx, frame);
} while (!end_of_stream);
av_write_trailer(ofmt_ctx);
av_frame_free(&frame);
avcodec_close(out_codec_ctx);
avio_close(ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
}
int main()
{
// av_log_set_level(AV_LOG_DEBUG);
double width = 1280, height = 720;
int camID = 0;
int fps = 5;
stream_video(width, height, fps, camID);
return 0;
}
enter code here