FFmpeg 图片编视频

N_FFmpeg

实例 图片编视频 再推rtmp

关键几点

1. 需要将图片转为 AVFrame
2. AVFrame RGB像素格式转为 YUV (因为H264标准就是YUV格式)
3. 注意PTS/DTS, I帧, SPS/PPS 几个参数!

c++ - images - ffmpeg video converter - https://code-examples.net/en/q/20e99d0

RGBdata 转 AVFrame

raw image to h264 stream

QImage 转 AVFrame/YUV

// QImage 转为 yuv AVFrame; 
void qImageToAvFrame(QImage &image,AVFrame *yuvFrame, int width,int height){
    for(int h=0; h<height; h++){
        for(int w=0; w<width; w++){
            QRgb rgb = image.pixel(w, h);
 
            int r = qRed(rgb);
            int g = qGreen(rgb);
            int b = qBlue(rgb);
 
            int dy = ((66*r + 129*g + 25*b) >> 8) + 16;
            int du = ((-38*r + -74*g + 112*b) >> 8) + 128;
            int dv = ((112*r + -94*g + -18*b) >> 8) + 128;
 
            uchar yy = (uchar)dy;
            uchar uu = (uchar)du;
            uchar vv = (uchar)dv;
 
            yuvFrame->data[0][h * yuvFrame->linesize[0] + w] = yy;
 
            if(h % 2 == 0 && w % 2 == 0)
            {
                yuvFrame->data[1][h/2 * (yuvFrame->linesize[1]) + w/2] = uu;
                yuvFrame->data[2][h/2 * (yuvFrame->linesize[2]) + w/2] = vv;
            }
        }
    }
}
void main(){
    QString path = "E:\\content-for-work\\2022-04图片编流\\";
    QString qif =  QString(path+"%1%2%3").arg("frame_").arg(1).arg(".jpg");
    QImage image(qif);
    int width = 1280, height = 720;
    AVFrame *yuvFrame = av_frame_alloc();
    av_image_alloc(yuvFrame->data, yuvFrame->linesize, width, height,AV_PIX_FMT_YUVJ420P, 4);
    //必须设置的一些属性
    yuvFrame->pict_type = AVPictureType::AV_PICTURE_TYPE_I;
    yuvFrame->height = height, yuvFrame->width = width, yuvFrame->key_frame=1;
    yuvFrame->format = 12;
    //by user 可设可不设 实测!
//    yuvFrame->color_range = AVColorRange::AVCOL_RANGE_JPEG;
//    yuvFrame->colorspace = AVColorSpace::AVCOL_SPC_BT709;
//    yuvFrame->chroma_location = AVChromaLocation::AVCHROMA_LOC_LEFT;
    //yuvFrame->sample_aspect_ratio = AVRational{0,1};
    //yuvFrame->time_base  = AVRational{0,1};
    yuvFrame->extended_data = yuvFrame->data;
    yuvFrame->pts = 512, yuvFrame->pkt_dts = 512;
    qImageToAvFrame(image, yuvFrame,width,height);
    SaveFrame(yuvFrame, QString(path+"%1%2%3").arg("frame_").arg(1).arg("_conv.jpg").toStdString().data() );
}

纯自定义的 AVFrame必须设置一些属性, 否则: Resource temporarily unavailable ! 编码失败! 不同分辨率可在QImage缩放处理

QImage与AVFrame互转

AVFrame/YUV 编码为视频

代码片段..

//图片转为YUV 再用 H264 编码为视频
int testH264Encoder(QList<QImage> *imgList, char *outMP4, int width,int height){
    int	ret = -1;
    //AVFormatContext 和 AVStream 初始化
    AVCodecContext	*pCodeCtx	= nullptr;
    AVFormatContext *pFormatCtx = avformat_alloc_context();
    pFormatCtx->oformat = av_guess_format( "mp4", nullptr, nullptr );
    if (avio_open( &pFormatCtx->pb, outMP4, AVIO_FLAG_READ_WRITE ) < 0 ){
        std::cout<<"Couldn't open output file.";return(-1);
    }
    AVStream *pAVStream = avformat_new_stream( pFormatCtx, nullptr );
    pAVStream->time_base = (AVRational) { 1, 90000 };//90k
    AVCodecParameters *parameters = pAVStream->codecpar;
    parameters->codec_id	= pFormatCtx->oformat->video_codec;    // h264 
    parameters->codec_type	= AVMEDIA_TYPE_VIDEO; 
    parameters->format= AV_PIX_FMT_YUVJ420P;
    parameters->width	= width;
    parameters->height	= height;
    parameters->bit_rate = 40*1000;
    //编码器 初始化
    const AVCodec *pCodec = avcodec_find_encoder( pAVStream->codecpar->codec_id );
    pCodeCtx = avcodec_alloc_context3( pCodec );
    avcodec_parameters_to_context( pCodeCtx, pAVStream->codecpar );
    pCodeCtx->time_base = (AVRational) { 1, 30 };//帧率
    av_opt_set(pCodeCtx->priv_data, "preset", "superfast", 0);
    av_opt_set(pCodeCtx->priv_data, "tune","zerolatency",0);//不缓存帧, 解决头16帧 -11 问题 resource temporarily unavailable 
    avcodec_open2(pCodeCtx, pCodec, nullptr );
    // 开始编码
    ret = avformat_write_header( pFormatCtx, nullptr );
    int y_size = width * height;
    AVPacket pkt;
    av_new_packet( &pkt, y_size * 3 );
    //YUV AVFrame图片的 常属性 初始化 <!>重要!!
    AVFrame *yuvFrame = av_frame_alloc();
    av_image_alloc(yuvFrame->data, yuvFrame->linesize, width, height,AV_PIX_FMT_YUVJ420P, 4);
    yuvFrame->pict_type = AVPictureType::AV_PICTURE_TYPE_I;
    yuvFrame->height = height, yuvFrame->width = width, yuvFrame->key_frame=1;
    yuvFrame->format = 12;
     qInfo()<<"time: "<<QDateTime::currentMSecsSinceEpoch()<<",开始编码";
    for(int i=0; i<150; i++){
        int select = 1;//图片帧..
        QImage image = imgList->at(select-1);
        qImageToAvFrame(image, yuvFrame,width,height);//转为 YUV AVFrame
        //每帧的数据设置, PTS,DTS
        int64_t calc_pts =  av_rescale_q_rnd(i * 3000, pAVStream->time_base, pAVStream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
        yuvFrame->pts = calc_pts;
        yuvFrame->pkt_dts = yuvFrame->pts;
        yuvFrame->extended_data = yuvFrame->data;//看头文件注释, 视频帧这俩是一样的, 设一下吧..
        ret = avcodec_send_frame( pCodeCtx, yuvFrame );
        while(true){
            ret = avcodec_receive_packet( pCodeCtx, &pkt );
            av_packet_unref( &pkt );
        }
    }
    av_write_trailer( pFormatCtx );
    qInfo()<<"time: "<<QDateTime::currentMSecsSinceEpoch()<<",编码结束";
    ...
}

踩坑指南

踩坑 -11 问题 resource temporarily unavailable

不知道原因, 头十几帧是 -11(resource temporarily unavailable), 最终的文件末尾则会丢失该数量帧;

在open编码器之前设置 zerolatency 可解决之..

av_opt_set(pCodeCtx->priv_data, "tune","zerolatency",0);
ret = avcodec_open2(pCodeCtx, pCodec, nullptr );

踩坑 灰屏问题

编出来的视频灰屏, 给编码器设置全参数~

pCodeCtx->me_range = 16;
pCodeCtx->max_qdiff = 4;
pCodeCtx->qmin = 3;
pCodeCtx->qmax = 30;
pCodeCtx->qcompress = 1;

踩坑 改为推rtmp -22 问题 Invalid argument ?

AV_CODEC_ID_MJPEG 直接编jpeg

AVFormatContext 的协议编码器, 跟真实的 AVStream 编码器不是强制绑定的!!! 用AV_CODEC_ID_MJPEG编码, 直接喂jpg数据, 初始化的是AVPacket, 不用AVFrame,有个问题无法控制码率,但编码速度非常快,资源消耗小

// 初始化AVStream, 用mjpeg编码器 AV_CODEC_ID_MJPEG
AVStream *pAVStream = avformat_new_stream( pFormatCtx, nullptr );
pAVStream->time_base = (AVRational) { 1, 90000 };//90k
AVCodecParameters *parameters = pAVStream->codecpar;
parameters->codec_id	= AVCodecID::AV_CODEC_ID_MJPEG;
parameters->codec_type	= AVMEDIA_TYPE_VIDEO;
parameters->format	= AV_PIX_FMT_YUVJ420P;
...
//初始化 AVCodec 编码器
const AVCodec *pCodec = avcodec_find_encoder( pAVStream->codecpar->codec_id );
pCodeCtx = avcodec_alloc_context3( pCodec );
avcodec_parameters_to_context( pCodeCtx, pAVStream->codecpar );
pCodeCtx->time_base = (AVRational) { 1, 30 };//帧率
avcodec_open2(pCodeCtx, pCodec, nullptr );
 
//初始化 AVPacket 编包
ret = avformat_write_header( pFormatCtx, nullptr );
int y_size = width * height;
AVPacket pkt;
av_new_packet( &pkt, y_size * 3 );
pkt.flags |= AV_PKT_FLAG_KEY;
pkt.stream_index = pAVStream->index;
...
//拿到QImage jpg的bytes 给
for(int i=0; i<150; i++){
    ...
    QImage image = imgList->at(select-1);
    QByteArray ba;
    QBuffer buffer(&ba);
    buffer.open(QIODevice::WriteOnly);
    image.save(&buffer, "jpeg");
    pkt.size = ba.size();
    pkt.data = (uint8_t *)ba.data();
    //每一帧的数据设置
    int64_t calc_pts =  av_rescale_q_rnd(i * 3000, pAVStream->time_base, pAVStream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
    pkt.dts = pkt.pts = calc_pts;
    ret = av_write_frame( pFormatCtx, &pkt );
...
 

推 RTMP

推RTMP时若缺失SPS/PPS, 需在写header时填充sps,pps等信息, 嗯.. 兼容性不好的播放器无法播放, 即使可以解码 延时也会很高!

AVFormatContext *pFormatCtx = nullptr;
ret = avformat_alloc_output_context2(&pFormatCtx,nullptr,"flv", output);
if (avio_open(&pFormatCtx->pb, output,AVIO_FLAG_WRITE) < 0) {
    std::cout<<"Couldn't open output file.";return(-1);
}
AVStream *pAVStream = avformat_new_stream( pFormatCtx, nullptr );
pAVStream->time_base = (AVRational) { 1, 90000 };//90k
AVCodecParameters *parameters = pAVStream->codecpar;
//
parameters->codec_id = AVCodecID::AV_CODEC_ID_H264;//pFormatCtx->oformat->video_codec; 是flv, 什么鬼?
parameters->codec_type	= AVMEDIA_TYPE_VIDEO;
parameters->format	= AV_PIX_FMT_YUVJ420P;
parameters->extradata = spsAndpps;
int size_of = sizeof (spsAndpps);

测试代码归档

ffmpeg_image_to_video.cpp

命令行工具

ffmpeg -loop 1 -framerate 24 -i frame_1.jpg -c:v libx264 
-preset slow -tune stillimage -crf 24 -vf format=yuv420p -t 5 -movflags +faststart cmd_output.mp4

mp4 转 hls

ffmpeg -i cmd_output.mp4 -c:v libx264 -hls_time 2  -hls_list_size 0  -strict -2 -f hls index.m3u8