FFmpeg 读取摄像头 封装FMP4

N_FFmpeg

实例 读取摄像头 封装FMP4 for linux

读取摄像头输出到文件 参考 - http://www.ffmpeg.org/ffmpeg-all.html#toc-fbdev

ffmpeg  -s 320*240 -r 10 -i /dev/video0 testvideo.mp4
 
//ffplay 播放
ffplay -i /dev/video0

C/C++ API

首先, 使用libavdevice的时候需要包含其头文件: #include "libavdevice/avdevice.h"

然后, 在程序中需要注册libavdevice: avdevice_register_all();

使用

AVFormatContext *avFormat = avformat_alloc_context();
AVInputFormat *ifmt=av_find_input_format("video4linux2");
 
int ret = avformat_open_input(&avFormat, "/dev/video0", ifmt, 0);
 
ret = avformat_find_stream_info(avFormat, 0);
 
printf("source find_stream_info= %d \n", ret);
printf("source >>>>>>>>>>>>>>>>>>>>\n");
av_dump_format(avFormat, 0, "",0 );
printf("source <<<<<<<<<<<<<<<<<<<<\n");
 

source >>>>>>>>>>>>>>>>>>>>
Input #0, video4linux2,v4l2, from '':
    Duration: N/A, start: 5016.665747, bitrate: 115200 kb/s
    Stream #0:0: Video: rawvideo (YUY2 / 0x32595559), yuyv422, 800x600, 115200 kb/s, 15 fps, 15 tbr, 1000k tbn, 1000k tbc
source <<<<<<<<<<<<<<<<<<<<

拿到是 rawvideo 格式,无需解码可直接显示.

封装为Fragmented MP4 浏览器直播

从树莓派读取摄像头数据, 推送流到服务器, 服务器再websocket 分发到浏览器MSE api可播放的格式

FFMPEG结构体分析: AVFormatContext Mp4 必选与可选的box

读取摄像头输入

关键代码

...
int source_open( struct source_ctx *ctx) {
 
AVFormatContext *avFormat = avformat_alloc_context();
AVInputFormat *ifmt=av_find_input_format("video4linux2");
int ret = avformat_open_input(&avFormat, ctx->name, ifmt, 0);
if (_global_print_info ){
    printf("source open_input= %d \n", ret);
}
 
ret = avformat_find_stream_info(avFormat, 0);
if (_global_print_info ){
    printf("source find_stream_info= %d \n", ret);
    printf(">>>>>>>>>source>>>>>>>>>>>\n");
    av_dump_format(avFormat, 0, "",0 );
    printf("<<<<<<<<<<<<<<<<<<<<\n");
}
for(int i=0; i<avFormat->nb_streams; i++) {
    if(avFormat->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO){
        ctx->video_index = i;
    }else if(avFormat->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO){
        ctx->audio_index = i;
    }
}
AVCodec *codec = avcodec_find_decoder(avFormat->streams[ ctx->video_index]->codecpar->codec_id);
AVCodecContext *coder_ctx  = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(coder_ctx,   avFormat->streams[ ctx->video_index]->codecpar);
ret = avcodec_open2(coder_ctx, codec, NULL);
if (_global_print_info ){
    printf("source avcodec open2: %d \n", ret);
}
//AVFormatContext 和 AVCodecContext
ctx->codecCtx = coder_ctx;
ctx->avFormat = avFormat;
return 0;

创建输出

关键代码

...
 // source_ctx=输入参数结构体, target_ctx=输出参数结构体
int target_open( struct source_ctx *s_ctx, struct target_ctx *ctx){
    ...
    //内存输出
    ctx->ioBuffer = (unsigned char*)av_malloc(40960);
    AVIOContext* avio = avio_alloc_context(  ctx->ioBuffer, 40960 ,1, ctx, NULL, target_push,NULL);
    AVOutputFormat *oformat = av_guess_format("mp4", NULL, NULL);
    AVFormatContext *avFormat = NULL;
    ret = avformat_alloc_output_context2(&avFormat,oformat,NULL,NULL);
    avFormat->pb = avio;
    avFormat->flags = AVFMT_FLAG_CUSTOM_IO;
    AVStream *av_stream = avformat_new_stream(avFormat, NULL);
    if (!av_stream) {
       return -301;
    }
    // AVCodecParameters 
    AVCodecParameters *codecpar = av_stream->codecpar;
    codecpar->codec_id = avFormat->oformat->video_codec;
    codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
    codecpar->format = AV_PIX_FMT_YUV420P;
 
    codecpar->width = s_ctx->avFormat->streams[s_ctx->video_index]->codecpar->width;
    codecpar->height = s_ctx->avFormat->streams[s_ctx->video_index]->codecpar->height;
 //    codecpar->width =400;
 //    codecpar->height = 300;
    //坑! SPS PPS, 输出不然输出的mp4 会缺少avcC box
    unsigned char sps_pps[] ={0x00,0x00,0x00,0x01,0x67,0x42,0xC0,0x1F,0xDA,0x03,0x20,0x4D,0xF9,0x61,0x00,0x00,0x03,0x00,0x01,0x00,0x00,0x03,0x00,0x28,0x8F,0x18,0x32,0xA0,
                              0x00,0x00,0x00,0x01,0x68,0xCE,0x3C,0x80};
    int size_of = sizeof (sps_pps);
    codecpar->extradata_size = size_of;
    codecpar->extradata = (uint8_t*)av_malloc(size_of + AV_INPUT_BUFFER_PADDING_SIZE);
    memcpy(codecpar->extradata, sps_pps, size_of);
 
    AVCodec *codec =  avcodec_find_encoder(av_stream->codecpar->codec_id);
    AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
    ret = avcodec_parameters_to_context(codec_ctx, s_ctx->avFormat->streams[s_ctx->video_index]->codecpar);
    if (_global_print_info ){
       printf("target cpoy parameters_context: %d \n", ret);
    }
    if (ret != 0) {
       //TODO
       return -302;
    }
    //对一下参数, 也许对 AVCodecParameters 统一设置也行..
    //FPS, 颜色格式, gop 等
    codec_ctx->time_base = (AVRational){1, 10};
    codec_ctx->codec_id = codec->id;
    codec_ctx->pix_fmt =  codecpar->format;
    codec_ctx->gop_size = 20;
    codec_ctx->width = codecpar->width ;
    codec_ctx->height = codecpar->height ;
    codec_ctx->bit_rate =5000*1000;
    //重复的把SPS/PPS放到关键帧前面
    codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    //参考下 [h264 编码低延迟]
    av_opt_set(codec_ctx->priv_data,"tune","zerolatency",0);
    av_opt_set(codec_ctx->priv_data,"preset","ultrafast",0);
    //还是设置一下吧..
    av_opt_set(codec_ctx->priv_data, "profile", "baseline", 0);
    av_stream->codec->codec_tag = 0;
    if (avFormat->oformat->flags & AVFMT_GLOBALHEADER){
       av_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    }
    AVDictionary* options = NULL;
    //av_dict_set(&options,"tune","zerolatency",0);
    //av_dict_set(&options,"preset","ultrafast",0);
    ret = avcodec_open2(codec_ctx, codec, &options);
    printf("target codec open: %d \n", ret);
    if (_global_print_info ){
        printf(">>>>>>>>>>target>>>>>>>>>>\n");
        av_dump_format(avFormat, 0, "",1 );
        printf("<<<<<<<<<<<<<<<<<<<<\n");
    }
    ctx->codecCtx = codec_ctx;
    ctx->avFormat = avFormat;
 
 
}

解封装循环

关键代码

int loop(struct source_ctx *s_ctx, struct target_ctx *t_ctx, struct push_ctx * ctx){
 
   AVFormatContext *in_av_format =  s_ctx->avFormat;
    AVFormatContext *out_av_format =  t_ctx->avFormat;
    AVStream *in_stream  = in_av_format->streams[s_ctx->video_index];
    AVStream *out_stream = out_av_format->streams[0];
   //
    AVDictionary* options = NULL;
    //封装 ffmp4
    av_dict_set(&options, "movflags", "frag_keyframe+empty_moov+default_base_moof",0);
    int result = avformat_write_header(out_av_format, &options);
 
 
    AVPacket temp_packet, out_packet;
    av_init_packet(&temp_packet);
    av_init_packet(&out_packet);
    int failCount = 0, data_count= 0;
 
    //max fial count
    const int failCountMax = 10;
 
    //raw video 帧
    AVFrame * out_frame= av_frame_alloc();
    //转换颜色后的帧
    AVFrame * convert_frame= av_frame_alloc();
    av_image_alloc(convert_frame->data, convert_frame->linesize, t_ctx->codecCtx->width, t_ctx->codecCtx->height,AV_PIX_FMT_YUV420P, 4);
    convert_frame->key_frame = out_frame->key_frame;
    convert_frame->height =  t_ctx->codecCtx->height;
    convert_frame->width = t_ctx->codecCtx->width;
    convert_frame->format = t_ctx->codecCtx->pix_fmt;
    convert_frame->pkt_pos = -1;
    int isFristEncodePkt = 0;
    ///
    while(1){
         //pull source
        result = av_read_frame(in_av_format, &temp_packet);
        if(result !=0){ continue; }
        if(temp_packet.stream_index != s_ctx->video_index){
             av_packet_unref(&temp_packet);
             continue;
        }
        data_count++;
        //新API 的解编码
        ///////////////////// decode ////////////////////////
        result = avcodec_send_packet(s_ctx->codecCtx,  &temp_packet);
        if (_global_print_debug){
            printf("push decode send_pack result=%d \n", result);
        }
        av_packet_unref(&temp_packet);
        result = avcodec_receive_frame(s_ctx->codecCtx,out_frame);
        if (_global_print_debug){
            printf("push decode receive_frame result=%d \n", result);
        }
        if(result == 0){
            //转换颜色
            result = sws_scale(ctx->img_convert_ctx, out_frame->data, out_frame->linesize,
                                    0, s_ctx->codecCtx->height, convert_frame->data,  convert_frame->linesize);
            //////////////// encode ////////////////////////
 
            //转PTS
            int64_t pts = av_rescale_q_rnd(out_frame->pts, in_stream->time_base, out_stream->time_base, (enum AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
            convert_frame->pts = pts;
 
//            char filename_[100];
//            sprintf(filename_, "/home/pi/data_count-%d.jpg",data_count);
//            save_avframe(convert_frame, filename_);
            result = avcodec_send_frame(t_ctx->codecCtx, convert_frame);
            result = avcodec_receive_packet(t_ctx->codecCtx, &out_packet);
   
            if (result == 0){
                isFristEncodePkt = 1;
 
                //push target
                result = av_write_frame (out_av_format, &out_packet);
//                result = av_interleaved_write_frame (out_av_format, &out_packet);
                av_packet_unref(&out_packet);
           
            }
        }// end for if(result == 0){
  
   }
    result = av_write_trailer(out_av_format);
    printf("push av_write_trailer result=%d,\n", result);
   return result;

踩坑指南

• 封装 ffmp4 需增加 flag av_dict_set(&options, "movflags", "frag_keyframe+empty_moov+default_base_moof",0);

• 输出FFMP4 没有宽高元素信息? 在Windows文件中也没有缩略图, 很明显缺少元数据, 能在Firefox的MSE API播放, 但在Chrome不能播放?

原因是没SPS, PPS信息!!! 跟这位老哥一模一样解决ffmpeg生成mp4文件不能正常预览的问题, mp4info工具打开则”open failed” 没有工具分析, 真恶心!

MP4Reader 工具可以打开分析; 另外自带的 ffprobe.exe工具也可以检查媒体信息, 灯下黑!

unsigned char sps_pps[] ={0x00,0x00,0x00,0x01,0x67,0x42,0xC0,0x1F,0xDA,0x03,0x20,0x4D,0xF9,0x61,0x00,0x00,0x03,0x00,0x01,0x00,0x00,0x03,0x00,0x28,0x8F,0x18,0x32,0xA0,
                0x00,0x00,0x00,0x01,0x68,0xCE,0x3C,0x80};
int size_of = sizeof (sps_pps);
codecpar->extradata_size = size_of;
 

SPS/PPS的问题

另 如果要求 open 编码器以后 AVCodecContext extradata 没存有 SPS,PPS 信息需要加上 AVCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER

if(pFormatCtx->oformat->flags & AVFMT_GLOBALHEADER) {
    pCodeCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER ;
}

但, 直播流不建议加 设置了 AV_CODEC_FLAG_GLOBAL_HEADER 就会导致 x264_param_ t参数中的 b_repeat_header 赋值为0 ?

b_repeat_headers 的意思是将SPS/PPS添加到每一个关键帧之前, 0代表不添加, 这样每个关键帧前面又没了SPS/PPS

H264 低延迟编码

ffmpeg-all

编码延迟的问题

AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
codec_ctx->time_base = (AVRational){1, 25};
//配置低延迟
av_opt_set(codec_ctx->priv_data, "preset", "ultrafast", 0);
av_opt_set(codec_ctx->priv_data, "tune","zerolatency",0);
av_opt_set(pCodeCtx->priv_data, "profile","baseline",0);
...
//avcodec_open2 打开编码器时 preset 选项 (OR?)
AVDictionary* options = NULL;
av_dict_set(&options,"preset","ultrafast",0);
ret = avcodec_open2(codec_ctx, codec,options);
   
... 
// 写帧时 使用 av_write_frame API; 相对于 av_interleaved_write_frame ,它直接将包写进Mux, 没有缓存和重新排序
result = av_write_frame (out_av_format, &out_packet);
 

lib x264 选项 H264-options

• --preset的参数主要调节编码速度和质量的平衡
• --tune的参数主要配合视频类型和视觉优化的参数, tune 的值有: film: 电影, 真人类型;
  animation: 动画;
  grain: 需要保留大量的grain时用;
  stillimage: 静态图像编码时使用;
  psnr: 为提高psnr做了优化的参数;
  ssim: 为提高ssim做了优化的参数;
  fastdecode: 可以快速解码的参数;
  zerolatency: 零延迟, 用在需要非常低的延迟的情况下, 比如电视电话会议的编码;
• profile baseline, 支持I/P 帧, 只支持无交错(Progressive)和CAVLC, 一般用于低阶或需要额外容错的应用 main, high, high10, high422, high444

N_FFmpeg

实例 读取摄像头 for win

命令行工具

要安装screen-capture-recorder, 否则查看device会提示没有;

screen-capture-recorder 下载

列出设备列表 ffmpeg -list_devices true -f dshow -i dummy

[dshow @ 00000211790f8c00] DirectShow video devices (some may be both video and audio devices)
[dshow @ 00000211790f8c00]  "Digieye"
[dshow @ 00000211790f8c00]     Alternative name "@device_pnp_\\?\usb#vid_eb1a&pid_2570&mi_00#6&327e6b33&0&0000#{65e8773d-8f56-11d0-a3b9-00a0c9223196}\global"

//获取指定视频采集设备支持的分辨率, 帧率和像素格式等属性
ffmpeg -list_options true -f dshow -i video="Digieye"
 
//录制为文件
ffmpeg   -f dshow -i video="Digieye" win_cam.mp4
 
//使用 ffplay 播放
ffplay -f dshow -i video="Digieye" 
 

C/C++ API

wcl_32 无人车视频项目, 移植到win测试性能, 记录与win平台的差异代码..

读取摄像头 参考

AVFormatContext *avFormat = avformat_alloc_context();
AVInputFormat *ifmt=av_find_input_format("dshow");
char *device_name = "video=Digieye" ;
//int ret = avformat_open_input(&avFormat, ctx->name, ifmt, 0);
int ret = avformat_open_input(&avFormat, device_name, ifmt, 0);
if (_global_print_info ){
    printf("source open_input= %d \n", ret);
}
 

windown API 的 socket 编程

windown socket api

通过 WSAGetLastError 获得错误码 win socket 错误码描述

int tcpClientInit(const char *ip, const uint16_t port, SOCKET *fd)
{
    WORD sockVersion = MAKEWORD(2,2);
    WSADATA data;
    if(WSAStartup(sockVersion, &data) != 0)
    {
        return -1;
    }
    //创建套接字
    *fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    //向服务器发起请求
    struct sockaddr_in sockAddr;
    memset(&sockAddr, 0, sizeof(sockAddr));  //每个字节都用0填充
    sockAddr.sin_family = PF_INET;
    sockAddr.sin_addr.s_addr = inet_addr(ip);
    sockAddr.sin_port = htons(port);
    int ret = connect(*fd, (SOCKADDR*)&sockAddr, sizeof(SOCKADDR));
    return ret;
}
 
 
int tcpClientClose(SOCKET sockfd)
{
   WSACleanup();
   return closesocket(sockfd);
}
 
int tcpClientRecv(int sockfd, char* buf, size_t len)
{
      return  recv(sockfd, buf, MAXBYTE, NULL);
}
 
int tcpClientSend(SOCKET sockfd, char* buf, size_t len)
{
     return send(sockfd, buf, len, NULL);
}

char data[255] ;
data[0] = '\0';
const char * TEMPLATE1 = "{ \"id\":\"";
const char * TEMPLATE2 = "\", \"type\": \"wcl\",\"token\": \"abc\"}";
strcat(data, TEMPLATE1);
strcat(data, ctx->id);
strcat(data,  TEMPLATE2);
size_t len =  strlen(data);