Quartz 4

FFmpeg 海康SDK流 转 浏览器 MSE

6 min read

N_FFmpeg

实例 海康SDK流 转 浏览器 MSE

转为 Media Source Extensions API支持的

最关键的是将海康sdk 预览回调的数据, 转为FMP4格式, Chrome, Firefox等浏览器原生 Media Source Extensions API 可支持的直播流

C++ 关键代码

关键代码

//内存输入数据
int fill_buffer_in(void * opaque,uint8_t *buf, int bufsize){
    WebSocketPlayer *instance = static_cast<WebSocketPlayer*>( opaque);
    callback_data * data =  instance->sDKSource2->takeDataPacketSync();
    memcpy( buf, data->data, data->size);//复制到容器 给 ffmpeg
    int size =  data->size;
    SDKSource2::destoryDataPacket(data);//销毁
    return size;
  }
 
 
//内存输出数据
int fill_buffer_out(void * opaque, uint8_t *buf, int bufsize){
    WebSocketPlayer *instance = static_cast<WebSocketPlayer*>( opaque);
//    callback_data * data = new callback_data;
//    data->size = bufsize;
//    memcpy( buf, data->data, data->size);
//    instance->onAVPacketOutData(data);
    instance->onOutData(buf, bufsize);//websocket 推送出去
   return bufsize;
}
 
 QVariantMap param;
    this->sDKSource2->initSource(param);
    int ret =-1;
    //////////////////////////////////////////////////////////////////////////
    //内存输入
    avFormat = avformat_alloc_context();
    unsigned char* in_buffer = (unsigned char*)av_malloc(40690);
    AVIOContext* in_avio = avio_alloc_context(in_buffer , 40690 ,0 , this, fill_buffer_in,nullptr,nullptr);
    assert( in_avio != NULL);
    avFormat->pb = in_avio;
     //open input
    if ( ( avformat_open_input(&avFormat, nullptr,nullptr, nullptr)  ) != 0) {
          qDebug()<<"Could not open input .";
           return ;
      }
    //获取流信息
    if((avformat_find_stream_info(avFormat, 0) )!=0){
         qDebug()<<"find stream fail,获取输入信息 失败";
    }
    //输出格式 看看
   // av_dump_format(avFormat, 0, "", 0);
    //流通道
    for(int i=0; i<avFormat->nb_streams; i++) {
        if(avFormat->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO){
            videoIndex = i;
             qDebug()<<"av index="<<i;
        }else if(avFormat->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO){
             qDebug()<<"ai index="<<i;
        }
    }
   //转码 // 内存输出
   unsigned char* out_buffer = (unsigned char*)av_malloc(40690);
   AVIOContext* out_avio = avio_alloc_context(out_buffer , 40690 , 1 , this, nullptr,fill_buffer_out,nullptr);
   assert( out_avio != NULL);
    AVOutputFormat *oformat = av_guess_format("mp4", nullptr, nullptr);
    ret = avformat_alloc_output_context2(&outAvFormatCtx,oformat,nullptr,nullptr);
    outAvFormatCtx->pb = out_avio;0.
    outAvFormatCtx->flags = AVFMT_FLAG_CUSTOM_IO;
   if (ret < 0) {
       qDebug()<<": alloc output ctx error 申请ctx创建出错";
       throw ("Could not create output context");
   }
    outAVstream = avformat_new_stream(outAvFormatCtx, avFormat->streams[videoIndex]->codec->codec);
    if (!outAVstream) {
        qDebug()<<" output stream channel create error 创建输出通道出错";
        throw ("Failed allocating output stream");
     }
     ret = avcodec_copy_context(outAVstream->codec, avFormat->streams[videoIndex]->codec);
    if (ret < 0) {
         qDebug()<<": copy context error 出错";
         throw ("Failed to copy context from input to output stream codec context");
    }
   //标记初值.
    outAVstream->codec->codec_tag = 0;
    if (outAvFormatCtx->oformat->flags & AVFMT_GLOBALHEADER)
        outAVstream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
   qDebug()<<"open out..";
    //open out
//    avio_open(&outAvFormatCtx->pb, "nothing",AVIO_FLAG_WRITE) ;
//      if (!outAvFormatCtx->pb ) {
//          qDebug()<<"错误: format->pd is NULL";
//           throw ("Error open ouput fail ");
//      }
    ///写头 FMP4 格式封装
    AVDictionary* options = nullptr;
    av_dict_set(&options, "movflags", "empty_moov+default_base_moof",0);
    av_dict_set_int(&options, "frag_duration", 200 * 1000, 0);
    ret = avformat_write_header(outAvFormatCtx, &options);//写头
      qDebug()<<"avformat_write_header   .";
    if (ret < 0) {
            char * errMesg = new char[512];
            av_strerror(ret,errMesg,512);
            std::cout<<" write header err:"<<errMesg<<std::endl;
            delete[]errMesg;
            throw ("Error occurred when write header");
        }
    //复用
     AVFrame *pFrame = av_frame_alloc();
     AVPacket packet;
     av_init_packet(&packet);
     int count= 0;
    qDebug()<<" push data is ready.";
    //
     while(runFlag) {
        ret = av_read_frame(avFormat, &packet);
        if(packet.stream_index != videoIndex){
           //qDebug()<<"非视频帧";
           av_packet_unref(&packet);
           continue;
        }
        count++;
        if(count %10 == 0){
               qDebug()<<" REV  video packet "<<count<<" count";
        }
        /* copy packet 转换PTS/DTS(Convert PTS/DTS)*/
       AVStream *in_stream, *out_stream;
       in_stream  = avFormat->streams[packet.stream_index];
       out_stream = outAvFormatCtx->streams[packet.stream_index];
       packet.pts = av_rescale_q_rnd(packet.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
       packet.dts = av_rescale_q_rnd(packet.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
       packet.duration = av_rescale_q(packet.duration, in_stream->time_base, out_stream->time_base);
       packet.pos = -1;
       //不解码 写原始内容
      ret = av_write_frame (outAvFormatCtx, &packet);//直接将包写进Mux, 没有缓存, 排序,  检验DTS
        //ret = av_interleaved_write_frame(outAvFormatCtx, &pkt);//这个 有缓存, 排序, 检验DTS 会导致延迟
       if (ret < 0) {
           char * errMesg = new char[256];
           av_strerror(ret,errMesg,256);
           std::cout<<" push fail "<<ret<<errMesg<<std::endl;
           delete[] errMesg;
           msleep(100);
       }
        av_packet_unref(&packet);
    }
//测试代码,  写尾释放 packet 等

html 前端

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="description" content="jMuxer - a simple javascript mp4 muxer for non-standard streaming communications protocol">
    <meta name="keywords" content="h264 player, mp4 player, mse, mp4 muxing, jmuxer, aac player">
    <title> stream demo</title>
</head>
<body>
<div id="container" style="margin: 0 auto; text-align: center;">	
	<div style="padding: 5px;">
		<button onclick="player()">player</button>
		<button onclick="fb()">-1S</button>
		<button onclick="fs()">+1S</button>
		<button onclick="debugg()">test</button>
	</div>
    <video style="border: 1px solid #333; max-width: 600px;height: 400px;"  
		controls autoplay poster="images/loader-thumb.jpg" id="player" muted></video>
</div>
<script>
var chunks = [];
var video = document.getElementById('player');
var mse = new (MediaSource || WebKitMediaSource)();
var sourceBuffer;
video.src = URL.createObjectURL(mse);
 
mse.addEventListener('sourceopen', onMediaSourceOpen);
function onMediaSourceOpen() {
//MIME = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"';
    sourceBuffer = mse.addSourceBuffer('video/mp4; codecs="avc1.4d401f"');
    sourceBuffer.addEventListener('updateend', addMoreBuffer);
    video.play();
}
function addMoreBuffer() {
    if (sourceBuffer.updating || !chunks.length) {
        return;
    }
    sourceBuffer.appendBuffer(chunks.shift());
}
 
 function player(){
    var socketURL = 'ws://localhost:8084';
	//var socketURL = 'ws://localhost:3265/open/api/we';
     var ws = new WebSocket(socketURL);
     ws.binaryType = 'arraybuffer';
     ws.addEventListener('message',function(event) {
        chunks.push(new Uint8Array(event.data));
        addMoreBuffer();
     });
 
     ws.addEventListener('error', function(e) {
        console.log('Socket Error');
     });
 }
function fs(){
	video.currentTime = video.currentTime+1
}
function fb(){
	video.currentTime = video.currentTime-1
}
function debugg(){
 console.log(video)
 //video.currentTime = sourceBuffer.buffered.end(0)
}
</script>
</body>
</html>

但是浏览器貌似有缓存… 延迟7,8秒, 可以通过video.currentTime控制, 但是还有2s左右延迟..

最终可以使用 video.currentTime = sourceBuffer.buffered.end(0) 直接设置跳到最新的缓存帧时间, 延迟在半秒左右!

Transclude of wcl-video-server.zip

转为 FLV.js 支持的

git hub flv.js

只需修改 oformat 格式, 并且 avformat_write_header 的 options去掉

AVOutputFormat *oformat =
        av_guess_format("flv", nullptr, nullptr);
        //av_guess_format("mp4", nullptr, nullptr);
ret = avformat_alloc_output_context2(&outAvFormatCtx,oformat,nullptr,nullptr);
 
 
AVDictionary* options = nullptr;
/* fmp4封装
av_dict_set(&options, "movflags", "empty_moov+default_base_moof",0);
av_dict_set_int(&options, "frag_duration", 0, 0);
av_dict_set_int(&options, "min_frag_duration", 0, 0);
av_dict_set_int(&options, "frag_size", 0, 0); */
////
ret = avformat_write_header(outAvFormatCtx, &options);//写头

videoTagDocment.currentTime = player.buffered.end(0) 可以通过 videoTagDocment.currentTime控制, 但是仍然有2s左右延迟..

这两个底层解码原理是一样的…

Graph View