Libav AVFrame do OpenCv Mat do konwersji AVPacket

Question

Jestem nowy w libav i piszę oprogramowanie do manipulacji wideo, które wykorzystuje opencv jako serce. Co zrobiłem to krótko, jak poniżej:

1- czytać pakiet wideo

2- dekodowania pakietu do AVFrame

3- nawróconego AVFrame do CV Mat

4- manipulować maty

5- przekonwertować CV Mat do AVFrame

6- kodują AVFrame do AVPacket

7- zapisu pakietu

8- goto 1

I odczytu poradnik Dranger w http://dranger.com/ffmpeg/tutorial01.html i również przykład decoding_encoding. Mogę odczytać wideo, wyodrębnić klatki wideo i przekonwertować je na matę CV. Mój problem zaczyna się od konwersji z cv Mat na AVFrame i zakodowania go do AVPacket.

Czy możesz mi w tym pomóc?

Oto mój kod:

int main(int argc, char **argv) 
{ 
AVOutputFormat *ofmt = NULL; 
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL; 
AVPacket pkt; 
AVCodecContext *pCodecCtx = NULL; 
AVCodec   *pCodec = NULL; 
AVFrame   *pFrame = NULL; 
AVFrame   *pFrameRGB = NULL; 
int videoStream=-1; 
int audioStream=-1; 
int    frameFinished; 
int    numBytes; 
uint8_t   *buffer = NULL; 
struct SwsContext *sws_ctx = NULL; 
FrameManipulation *mal_frame; 

const char *in_filename, *out_filename; 
int ret, i; 
if (argc < 3) { 

    printf("usage: %s input output\n" 
      "API example program to remux a media file with libavformat and libavcodec.\n" 
      "The output format is guessed according to the file extension.\n" 
      "\n", argv[0]); 
    return 1; 
} 
in_filename = arg[1]; 
out_filename = arg[2]; 
av_register_all(); 
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) { 
    fprintf(stderr, "Could not open input file '%s'", in_filename); 
    goto end; 
} 

if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) { 
    fprintf(stderr, "Failed to retrieve input stream information"); 
    goto end; 
} 

av_dump_format(ifmt_ctx, 0, in_filename, 0); 
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename); 

if (!ofmt_ctx) { 
    fprintf(stderr, "Could not create output context\n"); 
    ret = AVERROR_UNKNOWN; 
    goto end; 
} 

ofmt = ofmt_ctx->oformat; 

for (i = 0; i < ifmt_ctx->nb_streams; i++) { 
    AVStream *in_stream = ifmt_ctx->streams[i]; 
    AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec); 

    if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO && 
     videoStream < 0) { 
      videoStream=i; 
    } 

    if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO && 
     audioStream < 0) { 
      audioStream=i; 
    } 

    if (!out_stream) { 
     fprintf(stderr, "Failed allocating output stream\n"); 
     ret = AVERROR_UNKNOWN; 
     goto end; 
    } 

    ret = avcodec_copy_context(out_stream->codec, in_stream->codec); 

    if (ret < 0) { 
     fprintf(stderr, "Failed to copy context from input to output stream codec context\n"); 
     goto end; 
    } 

    out_stream->codec->codec_tag = 0; 

    if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER) 
     out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; 
} 

pCodec=avcodec_find_decoder(ifmt_ctx->streams[videoStream]->codec->codec_id); 
pCodecCtx = avcodec_alloc_context3(pCodec); 

if(avcodec_copy_context(pCodecCtx, ifmt_ctx->streams[videoStream]->codec) != 0) { 
    fprintf(stderr, "Couldn't copy codec context"); 
    return -1; // Error copying codec context 
} 

// Open codec 
if(avcodec_open2(pCodecCtx, pCodec, NULL)<0) 
    return -1; // Could not open codec 

// Allocate video frame 
pFrame=av_frame_alloc(); 

// Allocate an AVFrame structure 
pFrameRGB=av_frame_alloc(); 

// Determine required buffer size and allocate buffer 
numBytes=avpicture_get_size(AV_PIX_FMT_RGB24, ifmt_ctx->streams[videoStream]->codec->width, 
       ifmt_ctx->streams[videoStream]->codec->height); 

buffer=(uint8_t *)av_malloc(numBytes*sizeof(uint8_t)); 

// Assign appropriate parts of buffer to image planes in pFrameRGB 
// Note that pFrameRGB is an AVFrame, but AVFrame is a superset 
// of AVPicture 
avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24, 
     ifmt_ctx->streams[videoStream]->codec->width, ifmt_ctx->streams[videoStream]->codec->height); 

av_dump_format(ofmt_ctx, 0, out_filename, 1); 

if (!(ofmt->flags & AVFMT_NOFILE)) { 
    ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE); 
    if (ret < 0) { 
     fprintf(stderr, "Could not open output file '%s'", out_filename); 
     goto end; 
    } 
} 

ret = avformat_write_header(ofmt_ctx, NULL); 
if (ret < 0) { 
    fprintf(stderr, "Error occurred when opening output file\n"); 
    goto end; 
} 

// Assign appropriate parts of buffer to image planes in pFrameRGB 
// Note that pFrameRGB is an AVFrame, but AVFrame is a superset 
// of AVPicture 

avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24, 
        ifmt_ctx->streams[videoStream]->codec->width, 
        ifmt_ctx->streams[videoStream]->codec->height); 

// initialize SWS context for software scaling 
sws_ctx = sws_getContext(
      ifmt_ctx->streams[videoStream]->codec->width, 
      ifmt_ctx->streams[videoStream]->codec->height, 
      ifmt_ctx->streams[videoStream]->codec->pix_fmt, 
      ifmt_ctx->streams[videoStream]->codec->width, 
      ifmt_ctx->streams[videoStream]->codec->height, 
      AV_PIX_FMT_BGR24, 
      SWS_BICUBIC, 
      NULL, 
      NULL, 
      NULL 
      ); 
// Loop through packets 
while (1) { 

    AVStream *in_stream, *out_stream; 
    ret = av_read_frame(ifmt_ctx, &pkt); 
    if(pkt.stream_index==videoStream) 

    // Decode video frame 
     avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &pkt); 

     if(frameFinished) { 
       sws_scale(sws_ctx, (uint8_t const * const *)pFrame->data, 
       pFrame->linesize, 0, pCodecCtx->height, 
       pFrameRGB->data, pFrameRGB->linesize); 
       cv::Mat img= mal_frame->process(
          pFrameRGB,pFrame->width,pFrame->height); 
/* My problem is Here ------------*/ 


    avpicture_fill((AVPicture*)pFrameRGB, 
        img.data, 
        PIX_FMT_BGR24, 
        outStream->codec->width, 
        outStream->codec->height); 

    pFrameRGB->width = ifmt_ctx->streams[videoStream]->codec->width; 
    pFrameRGB->height = ifmt_ctx->streams[videoStream]->codec->height; 

      avcodec_encode_video2(ifmt_ctx->streams[videoStream]->codec , 
                &pkt , pFrameRGB , &gotPacket); 
/* 
I get this error 
[swscaler @ 0x14b58a0] bad src image pointers 
[swscaler @ 0x14b58a0] bad src image pointers 
*/ 

/* My Problem Ends here ---------- */ 

    } 

    if (ret < 0) 

     break; 

    in_stream = ifmt_ctx->streams[pkt.stream_index]; 

    out_stream = ofmt_ctx->streams[pkt.stream_index]; 



    //log_packet(ifmt_ctx, &pkt, "in"); 

    /* copy packet */ 

    pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, 

           AV_ROUND_NEAR_INF); 



    pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF); 

    pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base); 

    pkt.pos = -1; 

    log_packet(ofmt_ctx, &pkt, "out"); 

    ret = av_interleaved_write_frame(ofmt_ctx, &pkt); 

    if (ret < 0) { 

     fprintf(stderr, "Error muxing packet\n"); 

     break; 

    } 

    av_free_packet(&pkt); 

} 

av_write_trailer(ofmt_ctx); 

end: 

avformat_close_input(&ifmt_ctx); 

/* close output */ 

if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE)) 

    avio_closep(&ofmt_ctx->pb); 

avformat_free_context(ofmt_ctx); 

if (ret < 0 && ret != AVERROR_EOF) { 

    return 1; 

} 

return 0; 

} 

Kiedy uruchomić ten kod, pojawia się nieznany błąd krytyczny w tej części:

/* My problem is Here ------------*/ 


    avpicture_fill((AVPicture*)pFrameRGB, 
        img.data, 
        PIX_FMT_BGR24, 
        outStream->codec->width, 
        outStream->codec->height); 

    pFrameRGB->width = ifmt_ctx->streams[videoStream]->codec->width; 
    pFrameRGB->height = ifmt_ctx->streams[videoStream]->codec->height; 

      avcodec_encode_video2(ifmt_ctx->streams[videoStream]->codec , 
                &pkt , pFrameRGB , &gotPacket); 
/* 
I get this error 
[swscaler @ 0x14b58a0] bad src image pointers 
[swscaler @ 0x14b58a0] bad src image pointers 
*/ 

/* My Problem Ends here ---------- */ 

Oto gdzie chcę przekonwertować z powrotem cv matę AVFrame i zakodować go do AVPacket. Doceniam twoją pomoc.

Answer 1

Po przeczytaniu kilku przykładów, przeczytaniu kodu źródłowego i niektórym pomaga ludziom oferowanym, udało mi się uruchomić kod. Użyłem transkodowania i kodowania przykładów i pomieszałem je. Here is my code

Oto najważniejsze: 1- libswscale należy użyć do konwertowania AVFrame z wymaganym formatem pakietu, który ma być wprowadzony do macierzy OpenCV. Aby to zrobić, możemy zdefiniować

struct SwsContext *sws_ctx = NULL; 
sws_ctx = sws_getContext(pCodecCtx->width, 
      pCodecCtx->height, 
      pCodecCtx->pix_fmt, 
      pCodecCtx->width, 
      pCodecCtx->height, 
      AV_PIX_FMT_BGR24, 
      SWS_BICUBIC, 
      NULL, 
      NULL, 
      NULL 
      ); 

Aby przekonwertować z powrotem Mat OpenCV do AVFrame, należy ponownie użyć swscale i tłumaczyć OpenCV BGR format ramki na YUV. Tak, mogę to zrobić:

struct SwsContext *sws_ctx_bgr_yuv = NULL; 
sws_ctx_bgr_yuv = sws_getContext(pCodecCtx->width, 
           pCodecCtx->height, 
           AV_PIX_FMT_BGR24, 
           pCodecCtx->width, 
           pCodecCtx->height, 
           pCodecCtx->pix_fmt //AV_PIX_FMT_YUV420p 
           ,0,0,NULL,NULL); 

I tu jest pętla ramka odczytu/dekodowanie/kodowanie:

while (1) { 
    if ((ret = av_read_frame(ifmt_ctx, &packet)) < 0) 
     break; 
    stream_index = packet.stream_index; 
    type = ifmt_ctx->streams[packet.stream_index]->codec->codec_type; 
    av_log(NULL, AV_LOG_DEBUG, "Demuxer gave frame of stream_index %u\n", 
      stream_index); 
    if (filter_ctx[stream_index].filter_graph) { 
     av_log(NULL, AV_LOG_DEBUG, "Going to reencode&filter the frame\n"); 
     frame = av_frame_alloc(); 
     if (!frame) { 
      ret = AVERROR(ENOMEM); 
      break; 
     } 
     av_packet_rescale_ts(&packet, 
          ifmt_ctx->streams[stream_index]->time_base, 
          ifmt_ctx->streams[stream_index]->codec->time_base); 
     dec_func = (type == AVMEDIA_TYPE_VIDEO) ? avcodec_decode_video2 : 
      avcodec_decode_audio4; 
     ret = dec_func(ifmt_ctx->streams[stream_index]->codec, frame, 
       &got_frame, &packet); 
     if (ret < 0) { 
      av_frame_free(&frame); 
      av_log(NULL, AV_LOG_ERROR, "Decoding failed\n"); 
      break; 
     } 
     if (got_frame) { 
      if(stream_index==video_index){ 

       sws_scale(sws_ctx, (uint8_t const * const *)frame->data, 
         frame->linesize, 0, pCodecCtx->height, 
         pFrameRGB->data, pFrameRGB->linesize); 
    /*------------------------------------------------------------------------ 
    /* Frame converts to opencv Mat 
    /*------------------------------------------------------------------------*/ 
       cv::Mat img(frame->height,frame->width,CV_8UC3,pFrameRGB->data[0]); 
       img=manipulate_image(img); //this is opencv Mat, do whatever you want, but don't change its dimensions and format 
    //manipulate_function can be considered as as simple as blurring 
       const int stride[] = {img.step[0] }; 
    /* opencv Mat converts back to AVFrame   */ 
       sws_scale(sws_ctx_bgr_yuv, &img.data, stride, 0, img.rows, frame->data, frame->linesize); 

      } 
      frame->pts = av_frame_get_best_effort_timestamp(frame); 
    /* AVFrame re-encodes to AVPacket and will be sent to encoder */ 
      ret = filter_encode_write_frame(frame, stream_index); 
      av_frame_free(&frame); 

      if (ret < 0) 
       goto end; 
     } else { 
      av_frame_free(&frame); 
     } 
    } else { 
     /* remux this frame without reencoding */ 
     av_packet_rescale_ts(&packet, 
          ifmt_ctx->streams[stream_index]->time_base, 
          ofmt_ctx->streams[stream_index]->time_base); 
     ret = av_interleaved_write_frame(ofmt_ctx, &packet); 
     if (ret < 0) 
      goto end; 
    } 
    av_free_packet(&packet); 
}