Recherche avancée

Sur d’autres sites (7498)

• FFMPEG : AV out of sync when writing a part of a video to a new file

  30 mars 2017, par IT_Layman

  I’m developing a data preprocessing program for a computer vision project using FFMPEG and Face detection API. In this program, I need to extract the shots that contain human faces from a given input video file and output them into a new file. But when I played the output video file generated by that program, the video and audio track was out of sync. I think a possible reason is that the timestamp of video frame or audio frame is set incorrectly, but I can’t fix it by myself as I’m not very familiar with FFMPEG library, Please help me solving this out-of-sync issue.

  To simplify the code shown below, I have removed all face detection code and use an empty function called faceDetect to represent it instead.

  // ffmpegAPI.cpp : Defines the entry point for the console application.
  
  //
  
      #include "stdafx.h"
  
      #include <iostream>
  
  
  
      extern "C" {
  
      #include <libavutil></libavutil>opt.h>
  
      #include <libavcodec></libavcodec>avcodec.h>
  
      #include <libavformat></libavformat>avformat.h>
  
      #include <libavutil></libavutil>avutil.h>
  
      #include <libavutil></libavutil>channel_layout.h>
  
      #include <libavutil></libavutil>common.h>
  
      #include <libavutil></libavutil>imgutils.h>
  
      #include <libavutil></libavutil>mathematics.h>
  
      #include <libavutil></libavutil>samplefmt.h>
  
      #include <libavutil></libavutil>pixdesc.h>
  
      #include <libswscale></libswscale>swscale.h>
  
  
  
  }
  
  bool faceDetect(AVFrame *frame)
  
  {
  
      /*...*/
  
      return true;
  
  }
  
  int main(int argc, char **argv)
  
  {
  
      int64_t videoPts = 0, audioPts = 0;
  
      int samples_count = 0;
  
      AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
  
      AVOutputFormat *ofmt = NULL;
  
      AVPacket pkt;
  
      AVFrame *frame = NULL;
  
      int videoindex = -1; int audioindex = -1;
  
      double videoTime = DBL_MAX;
  
      const char *in_filename, *out_filename;
  
      int ret, i;
  
      in_filename = "C:\\input.flv";//Input file name
  
      out_filename = "C:\\output.avi";//Output file name
  
      av_register_all();
  
      //Open input file
  
      if ((ret = avformat_open_input(&amp;ifmt_ctx, in_filename, 0, 0)) &lt; 0) {
  
          fprintf(stderr, "Could not open input file '%s'", in_filename);
  
          goto end;
  
      }
  
      //Find input streams
  
      if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) &lt; 0) {
  
          fprintf(stderr, "Failed to retrieve input stream information");
  
          goto end;
  
      }
  
      //Retrive AV stream information
  
      for (i = 0; i &lt; ifmt_ctx->nb_streams; i++)
  
      {
  
          AVStream *stream;
  
          AVCodecContext *codec_ctx;
  
          stream = ifmt_ctx->streams[i];//Get current stream
  
          codec_ctx = stream->codec;//Get current stream codec
  
          if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO)
  
          {
  
              videoindex = i;//video stream index
  
          }
  
          else if (codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO)
  
          {
  
              audioindex = i;//audio stream index
  
          }
  
          if (videoindex == -1)//no video stream is found
  
          {
  
              printf("can't find video stream\n");
  
              goto end;
  
  
  
          }
  
      }
  
      av_dump_format(ifmt_ctx, 0, in_filename, 0);
  
      //Configure output
  
      avformat_alloc_output_context2(&amp;ofmt_ctx, NULL, NULL, out_filename);
  
      if (!ofmt_ctx) {
  
          fprintf(stderr, "Could not create output context\n");
  
          ret = AVERROR_UNKNOWN;
  
          goto end;
  
      }
  
      ofmt = ofmt_ctx->oformat;
  
      //Configure output streams
  
      for (i = 0; i &lt; ifmt_ctx->nb_streams; i++) {//Traversal input streams
  
          AVStream *in_stream = ifmt_ctx->streams[i];//Get current stream
  
          AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);//Create a corresponding output stream
  
          if (!out_stream) {
  
              fprintf(stderr, "Failed allocating output stream\n");
  
              ret = AVERROR_UNKNOWN;
  
              goto end;
  
          }
  
          //Copy codec from current input stream to corresponding output stream
  
          ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
  
          if (ret &lt; 0) {
  
              fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
  
              goto end;
  
          }
  
          if (i == videoindex)//Video stream
  
          {
  
              if (out_stream->codec->codec_id == AV_CODEC_ID_H264)
  
              {
  
                  out_stream->codec->me_range = 16;
  
                  out_stream->codec->max_qdiff = 4;
  
                  out_stream->codec->qmin = 10;
  
                  out_stream->codec->qmax = 51;
  
                  out_stream->codec->qcompress = 1;
  
  
  
              }
  
          }
  
          AVCodecContext *codec_ctx = out_stream->codec;
  
          if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO
  
              || codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
  
              //Find codec encoder
  
              AVCodec *encoder = avcodec_find_encoder(codec_ctx->codec_id);
  
              if (!encoder) {
  
                  av_log(NULL, AV_LOG_FATAL, "Necessary encoder not found\n");
  
                  ret = AVERROR_INVALIDDATA;
  
                  goto end;
  
              }
  
              //Open encoder
  
              ret = avcodec_open2(codec_ctx, encoder, NULL);
  
              if (ret &lt; 0) {
  
                  av_log(NULL, AV_LOG_ERROR, "Cannot open video encoder for stream #%u\n", i);
  
                  goto end;
  
              }
  
              out_stream->codec->codec_tag = 0;
  
              if (ofmt_ctx->oformat->flags &amp; AVFMT_GLOBALHEADER)
  
                  out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
  
          }
  
          //Open the decoder for input stream
  
          codec_ctx = in_stream->codec;
  
          if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO
  
              || codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
  
              ret = avcodec_open2(codec_ctx,
  
                  avcodec_find_decoder(codec_ctx->codec_id), NULL);
  
              if (ret &lt; 0) {
  
                  av_log(NULL, AV_LOG_ERROR, "Failed to open decoder for stream #%u\n", i);
  
              }
  
          }
  
      }
  
      av_dump_format(ofmt_ctx, 0, out_filename, 1);
  
      //Open output file for writing
  
      if (!(ofmt->flags &amp; AVFMT_NOFILE)) {
  
          ret = avio_open(&amp;ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
  
          if (ret &lt; 0) {
  
              fprintf(stderr, "Could not open output file '%s'", out_filename);
  
              goto end;
  
          }
  
      }
  
  
  
      //Write video header
  
      ret = avformat_write_header(ofmt_ctx, NULL);
  
      if (ret &lt; 0) {
  
          fprintf(stderr, "Error occurred when opening output file\n");
  
          goto end;
  
      }
  
      //Write frames in a loop
  
      while (1) {
  
          AVStream *in_stream, *out_stream;
  
          //Read one frame from the input file
  
          ret = av_read_frame(ifmt_ctx, &amp;pkt);
  
          if (ret &lt; 0)
  
              break;
  
          in_stream = ifmt_ctx->streams[pkt.stream_index];//Get current input stream
  
          out_stream = ofmt_ctx->streams[pkt.stream_index];//Get current output stream
  
          if (pkt.stream_index == videoindex)//video frame
  
          {
  
              int got_frame;
  
              frame = av_frame_alloc();
  
              if (!frame) {
  
                  ret = AVERROR(ENOMEM);
  
                  break;
  
              }
  
              //Readjust packet timestamp for decoding
  
              av_packet_rescale_ts(&amp;pkt,
  
                  in_stream->time_base,
  
                  in_stream->codec->time_base);
  
              //Decode video frame
  
              int len = avcodec_decode_video2(in_stream->codec, frame, &amp;got_frame, &amp;pkt);
  
              if (len &lt; 0)
  
              {
  
                  av_frame_free(&amp;frame);
  
                  av_log(NULL, AV_LOG_ERROR, "Decoding failed\n");
  
                  break;
  
              }
  
              if (got_frame)//Got a decoded video frame
  
              {
  
                  int64_t pts = av_frame_get_best_effort_timestamp(frame);
  
                  //determine if the frame image contains human face
  
                  bool result = faceDetect(frame);
  
                  if (result) //face contained
  
                  {
  
                      videoTime = pts* av_q2d(out_stream->time_base);
  
                      frame->pts = videoPts++;//Set pts of video frame
  
                      AVPacket enc_pkt;
  
                      av_log(NULL, AV_LOG_INFO, "Encoding video frame\n");
  
                      //Create packet for encoding
  
                      enc_pkt.data = NULL;
  
                      enc_pkt.size = 0;
  
                      av_init_packet(&amp;enc_pkt);
  
                      //Encoding frame
  
                      ret = avcodec_encode_video2(out_stream->codec, &amp;enc_pkt,
  
                          frame, &amp;got_frame);
  
                      av_frame_free(&amp;frame);
  
                      if (!(got_frame))
  
                          ret = 0;
  
                      /* Configure encoding properties */
  
                      enc_pkt.stream_index = videoindex;
  
                      av_packet_rescale_ts(&amp;enc_pkt,
  
                          out_stream->codec->time_base,
  
                          out_stream->time_base);
  
                      av_log(NULL, AV_LOG_DEBUG, "Muxing frame\n");
  
                      /* Write encoded frame */
  
                      ret = av_interleaved_write_frame(ofmt_ctx, &amp;enc_pkt);
  
                      if (ret &lt; 0)
  
                          break;
  
                  }
  
                  else //no face contained
  
                  {
  
                      //Set the videoTime as maximum double value, 
  
                      //making the corresponding audio frame not been processed
  
                      if (videoTime &lt; DBL_MAX)
  
                          videoTime = DBL_MAX;
  
                  }
  
  
  
              }
  
              else
  
              {
  
                  av_frame_free(&amp;frame);
  
              }
  
          }
  
          else//Audio frame
  
          {
  
              //Get current frame time
  
              double audioTime = pkt.pts * av_q2d(in_stream->time_base);
  
              if (audioTime >= videoTime)
  
              {//The current frame should be written into output file
  
                  int got_frame;
  
                  frame = av_frame_alloc();
  
                  if (!frame) {
  
                      ret = AVERROR(ENOMEM);
  
                      break;
  
                  }
  
                  //Readjust packet timestamp for decoding
  
                  av_packet_rescale_ts(&amp;pkt,
  
                      in_stream->time_base,
  
                      in_stream->codec->time_base);
  
                  //Decode audio frame
  
                  int len = avcodec_decode_audio4(in_stream->codec, frame, &amp;got_frame, &amp;pkt);
  
                  if (len &lt; 0)
  
                  {
  
                      av_frame_free(&amp;frame);
  
                      av_log(NULL, AV_LOG_ERROR, "Decoding failed\n");
  
                      break;
  
                  }
  
                  if (got_frame)//Got a decoded audio frame
  
                  {
  
                      //Set pts of audio frame
  
                      frame->pts = audioPts;
  
                      audioPts += frame->nb_samples;
  
                      AVPacket enc_pkt;
  
                      av_log(NULL, AV_LOG_INFO, "Encoding audio frame");
  
                      //Create packet for encoding
  
                      enc_pkt.data = NULL;
  
                      enc_pkt.size = 0;
  
                      av_init_packet(&amp;enc_pkt);
  
                      //Encode audio frame
  
                      ret = avcodec_encode_audio2(out_stream->codec, &amp;enc_pkt,
  
                          frame, &amp;got_frame);
  
                      av_frame_free(&amp;frame);
  
                      if (!(got_frame))
  
                          ret = 0;
  
                      /* Configure encoding properties */
  
                      enc_pkt.stream_index = audioindex;
  
                      av_packet_rescale_ts(&amp;enc_pkt,
  
                          out_stream->codec->time_base,
  
                          out_stream->time_base);
  
                      av_log(NULL, AV_LOG_DEBUG, "Muxing frame\n");
  
                      /* Write encoded frame */
  
                      ret = av_interleaved_write_frame(ofmt_ctx, &amp;enc_pkt);
  
                      if (ret &lt; 0)
  
                          break;
  
                  }
  
                  else //Shouldn't be written
  
                  {
  
                      av_frame_free(&amp;frame);
  
                  }
  
              }
  
          }
  
          av_packet_unref(&amp;pkt);
  
      }
  
      //Write video trailer
  
      av_write_trailer(ofmt_ctx);
  
  end://Clean up
  
      av_log(NULL, AV_LOG_INFO, "Clean up\n");
  
      av_frame_free(&amp;frame);
  
      for (i = 0; i &lt; ifmt_ctx->nb_streams; i++) {
  
          avcodec_close(ifmt_ctx->streams[i]->codec);
  
          if (ofmt_ctx &amp;&amp; ofmt_ctx->nb_streams > i &amp;&amp; ofmt_ctx->streams[i] &amp;&amp; ofmt_ctx->streams[i]->codec)
  
              avcodec_close(ofmt_ctx->streams[i]->codec);
  
      }
  
      avformat_close_input(&amp;ifmt_ctx);
  
      /* Close output file */
  
      if (ofmt_ctx &amp;&amp; !(ofmt_ctx->oformat->flags &amp; AVFMT_NOFILE))
  
          avio_closep(&amp;ofmt_ctx->pb);
  
      avformat_free_context(ofmt_ctx);
  
      if (ret &lt; 0 &amp;&amp; ret != AVERROR_EOF) {
  
          char buf[256];
  
          av_strerror(ret, buf, sizeof(buf));
  
          av_log(NULL, AV_LOG_ERROR, "Error occurred:%s\n", buf);
  
          system("Pause");
  
          return 1;
  
      }
  
      //Program end
  
      printf("The End.\n");
  
      system("Pause");
  
      return 0;
  
  }
  
  </iostream>

• Revision 5de9280ae9 : tx_skip mode for lossy coding This patch improves the non-transform coding mode

  23 novembre 2014, par hui su

  Changed Paths :
   Modify /vp9/common/vp9_blockd.h

  
   Modify /vp9/common/vp9_entropymode.c

  
   Modify /vp9/common/vp9_entropymode.h

  
   Modify /vp9/common/vp9_idct.c

  
   Modify /vp9/common/vp9_idct.h

  
   Modify /vp9/common/vp9_quant_common.h

  
   Modify /vp9/decoder/vp9_decodeframe.c

  
   Modify /vp9/decoder/vp9_decodemv.c

  
   Modify /vp9/encoder/vp9_bitstream.c

  
   Modify /vp9/encoder/vp9_dct.c

  
   Modify /vp9/encoder/vp9_encodeframe.c

  
   Modify /vp9/encoder/vp9_encodemb.c

  
   Modify /vp9/encoder/vp9_encodemb.h

  
   Modify /vp9/encoder/vp9_rdopt.c

  
  
  tx_skip mode for lossy coding

  This patch improves the non-transform coding mode. At this
  point, the coding gain on screen content videos is about
  12% for lossless, an 15% for lossy case.

  1. Encode tx_skip flags with context. Y tx_skip flag context is
  whether the prediction mode is inter or intra. UV flag context
  is Y flag.

  2. Transform skipping is less helpful when the Q-index is high.
  So it is enabled only when the Q-index is smaller than a
  threshold. Currently the threshold is set as 255 for intra blocks,
  and 0 for inter blocks.

  3. The shift of the prediction residue, when copying them to the
  coeff buffer, is set as 3 when the Q-index is larger than a
  threshold (currently set as 0), and 2 otherwise.

  Change-Id : I372973c7518cf385f6e542b22d0f803016e693b0

• Révision 23459 : Arrêter le traitement d’image dès que ça coince qq part.

  6 mars 2017, par esj@rezo.net