Recherche avancée

Sur d’autres sites (9595)

• FFmpeg encoding in c

  25 décembre 2012, par barisatbas

  I have been working on a project about video summarization on android platfrom. and I am stuck in encoding. I think ;
  first I must convert my frame into RGB Frame, then convert that RGB FRame into YUV Frame. Then encode the frame. After this operations, The output video was so weird. I think I missed something. Here is my las optimized code. Maybe someone has an experiement in this subject.

  Its syntax is changed according to android ndk syntax :

  jint Java_com_test_Test_encodeVideo(JNIEnv* env, jobject javaThis) 
  
  { 
  
       char *flname, *err, *info; 
  
       AVCodec *codec; 
  
       AVCodecContext *c= NULL; 
  
       int i,out_size, size, x, y,z, outbuf_size; 
  
       int frameCount=99; 
  
       FILE *f; 
  
       AVFrame *picture, *yuvFrame; 
  
       uint8_t *outbuf, *picture_buf; 
  
       PPMImage *img; 
  
       const char *destfilename = "/sdcard/new.mp4"; 
  
       int             numBytes; 
  
       uint8_t         *buffer; 
  
  
  
       av_register_all(); 
  
       // must be called before using avcodec lib 
  
       avcodec_init(); 
  
  
  
       // register all the codecs 
  
       avcodec_register_all(); 
  
       log_message("Video encoding\n"); 
  
  
  
       // find the H263 video encoder 
  
       codec = avcodec_find_encoder(CODEC_ID_H263); 
  
       if (!codec) { 
  
           sprintf(err, "codec not found\n"); 
  
           log_message(err); 
  
       } 
  
  
  
       c= avcodec_alloc_context(); 
  
       picture= avcodec_alloc_frame(); 
  
       yuvFrame= avcodec_alloc_frame(); 
  
  
  
       // get first ppm context. it is because I need width and height values. 
  
       img = getPPM("/sdcard/frame1.ppm"); 
  
  
  
       c->bit_rate = 400000; 
  
       // resolution must be a multiple of two 
  
  
  
       c->width = img->x; 
  
       c->height = img->y; 
  
       free(img); 
  
       // frames per second 
  
       c->time_base= (AVRational){1,25}; 
  
       c->gop_size = 10; // emit one intra frame every ten frames 
  
       //c->max_b_frames=1; 
  
       c->pix_fmt = PIX_FMT_YUV420P; 
  
       // open it 
  
       if (avcodec_open(c, codec) < 0){ 
  
      log_message("codec couldn't open"); 
  
      return -1; 
  
       } 
  
       //destfilename = (*env)->GetStringUTFChars(env, dst, 0); 
  
       f = fopen(destfilename, "wb"); 
  
       log_message(destfilename); 
  
       if (!f) { 
  
           sprintf(err, "could not open %s", destfilename); 
  
           log_message(err); 
  
       } 
  
  
  
       log_message("after destination file opening"); 
  
       // alloc image and output buffer 
  
       outbuf_size = 100000; 
  
       outbuf = malloc(outbuf_size); 
  
       size = c->width * c->height; 
  
       picture_buf = malloc(size * 3); // size for RGB 
  
       picture->data[0] = picture_buf; 
  
       picture->data[1] = picture->data[0] + size; 
  
       picture->data[2] = picture->data[1] + size / 4; 
  
       picture->linesize[0] = c->width; 
  
       picture->linesize[1] = c->width / 2; 
  
       picture->linesize[2] = c->width / 2; 
  
  
  
       numBytes=avpicture_get_size(PIX_FMT_YUV420P, c->width, 
  
                c->height); 
  
  
  
       buffer=malloc(numBytes); 
  
  
  
            // Assign appropriate parts of buffer to image planes in FrameYUV 
  
       avpicture_fill((AVPicture *)yuvFrame, buffer, PIX_FMT_YUV420P, 
  
                c->width, c->height); 
  
       // encode the video 
  
       log_message("before for loop"); 
  
       for(z=1;zsdcard/frame%d.ppm",z); 
  
                           // read the ppm file 
  
                           img = getPPM(flname); 
  
                           picture->data[0] = img->data; 
  
                           // convert the rgb frame into yuv frame 
  
                           rgb2yuv(picture,yuvFrame,c); 
  
                           log_message("translation completed."); 
  
                           // encode the image 
  
                           out_size = avcodec_encode_video(c, outbuf, outbuf_size, yuvFrame); 
  
                           sprintf(info,"encoding frame %3d (size=%5d)\n", z, out_size); 
  
                           log_message(info); 
  
                           fwrite(outbuf, 1, out_size, f); 
  
                           free(img); 
  
       } 
  
  
  
       // get the delayed frames 
  
       for(; out_size; i++) { 
  
           //fflush(stdout); 
  
           out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL); 
  
           sprintf(info,"write frame %3d (size=%5d)\n", i, out_size); 
  
               log_message(info); 
  
           fwrite(outbuf, 1, out_size, f); 
  
       } 
  
  
  
       // add sequence end code to have a real mpeg file 
  
       outbuf[0] = 0x00; 
  
       outbuf[1] = 0x00; 
  
       outbuf[2] = 0x01; 
  
       outbuf[3] = 0xb7; 
  
       fwrite(outbuf, 1, 4, f); 
  
       fclose(f); 
  
       free(picture_buf); 
  
       free(outbuf); 
  
  
  
       avcodec_close(c); 
  
       av_free(c); 
  
       av_free(picture); 
  
       av_free(yuvFrame); 
  
  } 
  
  
  
  int rgb2yuv(AVFrame *frameRGB, AVFrame *frameYUV, AVCodecContext *c) 
  
  { 
  
          char *err; 
  
          static struct SwsContext *img_convert_ctx; 
  
  
  
  
  
          log_message("conversion starts"); 
  
          // Convert the image into YUV format from RGB format 
  
          if(img_convert_ctx == NULL) { 
  
                  int w = c->width; 
  
                  int h = c->height; 
  
  
  
                  img_convert_ctx = sws_getContext(w, h, PIX_FMT_RGB24,w, h, c->pix_fmt, SWS_BICUBIC,NULL, NULL, NULL); 
  
  
  
                  if(img_convert_ctx == NULL) { 
  
                          sprintf(err, "Cannot initialize the conversion context!\n"); 
  
                          log_message(err); 
  
                          return -1; 
  
                  } 
  
          } 
  
    int ret = sws_scale(img_convert_ctx,frameRGB->data, frameRGB->linesize , 0,c->height,frameYUV->data, frameYUV->linesize ); 
  
          return; 
  
  }

• FFMPEG C api h.264 encoding / MPEG2 ts streaming problems

  3 mars 2015, par ccoral

  Class prototype is as follows :

  #ifndef _FULL_MOTION_VIDEO_STREAM_H_
  
  #define _FULL_MOTION_VIDEO_STREAM_H_
  
  
  
  #include <memory>
  
  #include <string>
  
  
  
  #ifndef INT64_C
  
  # define INT64_C(c) (c ## LL)
  
  # define UINT64_C(c) (c ## ULL)
  
  #endif
  
  
  
  extern "C" 
  
  {
  
      #include "libavutil/opt.h"
  
      #include "libavcodec/avcodec.h"
  
      #include "libavutil/channel_layout.h"
  
      #include "libavutil/common.h"
  
      #include "libavutil/imgutils.h"
  
      #include "libavutil/mathematics.h"
  
      #include "libavutil/samplefmt.h"
  
      #include "libavformat/avformat.h"
  
  
  
      #include <libavutil></libavutil>timestamp.h>
  
      #include <libswscale></libswscale>swscale.h>
  
      #include <libswresample></libswresample>swresample.h>
  
  }
  
  
  
  class FMVStream 
  
  {
  
      public:
  
          struct OutputStream 
  
          {
  
              OutputStream() :
  
              st(0),
  
              next_pts(0),
  
              samples_count(0),
  
              frame(0),
  
              tmpFrame(0),
  
              sws_ctx(0)
  
              {
  
              }
  
  
  
              AVStream *st;
  
  
  
              /* pts of the next frame that will be generated */
  
              int64_t next_pts;
  
              int samples_count;
  
  
  
              AVFrame *frame;
  
              AVFrame *tmpFrame;
  
  
  
              struct SwsContext *sws_ctx;
  
          };
  
  
  
          ///
  
          /// Constructor
  
          ///
  
          FMVStream();
  
  
  
          ///
  
          /// Destructor
  
          ///
  
          ~FMVStream();
  
  
  
          ///
  
          /// Frame encoder helper function
  
          ///
  
          /// Encodes a raw RGB frame into the transport stream
  
          ///
  
          int EncodeFrame(uint8_t* frame);
  
  
  
          ///
  
          /// Frame width setter
  
          ///
  
          void setFrameWidth(int width);
  
  
  
          ///
  
          /// Frame width getter
  
          ///
  
          int getFrameWidth() const;
  
  
  
          ///
  
          /// Frame height setter
  
          ///
  
          void setFrameHeight(int height);
  
  
  
          ///
  
          /// Frame height getter
  
          ///
  
          int getFrameHeight() const;
  
  
  
          ///
  
          /// Stream address setter
  
          ///
  
          void setStreamAddress(const std::string&amp; address);
  
  
  
          ///
  
          /// Stream address getter
  
          ///
  
          std::string getStreamAddress() const;
  
  
  
      private:
  
  
  
          ///
  
          /// Video Stream creation
  
          ///
  
          AVStream* initVideoStream(AVFormatContext* oc);
  
  
  
          ///
  
          /// Raw frame transcoder
  
          ///
  
          /// This will convert the raw RGB frame to a raw YUV frame necessary for h.264 encoding
  
          ///
  
          void CopyFrameData(uint8_t* src_frame);
  
  
  
          ///
  
          /// Video frame allocator
  
          ///
  
          AVFrame* AllocPicture(PixelFormat pix_fmt, int width, int height);
  
  
  
          ///
  
          /// Debug print helper function
  
          ///
  
          void print_sdp(AVFormatContext **avc, int n);
  
  
  
          ///
  
          /// Write the frame to the stream
  
          ///
  
          int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt);
  
  
  
          ///
  
          /// initialize the frame data
  
          ///
  
          void initFrame();
  
  
  
          // formatting data needed for output streaming and the output container (MPEG 2 TS)
  
          AVOutputFormat* format;
  
          AVFormatContext* format_ctx;
  
  
  
          // structure container for our video stream
  
          OutputStream stream;
  
  
  
          AVIOContext* io_ctx;
  
  
  
          std::string streamFilename;
  
  
  
          int frameWidth;
  
          int frameHeight;
  
  };
  
  
  
  #endif
  
  </string></memory>

  This block starts the class declaration.

  #include "FullMotionVideoStream.h"
  
  
  
  #include <stdexcept>
  
  #include <iostream>
  
  
  
  FMVStream::FMVStream()
  
      : format(0),
  
      format_ctx(0),
  
      stream(),
  
      io_ctx(0),
  
      streamFilename("test.mpeg"),
  
      frameWidth(640),
  
      frameHeight(480)
  
  {
  
      // Register all formats and codecs
  
      av_register_all();
  
      avcodec_register_all();
  
  
  
      // Init networking
  
      avformat_network_init();
  
  
  
      // Find format
  
      this->format = av_guess_format("mpegts", NULL, NULL);
  
  
  
      // allocate the AVFormatContext
  
      this->format_ctx = avformat_alloc_context();
  
  
  
      if (!this->format_ctx) 
  
      {
  
          throw std::runtime_error("avformat_alloc_context failed");
  
      }
  
  
  
      this->format_ctx->oformat = this->format;
  
      //sprintf_s(this->format_ctx->filename, sizeof(this->format_ctx->filename), "%s", this->streamFilename.c_str());
  
  
  
      this->stream.st = initVideoStream(this->format_ctx);
  
  
  
      this->initFrame();
  
  
  
      // Allocate AVIOContext
  
      int ret = avio_open(&amp;this->io_ctx, this->streamFilename.c_str(), AVIO_FLAG_WRITE);
  
  
  
      if (ret != 0) 
  
      {
  
          throw std::runtime_error("avio_open failed");
  
      }
  
  
  
      this->format_ctx->pb = this->io_ctx;
  
  
  
      // Print some debug info about the format
  
      av_dump_format(this->format_ctx, 0, NULL, 1);
  
  
  
      // Begin the output by writing the container header
  
      avformat_write_header(this->format_ctx, NULL);
  
  
  
      AVFormatContext* ac[] = { this->format_ctx };
  
      print_sdp(ac, 1);
  
  }
  
  
  
  FMVStream::~FMVStream()
  
  {
  
      av_write_trailer(this->format_ctx);
  
      avcodec_close(this->stream.st->codec);
  
  
  
      avio_close(io_ctx);
  
  
  
      avformat_free_context(this->format_ctx);
  
  
  
      av_frame_free(&amp;this->stream.frame);
  
      av_free(this->format);
  
  }
  
  
  
  AVFrame* FMVStream::AllocPicture(PixelFormat pix_fmt, int width, int height)
  
  {
  
      // Allocate a frame
  
      AVFrame* frame = av_frame_alloc();
  
  
  
      if (frame == nullptr)
  
      {
  
          throw std::runtime_error("avcodec_alloc_frame failed");
  
      }
  
  
  
      if (av_image_alloc(frame->data, frame->linesize, width, height, pix_fmt, 1) &lt; 0)
  
      {
  
          throw std::runtime_error("av_image_alloc failed");
  
      }
  
  
  
      frame->width = width;
  
      frame->height = height;
  
      frame->format = pix_fmt;
  
  
  
      return frame;
  
  }
  
  
  
  void FMVStream::print_sdp(AVFormatContext **avc, int n)
  
  {
  
      char sdp[2048];
  
      av_sdp_create(avc, n, sdp, sizeof(sdp));
  
      printf("SDP:\n%s\n", sdp);
  
      fflush(stdout);
  
  }
  
  
  
  AVStream* FMVStream::initVideoStream(AVFormatContext *oc)
  
  {
  
      AVStream* st = avformat_new_stream(oc, NULL);
  
  
  
      if (st == nullptr) 
  
      {
  
          std::runtime_error("Could not alloc stream");
  
      }
  
  
  
      AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_H264);
  
  
  
      if (codec == nullptr) 
  
      {
  
          throw std::runtime_error("couldn't find mpeg2 encoder");
  
      }
  
  
  
      st->codec = avcodec_alloc_context3(codec);
  
  
  
      st->codec->codec_id = AV_CODEC_ID_H264;
  
      st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
  
      st->codec->bit_rate = 400000;
  
  
  
      st->codec->width = this->frameWidth;
  
      st->codec->height = this->frameHeight;
  
  
  
      st->time_base.num = 1;
  
      st->time_base.den = 30;
  
  
  
      st->codec->framerate.num = 1;
  
      st->codec->framerate.den = 30;
  
  
  
      st->codec->max_b_frames = 2;
  
      st->codec->gop_size = 12;
  
      st->codec->pix_fmt = PIX_FMT_YUV420P;
  
  
  
      st->id = oc->nb_streams - 1;
  
  
  
      if (oc->oformat->flags &amp; AVFMT_GLOBALHEADER)
  
      {
  
          st->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
  
      }
  
  
  
      // option setup for the codec
  
      av_opt_set(st->codec->priv_data, "profile", "baseline", AV_OPT_SEARCH_CHILDREN);
  
  
  
      if (avcodec_open2(st->codec, codec, NULL) &lt; 0)
  
      {
  
          throw std::runtime_error("avcodec_open failed");
  
      }
  
  
  
      return st;
  
  }
  
  
  
  void FMVStream::initFrame()
  
  {
  
      // Allocate a tmp frame for converting our raw RGB data to YUV for encoding
  
      this->stream.tmpFrame = this->AllocPicture(PIX_FMT_RGB24, this->frameWidth, this->frameHeight);
  
  
  
      // Allocate a main frame
  
      this->stream.frame = this->AllocPicture(PIX_FMT_YUV420P, this->frameWidth, this->frameHeight);
  
  }
  
  </iostream></stdexcept>

  This block is attempting to convert from the raw RGB to our needed YUV format for h.264 encoding.

  void FMVStream::CopyFrameData(uint8_t* data)
  
  {
  
      // fill image with our raw RGB data
  
      //avpicture_alloc((AVPicture*)this->stream.tmpFrame, PIX_FMT_RGB24, this->stream.st->codec->width, this->stream.st->codec->height);
  
  
  
      int numBytes = avpicture_get_size(PIX_FMT_RGB24, this->stream.st->codec->width, this->stream.st->codec->height);
  
  
  
      uint8_t* buffer = (uint8_t*) av_malloc(numBytes * sizeof(uint8_t));
  
  
  
      avpicture_fill((AVPicture*)this->stream.tmpFrame, buffer, PIX_FMT_RGB24, this->stream.st->codec->width, this->stream.st->codec->height);
  
  
  
      for (int y = 0; y < this->stream.st->codec->height; y++) 
  
      {
  
          for (int x = 0; x < this->stream.st->codec->width; x++) 
  
          {
  
              int offset = 3 * (x + y * this->stream.st->codec->width);
  
              this->stream.tmpFrame->data[0][offset + 0] = data[x + y * this->stream.st->codec->width]; // R
  
              this->stream.tmpFrame->data[0][offset + 1] = data[x + y * this->stream.st->codec->width + 1]; // G
  
              this->stream.tmpFrame->data[0][offset + 2] = data[x + y * this->stream.st->codec->width + 2]; // B
  
          }
  
      }
  
  
  
      // convert the RGB frame to a YUV frame using the sws Context
  
      this->stream.sws_ctx = sws_getContext(this->stream.st->codec->width, this->stream.st->codec->height, PIX_FMT_RGB32, this->stream.st->codec->width, this->stream.st->codec->height, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL);
  
  
  
      // use the scale function to transcode this raw frame to the correct type
  
      sws_scale(this->stream.sws_ctx, this->stream.tmpFrame->data, this->stream.tmpFrame->linesize, 0, this->stream.st->codec->height, this->stream.frame->data, this->stream.frame->linesize);
  
  }

  This is the block that encodes the raw data to h.264, and then send it out the Mpeg2 ts. I believe the problem lies within this block. I can put a break point in my write frame block and see that frames are being written, however, opening the resulting file in VLC results in a blank video. The file is approx 2Mb.

  int FMVStream::EncodeFrame(uint8_t* data)
  
  {
  
      AVCodecContext* c = this->stream.st->codec;
  
  
  
      AVRational one;
  
      one.den = one.num = 1;
  
  
  
      // check to see if we want to keep writing frames we can probably change this to a toggle switch
  
      if (av_compare_ts(this->stream.next_pts, this->stream.st->codec->time_base, 10, one) >= 0)
  
      {
  
          this->stream.frame = nullptr;
  
      }
  
      else
  
      {
  
          // Convert and load the frame data into the AVFrame struct
  
          CopyFrameData(data);
  
      }
  
  
  
      // setup the timestamp stepping
  
      AVPacket pkt = { 0 };
  
      av_init_packet(&pkt);
  
      this->stream.frame->pts = (int64_t)((1.0 / this->stream.st->codec->framerate.den) * 90000.0 * this->stream.next_pts++);
  
  
  
      int gotPacket, out_size, ret;
  
  
  
      out_size = avcodec_encode_video2(c, &pkt, this->stream.frame, &gotPacket);
  
  
  
  
  
      if (gotPacket == 1)
  
      {
  
          ret = write_frame(this->format_ctx, &c->time_base, this->stream.st, &pkt);
  
      }
  
      else
  
      {
  
          ret = 0;
  
      }
  
  
  
      if (ret < 0)
  
      {
  
          std::cerr << "Error writing video frame" << std::endl;
  
      }
  
  
  
      av_free_packet(&pkt);
  
  
  
      return ((this->stream.frame != nullptr) || gotPacket) ? 0 : 1;
  
  }
  
  
  
  int FMVStream::write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt)
  
  {
  
      /* rescale output packet timestamp values from codec to stream timebase */
  
      av_packet_rescale_ts(pkt, *time_base, st->time_base);
  
      pkt->stream_index = st->index;
  
  
  
      return av_interleaved_write_frame(fmt_ctx, pkt);
  
  }
  
  
  
  void FMVStream::setFrameWidth(const int width)
  
  {
  
      this->frameWidth = width;
  
  }
  
  
  
  int FMVStream::getFrameWidth() const
  
  {
  
      return this->frameWidth;
  
  }
  
  
  
  void FMVStream::setFrameHeight(const int height)
  
  {
  
      this->frameHeight = height;
  
  }
  
  
  
  int FMVStream::getFrameHeight() const
  
  {
  
      return this->frameHeight;
  
  }
  
  
  
  void FMVStream::setStreamAddress(const std::string& address)
  
  {
  
      this->streamFilename = address;
  
  }
  
  
  
  std::string FMVStream::getStreamAddress() const
  
  {
  
      return this->streamFilename;
  
  }

  Here is the Main function.

  #include "FullMotionVideoStream.h"
  
  
  
  #include <iostream>
  
  #include <thread>
  
  #include <chrono>
  
  
  
  int main(int argc, char** argv)
  
  {
  
      FMVStream* fmv = new FMVStream;
  
  
  
      fmv->setFrameWidth(640);
  
      fmv->setFrameHeight(480);
  
  
  
      std::cout &lt;&lt; "Streaming Address: " &lt;&lt; fmv->getStreamAddress() &lt;&lt; std::endl;
  
  
  
      // create our alternating frame of black and white to test the streaming functionality
  
      uint8_t white[640 * 480 * sizeof(uint8_t) * 3];
  
      uint8_t black[640 * 480 * sizeof(uint8_t) * 3];
  
  
  
      std::memset(white, 255, 640 * 480 * sizeof(uint8_t) * 3);
  
      std::memset(black, 0, 640 * 480 * sizeof(uint8_t)* 3);
  
  
  
      for (auto i = 0; i &lt; 100; i++)
  
      {
  
          auto ret = fmv->EncodeFrame(white);
  
  
  
          if (ret != 0)
  
          {
  
              std::cerr &lt;&lt; "There was a problem encoding the frame: " &lt;&lt; i &lt;&lt; std::endl;
  
          }
  
  
  
          std::this_thread::sleep_for(std::chrono::milliseconds(10));
  
      }
  
  
  
      for (auto i = 0; i &lt; 100; i++)
  
      {
  
          auto ret = fmv->EncodeFrame(black);
  
  
  
          if (ret != 0)
  
          {
  
              std::cerr &lt;&lt; "There was a problem encoding the frame: " &lt;&lt; i &lt;&lt; std::endl;
  
          }
  
  
  
          std::this_thread::sleep_for(std::chrono::milliseconds(10));
  
      }
  
  
  
      delete fmv;
  
  }
  
  </chrono></thread></iostream>

  Here is the resultant output via the console / my print SDP function.

  [libx264 @ 000000ac95f58440] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2
  
  AVX FMA3 AVX2 LZCNT BMI2
  
  [libx264 @ 000000ac95f58440] profile Constrained Baseline, level 3.0
  
  Output #0, mpegts, to '(null)':
  
      Stream #0:0: Video: h264 (libx264), yuv420p, 640x480, q=-1--1, 400 kb/s, 30
  
  tbn
  
  SDP:
  
  v=0
  
  o=- 0 0 IN IP4 127.0.0.1
  
  s=No Name
  
  t=0 0
  
  a=tool:libavformat 56.23.104
  
  m=video 0 RTP/AVP 96
  
  b=AS:400
  
  a=rtpmap:96 H264/90000
  
  a=fmtp:96 packetization-mode=1
  
  a=control:streamid=0
  
  
  
  Streaming Address: test.mpeg
  
  [libx264 @ 000000ac95f58440] frame I:45    Avg QP: 0.51  size:  1315
  
  [libx264 @ 000000ac95f58440] frame P:136   Avg QP: 0.29  size:   182
  
  [libx264 @ 000000ac95f58440] mb I  I16..4: 99.7%  0.0%  0.3%
  
  [libx264 @ 000000ac95f58440] mb P  I16..4:  0.1%  0.0%  0.1%  P16..4:  0.1%  0.0
  
  %  0.0%  0.0%  0.0%    skip:99.7%
  
  [libx264 @ 000000ac95f58440] final ratefactor: -68.99
  
  [libx264 @ 000000ac95f58440] coded y,uvDC,uvAC intra: 0.5% 0.5% 0.5% inter: 0.0%
  
   0.1% 0.1%
  
  [libx264 @ 000000ac95f58440] i16 v,h,dc,p: 96%  0%  3%  0%
  
  [libx264 @ 000000ac95f58440] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu:  1% 10% 85%  0%  3%
  
   0%  1%  0%  0%
  
  [libx264 @ 000000ac95f58440] i8c dc,h,v,p: 100%  0%  0%  0%
  
  [libx264 @ 000000ac95f58440] ref P L0: 46.8% 25.2% 28.0%
  
  [libx264 @ 000000ac95f58440] kb/s:0.03

  I know there are probably many issues with this program, I am very new with FFMPEG and multimedia programming in general. Ive used many pieces of code found through searching google/ stack overflow to get to this point as is. The file has a good size but comes up as length 0.04 tells me that my time stamping must be broken between the frames / pkts, but I am unsure on how to fix this issue.

  I tried inspecting the file with ffmpeg.exe using ffmpeg -i and outputting to a regular TS. It seems my code works more then I originally intended however, I am simply trying to output a bunch of all white frames.

  ffmpeg -i test.mpeg test.ts
  
  ffmpeg version N-70125-g6c9537b Copyright (c) 2000-2015 the FFmpeg developers
  
    built with gcc 4.9.2 (GCC)
  
    configuration: --disable-static --enable-shared --enable-gpl --enable-version3
  
   --disable-w32threads --enable-avisynth --enable-bzlib --enable-fontconfig --ena
  
  ble-frei0r --enable-gnutls --enable-iconv --enable-libass --enable-libbluray --e
  
  nable-libbs2b --enable-libcaca --enable-libfreetype --enable-libgme --enable-lib
  
  gsm --enable-libilbc --enable-libmodplug --enable-libmp3lame --enable-libopencor
  
  e-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-libopus --enabl
  
  e-librtmp --enable-libschroedinger --enable-libsoxr --enable-libspeex --enable-l
  
  ibtheora --enable-libtwolame --enable-libvidstab --enable-libvo-aacenc --enable-
  
  libvo-amrwbenc --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-l
  
  ibwebp --enable-libx264 --enable-libx265 --enable-libxavs --enable-libxvid --ena
  
  ble-lzma --enable-decklink --enable-zlib
  
    libavutil      54. 19.100 / 54. 19.100
  
    libavcodec     56. 26.100 / 56. 26.100
  
    libavformat    56. 23.104 / 56. 23.104
  
    libavdevice    56.  4.100 / 56.  4.100
  
    libavfilter     5. 11.101 /  5. 11.101
  
    libswscale      3.  1.101 /  3.  1.101
  
    libswresample   1.  1.100 /  1.  1.100
  
    libpostproc    53.  3.100 / 53.  3.100
  
  Input #0, mpegts, from 'test.mpeg':
  
    Duration: 00:00:00.04, start: 0.000000, bitrate: 24026 kb/s
  
    Program 1
  
      Metadata:
  
        service_name    : Service01
  
        service_provider: FFmpeg
  
      Stream #0:0[0x100]: Video: h264 (Constrained Baseline) ([27][0][0][0] / 0x00
  
  1B), yuv420p, 640x480, 25 fps, 25 tbr, 90k tbn, 50 tbc
  
  File 'test.ts' already exists. Overwrite ? [y/N] y
  
  Output #0, mpegts, to 'test.ts':
  
    Metadata:
  
      encoder         : Lavf56.23.104
  
      Stream #0:0: Video: mpeg2video, yuv420p, 640x480, q=2-31, 200 kb/s, 25 fps,
  
  90k tbn, 25 tbc
  
      Metadata:
  
        encoder         : Lavc56.26.100 mpeg2video
  
  Stream mapping:
  
    Stream #0:0 -> #0:0 (h264 (native) -> mpeg2video (native))
  
  Press [q] to stop, [?] for help
  
  frame=    3 fps=0.0 q=2.0 Lsize=       9kB time=00:00:00.08 bitrate= 883.6kbits/
  
  s dup=0 drop=178
  
  video:7kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing ove
  
  rhead: 22.450111%

• encode h264 video using ffmpeg library memory issues

  31 mars 2015, par Zeppa

  I’m trying to do screen capture on OS X using ffmpeg’s avfoundation library. I capture frames from the screen and encode it using H264 into an flv container.

  Here’s the command line output of the program :

  Input #0, avfoundation, from 'Capture screen 0':
  
    Duration: N/A, start: 9.253649, bitrate: N/A
  
      Stream #0:0: Video: rawvideo (UYVY / 0x59565955), uyvy422, 1440x900, 14.58 tbr, 1000k tbn, 1000k tbc
  
  raw video is inCodec
  
  FLV (Flash Video)http://localhost:8090/test.flv
  
  [libx264 @ 0x102038e00] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX
  
  [libx264 @ 0x102038e00] profile High, level 4.0
  
  [libx264 @ 0x102038e00] 264 - core 142 r2495 6a301b6 - H.264/MPEG-4 AVC codec - Copyleft 2003-2014 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=6 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=1 weightp=2 keyint=50 keyint_min=5 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=abr mbtree=1 bitrate=400 ratetol=1.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
  
  [tcp @ 0x101a5fe70] Connection to tcp://localhost:8090 failed (Connection refused), trying next address
  
  [tcp @ 0x101a5fe70] Connection to tcp://localhost:8090 failed: Connection refused
  
  url_fopen failed: Operation now in progress
  
  [flv @ 0x102038800] Using AVStream.codec.time_base as a timebase hint to the muxer is deprecated. Set AVStream.time_base instead.
  
  encoded frame #0
  
  encoded frame #1
  
  ......
  
  encoded frame #49
  
  encoded frame #50
  
  testmee(8404,0x7fff7e05c300) malloc: *** error for object 0x102053e08: incorrect checksum for freed object - object was probably modified after being freed.
  
  *** set a breakpoint in malloc_error_break to debug
  
  (lldb) bt
  
  * thread #10: tid = 0x43873, 0x00007fff95639286 libsystem_kernel.dylib`__pthread_kill + 10, stop reason = signal SIGABRT
  
    * frame #0: 0x00007fff95639286 libsystem_kernel.dylib`__pthread_kill + 10
  
      frame #1: 0x00007fff9623742f libsystem_pthread.dylib`pthread_kill + 90
  
      frame #2: 0x00007fff977ceb53 libsystem_c.dylib`abort + 129
  
      frame #3: 0x00007fff9ab59e06 libsystem_malloc.dylib`szone_error + 625
  
      frame #4: 0x00007fff9ab4f799 libsystem_malloc.dylib`small_malloc_from_free_list + 1105
  
      frame #5: 0x00007fff9ab4d3bc libsystem_malloc.dylib`szone_malloc_should_clear + 1449
  
      frame #6: 0x00007fff9ab4c877 libsystem_malloc.dylib`malloc_zone_malloc + 71
  
      frame #7: 0x00007fff9ab4b395 libsystem_malloc.dylib`malloc + 42
  
      frame #8: 0x00007fff94aa63d2 IOSurface`IOSurfaceClientLookupFromMachPort + 40
  
      frame #9: 0x00007fff94aa6b38 IOSurface`IOSurfaceLookupFromMachPort + 12
  
      frame #10: 0x00007fff92bfa6b2 CoreGraphics`_CGYDisplayStreamFrameAvailable + 342
  
      frame #11: 0x00007fff92f6759c CoreGraphics`CGYDisplayStreamNotification_server + 336
  
      frame #12: 0x00007fff92bfada6 CoreGraphics`display_stream_runloop_callout + 46
  
      frame #13: 0x00007fff956eba07 CoreFoundation`__CFMachPortPerform + 247
  
      frame #14: 0x00007fff956eb8f9 CoreFoundation`__CFRUNLOOP_IS_CALLING_OUT_TO_A_SOURCE1_PERFORM_FUNCTION__ + 41
  
      frame #15: 0x00007fff956eb86b CoreFoundation`__CFRunLoopDoSource1 + 475
  
      frame #16: 0x00007fff956dd3e7 CoreFoundation`__CFRunLoopRun + 2375
  
      frame #17: 0x00007fff956dc858 CoreFoundation`CFRunLoopRunSpecific + 296
  
      frame #18: 0x00007fff95792ef1 CoreFoundation`CFRunLoopRun + 97
  
      frame #19: 0x0000000105f79ff1 CMIOUnits`___lldb_unnamed_function2148$$CMIOUnits + 875
  
      frame #20: 0x0000000105f6f2c2 CMIOUnits`___lldb_unnamed_function2127$$CMIOUnits + 14
  
      frame #21: 0x00007fff97051765 CoreMedia`figThreadMain + 417
  
      frame #22: 0x00007fff96235268 libsystem_pthread.dylib`_pthread_body + 131
  
      frame #23: 0x00007fff962351e5 libsystem_pthread.dylib`_pthread_start + 176
  
      frame #24: 0x00007fff9623341d libsystem_pthread.dylib`thread_start + 13

  I’ve attached the code I used below.

  #include <libavcodec></libavcodec>avcodec.h>
  
  #include <libavformat></libavformat>avformat.h>
  
  #include <libswscale></libswscale>swscale.h>
  
  #include <libavdevice></libavdevice>avdevice.h>
  
  #include <libavutil></libavutil>opt.h>
  
  #include 
  
  #include 
  
  #include 
  
  /* compile using
  
  gcc -g -o stream test.c -lavformat -lavutil -lavcodec -lavdevice -lswscale
  
  */
  
  
  
  // void show_av_device() {
  
  
  
  //    inFmt->get_device_list(inFmtCtx, device_list);
  
  //    printf("Device Info=============\n");
  
  //    //avformat_open_input(&amp;inFmtCtx,"video=Capture screen 0",inFmt,&amp;inOptions);
  
  //    printf("===============================\n");
  
  // }
  
  
  
  void AVFAIL (int code, const char *what) {
  
      char msg[500];
  
      av_strerror(code, msg, sizeof(msg));
  
      fprintf(stderr, "failed: %s\nerror: %s\n", what, msg);
  
      exit(2);
  
  }
  
  
  
  #define AVCHECK(f) do { int e = (f); if (e &lt; 0) AVFAIL(e, #f); } while (0)
  
  #define AVCHECKPTR(p,f) do { p = (f); if (!p) AVFAIL(AVERROR_UNKNOWN, #f); } while (0)
  
  
  
  void registerLibs() {
  
      av_register_all();
  
      avdevice_register_all();
  
      avformat_network_init();
  
      avcodec_register_all();
  
  }
  
  
  
  int main(int argc, char *argv[]) {
  
  
  
      //conversion variables
  
      struct SwsContext *swsCtx = NULL;
  
      //input stream variables
  
      AVFormatContext   *inFmtCtx = NULL;
  
      AVCodecContext    *inCodecCtx = NULL;
  
      AVCodec           *inCodec = NULL;
  
      AVInputFormat     *inFmt = NULL;
  
      AVFrame           *inFrame = NULL;
  
      AVDictionary      *inOptions = NULL;
  
      const char *streamURL = "http://localhost:8090/test.flv";
  
      const char *name = "avfoundation";
  
  
  
  //    AVFrame           *inFrameYUV = NULL;
  
      AVPacket          inPacket;
  
  
  
  
  
      //output stream variables
  
      AVCodecContext    *outCodecCtx = NULL;
  
      AVCodec           *outCodec;
  
      AVFormatContext   *outFmtCtx = NULL;
  
      AVOutputFormat    *outFmt = NULL;
  
      AVFrame           *outFrameYUV = NULL;
  
      AVStream          *stream = NULL;
  
  
  
      int               i, videostream, ret;
  
      int               numBytes, frameFinished;
  
  
  
      registerLibs();
  
      inFmtCtx = avformat_alloc_context(); //alloc input context
  
      av_dict_set(&amp;inOptions, "pixel_format", "uyvy422", 0);
  
      av_dict_set(&amp;inOptions, "probesize", "7000000", 0);
  
  
  
      inFmt = av_find_input_format(name);
  
      ret = avformat_open_input(&amp;inFmtCtx, "Capture screen 0:", inFmt, &amp;inOptions);
  
      if (ret &lt; 0) {
  
          printf("Could not load the context for the input device\n");
  
          return -1;
  
      }
  
      if (avformat_find_stream_info(inFmtCtx, NULL) &lt; 0) {
  
          printf("Could not find stream info for screen\n");
  
          return -1;
  
      }
  
      av_dump_format(inFmtCtx, 0, "Capture screen 0", 0);
  
      // inFmtCtx->streams is an array of pointers of size inFmtCtx->nb_stream
  
  
  
      videostream = av_find_best_stream(inFmtCtx, AVMEDIA_TYPE_VIDEO, -1, -1, &amp;inCodec, 0);
  
      if (videostream == -1) {
  
          printf("no video stream found\n");
  
          return -1;
  
      } else {
  
          printf("%s is inCodec\n", inCodec->long_name);
  
      }
  
      inCodecCtx = inFmtCtx->streams[videostream]->codec;
  
      // open codec
  
      if (avcodec_open2(inCodecCtx, inCodec, NULL) > 0) {
  
          printf("Couldn't open codec");
  
          return -1;  // couldn't open codec
  
      }
  
  
  
  
  
          //setup output params
  
      outFmt = av_guess_format(NULL, streamURL, NULL);
  
      if(outFmt == NULL) {
  
          printf("output format was not guessed properly");
  
          return -1;
  
      }
  
  
  
      if((outFmtCtx = avformat_alloc_context()) &lt; 0) {
  
          printf("output context not allocated. ERROR");
  
          return -1;
  
      }
  
  
  
      printf("%s", outFmt->long_name);
  
  
  
      outFmtCtx->oformat = outFmt;
  
  
  
      snprintf(outFmtCtx->filename, sizeof(outFmtCtx->filename), streamURL);
  
      printf("%s\n", outFmtCtx->filename);
  
  
  
      outCodec = avcodec_find_encoder(AV_CODEC_ID_H264);
  
      if(!outCodec) {
  
          printf("could not find encoder for H264 \n" );
  
          return -1;
  
      }
  
  
  
      stream = avformat_new_stream(outFmtCtx, outCodec);
  
      outCodecCtx = stream->codec;
  
      avcodec_get_context_defaults3(outCodecCtx, outCodec);
  
  
  
      outCodecCtx->codec_id = AV_CODEC_ID_H264;
  
      outCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
  
      outCodecCtx->flags = CODEC_FLAG_GLOBAL_HEADER;
  
      outCodecCtx->width = inCodecCtx->width;
  
      outCodecCtx->height = inCodecCtx->height;
  
      outCodecCtx->time_base.den = 25;
  
      outCodecCtx->time_base.num = 1;
  
      outCodecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
  
      outCodecCtx->gop_size = 50;
  
      outCodecCtx->bit_rate = 400000;
  
  
  
      //setup output encoders etc
  
      if(stream) {
  
          ret = avcodec_open2(outCodecCtx, outCodec, NULL);
  
          if (ret &lt; 0) {
  
              printf("Could not open output encoder");
  
              return -1;
  
          }
  
      }
  
  
  
      if (avio_open(&amp;outFmtCtx->pb, outFmtCtx->filename, AVIO_FLAG_WRITE ) &lt; 0) {
  
          perror("url_fopen failed");
  
      }
  
  
  
      avio_open_dyn_buf(&amp;outFmtCtx->pb);
  
      ret = avformat_write_header(outFmtCtx, NULL);
  
      if (ret != 0) {
  
          printf("was not able to write header to output format");
  
          return -1;
  
      }
  
      unsigned char *pb_buffer;
  
      int len = avio_close_dyn_buf(outFmtCtx->pb, (unsigned char **)(&amp;pb_buffer));
  
      avio_write(outFmtCtx->pb, (unsigned char *)pb_buffer, len);
  
  
  
  
  
      numBytes = avpicture_get_size(PIX_FMT_UYVY422, inCodecCtx->width, inCodecCtx->height);
  
      // Allocate video frame
  
      inFrame = av_frame_alloc();
  
  
  
      swsCtx = sws_getContext(inCodecCtx->width, inCodecCtx->height, inCodecCtx->pix_fmt, inCodecCtx->width,
  
                              inCodecCtx->height, PIX_FMT_YUV420P, SWS_BILINEAR, NULL, NULL, NULL);
  
      int frame_count = 0;
  
      while(av_read_frame(inFmtCtx, &amp;inPacket) >= 0) {
  
          if(inPacket.stream_index == videostream) {
  
              avcodec_decode_video2(inCodecCtx, inFrame, &amp;frameFinished, &amp;inPacket);
  
              // 1 Frame might need more than 1 packet to be filled
  
              if(frameFinished) {
  
                  outFrameYUV = av_frame_alloc();
  
  
  
                  uint8_t *buffer = (uint8_t *)av_malloc(numBytes);
  
  
  
                  int ret = avpicture_fill((AVPicture *)outFrameYUV, buffer, PIX_FMT_YUV420P,
  
                                           inCodecCtx->width, inCodecCtx->height);
  
                  if(ret &lt; 0){
  
                      printf("%d is return val for fill\n", ret);
  
                      return -1;
  
                  }
  
                  //convert image to YUV
  
                  sws_scale(swsCtx, (uint8_t const * const* )inFrame->data,
  
                            inFrame->linesize, 0, inCodecCtx->height,
  
                            outFrameYUV->data, outFrameYUV->linesize);
  
                  //outFrameYUV now holds the YUV scaled frame/picture
  
                  outFrameYUV->format = outCodecCtx->pix_fmt;
  
                  outFrameYUV->width = outCodecCtx->width;
  
                  outFrameYUV->height = outCodecCtx->height;
  
  
  
  
  
                  AVPacket pkt;
  
                  int got_output;
  
                  av_init_packet(&amp;pkt);
  
                  pkt.data = NULL;
  
                  pkt.size = 0;
  
  
  
                  outFrameYUV->pts = frame_count;
  
  
  
                  ret = avcodec_encode_video2(outCodecCtx, &amp;pkt, outFrameYUV, &amp;got_output);
  
                  if (ret &lt; 0) {
  
                      fprintf(stderr, "Error encoding video frame: %s\n", av_err2str(ret));
  
                      return -1;
  
                  }
  
  
  
                  if(got_output) {
  
                      if(stream->codec->coded_frame->key_frame) {
  
                          pkt.flags |= AV_PKT_FLAG_KEY;
  
                      }
  
                      pkt.stream_index = stream->index;
  
                      if(pkt.pts != AV_NOPTS_VALUE)
  
                          pkt.pts = av_rescale_q(pkt.pts, stream->codec->time_base, stream->time_base);
  
                      if(pkt.dts != AV_NOPTS_VALUE)
  
                          pkt.dts = av_rescale_q(pkt.dts, stream->codec->time_base, stream->time_base);
  
                      if(avio_open_dyn_buf(&amp;outFmtCtx->pb)!= 0) {
  
                          printf("ERROR: Unable to open dynamic buffer\n");
  
                      }
  
                      ret = av_interleaved_write_frame(outFmtCtx, &amp;pkt);
  
                      unsigned char *pb_buffer;
  
                      int len = avio_close_dyn_buf(outFmtCtx->pb, (unsigned char **)&amp;pb_buffer);
  
                      avio_write(outFmtCtx->pb, (unsigned char *)pb_buffer, len);
  
  
  
                  } else {
  
                      ret = 0;
  
                  }
  
                  if(ret != 0) {
  
                      fprintf(stderr, "Error while writing video frame: %s\n", av_err2str(ret));
  
                      exit(1);
  
                  }
  
  
  
                  fprintf(stderr, "encoded frame #%d\n", frame_count);
  
                  frame_count++;
  
  
  
                  av_free_packet(&amp;pkt);
  
                  av_frame_free(&amp;outFrameYUV);
  
                  av_free(buffer);
  
  
  
              }
  
          }
  
          av_free_packet(&amp;inPacket);
  
      }
  
      av_write_trailer(outFmtCtx);
  
  
  
      //close video stream
  
      if(stream) {
  
          avcodec_close(outCodecCtx);
  
      }
  
      for (i = 0; i &lt; outFmtCtx->nb_streams; i++) {
  
          av_freep(&amp;outFmtCtx->streams[i]->codec);
  
          av_freep(&amp;outFmtCtx->streams[i]);
  
      }
  
      if (!(outFmt->flags &amp; AVFMT_NOFILE))
  
      /* Close the output file. */
  
          avio_close(outFmtCtx->pb);
  
      /* free the output format context */
  
      avformat_free_context(outFmtCtx);
  
  
  
      // Free the YUV frame populated by the decoder
  
      av_free(inFrame);
  
  
  
      // Close the video codec (decoder)
  
      avcodec_close(inCodecCtx);
  
  
  
      // Close the input video file
  
      avformat_close_input(&amp;inFmtCtx);
  
  
  
      return 1;
  
  
  
  }

  I’m not sure what I’ve done wrong here. But, what I’ve observed is that for each frame thats been encoded, my memory usage goes up by about 6MB. Backtracking afterward usually leads one of the following two culprits :

  1. avf_read_frame function in avfoundation.m
  2. av_dup_packet function in avpacket.h

  Can I also get advice on the way I’m using avio_open_dyn_buff function to be able to stream over http? I’ve also attached my ffmpeg library versions below :

      ffmpeg version N-70876-g294bb6c Copyright (c) 2000-2015 the FFmpeg developers
  
    built with Apple LLVM version 6.0 (clang-600.0.56) (based on LLVM 3.5svn)
  
    configuration: --prefix=/usr/local --enable-gpl --enable-postproc --enable-pthreads --enable-libmp3lame --enable-libtheora --enable-libx264 --enable-libvorbis --disable-mmx --disable-ssse3 --disable-armv5te --disable-armv6 --disable-neon --enable-shared --disable-static --disable-stripping
  
    libavutil      54. 20.100 / 54. 20.100
  
    libavcodec     56. 29.100 / 56. 29.100
  
    libavformat    56. 26.101 / 56. 26.101
  
    libavdevice    56.  4.100 / 56.  4.100
  
    libavfilter     5. 13.101 /  5. 13.101
  
    libswscale      3.  1.101 /  3.  1.101
  
    libswresample   1.  1.100 /  1.  1.100
  
    libpostproc    53.  3.100 / 53.  3.100
  
  Hyper fast Audio and Video encoder

  Valgrind analysis attached here because I exceeded stack overflow’s character limit. http://pastebin.com/MPeRhjhN