Recherche avancée

Sur d’autres sites (13118)

• Decode audio and video and process both streams — ffmpeg, sdl, opencv

  6 mai 2013, par Eric

  My goal is to proceed on audio and video of mpeg-2 file independently, and to keep synchronicity on both flows. Duration of video is about 1 or 2 minutes maximum.

  1. First, following this post "opencv for reading videos (and do the process),ffmpeg for audio , and SDL used to play both" sounds perfect. I have done some modification on the code considering recent ffmpeg naming changes. Compilation with cmake on 64-bits machine is fine. I get an error "Unsupported codec [3]" when opening codec.
     The code is following.

  2. Second, I looking for code dealing with synchronicity on both flows.

  ---

  #include "opencv/highgui.h"
  
  #include "opencv/cv.h"
  
  
  
  #ifndef INT64_C
  
  #define INT64_C(c) (c ## LL)
  
  #define UINT64_C(c) (c ## ULL)
  
  #endif
  
  
  
  extern "C"{
  
  #include <sdl></sdl>SDL.h>
  
  #include <sdl></sdl>SDL_thread.h>
  
  #include <libavcodec></libavcodec>avcodec.h>
  
  #include <libavformat></libavformat>avformat.h>
  
  }
  
  
  
  #include <iostream>
  
  #include 
  
  #include 
  
  
  
  using namespace cv;
  
  
  
  #define SDL_AUDIO_BUFFER_SIZE 1024
  
  
  
  typedef struct PacketQueue
  
  {
  
     AVPacketList *first_pkt, *last_pkt;
  
     int nb_packets;
  
     int size;
  
     SDL_mutex *mutex;
  
     SDL_cond *cond;
  
  } PacketQueue;
  
  PacketQueue audioq;
  
  
  
  int audioStream = -1;
  
  int videoStream = -1;
  
  int quit = 0;
  
  
  
  SDL_Surface* screen = NULL;
  
  SDL_Surface* surface = NULL;
  
  
  
  AVFormatContext* pFormatCtx = NULL;
  
  AVCodecContext* aCodecCtx = NULL;
  
  AVCodecContext* pCodecCtx = NULL;
  
  
  
  void show_frame(IplImage* img){
  
     if (!screen){
  
        screen = SDL_SetVideoMode(img->width, img->height, 0, 0);
  
        if (!screen){
  
           fprintf(stderr, "SDL: could not set video mode - exiting\n");
  
           exit(1);
  
        }
  
     }
  
     // Assuming IplImage packed as BGR 24bits
  
     SDL_Surface* surface = SDL_CreateRGBSurfaceFrom((void*)img->imageData,
  
                                                     img->width,
  
                                                     img->height,
  
                                                     img->depth * img->nChannels,
  
                                                     img->widthStep,
  
                                                     0xff0000, 0x00ff00, 0x0000ff, 0
  
                                                    );
  
  
  
     SDL_BlitSurface(surface, 0, screen, 0);
  
     SDL_Flip(screen);
  
  }
  
  
  
  void packet_queue_init(PacketQueue *q){
  
     memset(q, 0, sizeof(PacketQueue));
  
     q->mutex = SDL_CreateMutex();
  
     q->cond = SDL_CreateCond();
  
  }
  
  
  
  int packet_queue_put(PacketQueue *q, AVPacket *pkt){
  
     AVPacketList *pkt1;
  
     if (av_dup_packet(pkt) &lt; 0){
  
        return -1;
  
     }
  
  
  
     pkt1 = (AVPacketList*) av_malloc(sizeof(AVPacketList));
  
     //pkt1 = (AVPacketList*) malloc(sizeof(AVPacketList));
  
     if (!pkt1) return -1;
  
     pkt1->pkt = *pkt;
  
     pkt1->next = NULL;
  
  
  
     SDL_LockMutex(q->mutex);
  
  
  
     if (!q->last_pkt)
  
        q->first_pkt = pkt1;
  
     else
  
        q->last_pkt->next = pkt1;
  
  
  
     q->last_pkt = pkt1;
  
     q->nb_packets++;
  
     q->size += pkt1->pkt.size;
  
     SDL_CondSignal(q->cond);
  
  
  
     SDL_UnlockMutex(q->mutex);
  
     return 0;
  
  }
  
  
  
  static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block){
  
     AVPacketList *pkt1;
  
     int ret;
  
  
  
     SDL_LockMutex(q->mutex);
  
     for (;;){
  
        if( quit){
  
           ret = -1;
  
           break;
  
        }
  
  
  
        pkt1 = q->first_pkt;
  
        if (pkt1){
  
           q->first_pkt = pkt1->next;
  
           if (!q->first_pkt)
  
              q->last_pkt = NULL;
  
  
  
           q->nb_packets--;
  
           q->size -= pkt1->pkt.size;
  
           *pkt = pkt1->pkt;
  
           av_free(pkt1);
  
           //free(pkt1);
  
           ret = 1;
  
           break;
  
        }
  
  
  
        else if (!block){
  
           ret = 0;
  
           break;
  
        }
  
        else{
  
           SDL_CondWait(q->cond, q->mutex);
  
        }
  
     }
  
  
  
     SDL_UnlockMutex(q->mutex);
  
     return ret;
  
  }
  
  
  
  int audio_decode_frame(AVCodecContext *aCodecCtx, uint8_t *audio_buf, int buf_size){
  
     static AVPacket pkt;
  
     static uint8_t *audio_pkt_data = NULL;
  
     static int audio_pkt_size = 0;
  
  
  
     int len1, data_size;
  
  
  
     for (;;){
  
        while (audio_pkt_size > 0){
  
           data_size = buf_size;
  
           len1 = avcodec_decode_audio3(aCodecCtx, (int16_t*)audio_buf, &amp;data_size, &amp;pkt);
  
           if (len1 &lt; 0){
  
              // if error, skip frame
  
              audio_pkt_size = 0;
  
              break;
  
           }
  
           audio_pkt_data += len1;
  
           audio_pkt_size -= len1;
  
           if (data_size &lt;= 0){
  
              // No data yet, get more frames
  
              continue;
  
           }
  
           // We have data, return it and come back for more later
  
           return data_size;
  
       }
  
  
  
       if (pkt.data)
  
          av_free_packet(&amp;pkt);
  
       if (quit) return -1;
  
       if (packet_queue_get(&amp;audioq, &amp;pkt, 1) &lt; 0) return -1;
  
       audio_pkt_data = pkt.data;
  
       audio_pkt_size = pkt.size;
  
    }
  
  }
  
  
  
  void audio_callback(void *userdata, Uint8 *stream, int len){
  
    AVCodecContext *aCodecCtx = (AVCodecContext *)userdata;
  
    int len1, audio_size;
  
  
  
    static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
  
    static unsigned int audio_buf_size = 0;
  
    static unsigned int audio_buf_index = 0;
  
  
  
    while (len > 0){
  
       if (audio_buf_index >= audio_buf_size){
  
          // We have already sent all our data; get more
  
          audio_size = audio_decode_frame(aCodecCtx, audio_buf, sizeof(audio_buf));
  
          if(audio_size &lt; 0){
  
             // If error, output silence
  
             audio_buf_size = 1024; // arbitrary?
  
             memset(audio_buf, 0, audio_buf_size);
  
          }
  
          else{
  
             audio_buf_size = audio_size;
  
          }
  
          audio_buf_index = 0;
  
      }
  
  
  
      len1 = audio_buf_size - audio_buf_index;
  
      if (len1 > len)
  
         len1 = len;
  
      memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);
  
      len -= len1;
  
      stream += len1;
  
      audio_buf_index += len1;
  
    }
  
  }
  
  
  
       void setup_ffmpeg(char* filename)
  
       {
  
          if (avformat_open_input(&amp;pFormatCtx, filename, NULL, NULL) != 0){
  
             fprintf(stderr, "FFmpeg failed to open file %s!\n", filename);
  
             exit(-1);
  
          }
  
  
  
          if (av_find_stream_info(pFormatCtx) &lt; 0){
  
             fprintf(stderr, "FFmpeg failed to retrieve stream info!\n");
  
             exit(-1);
  
          }
  
  
  
          // Dump information about file onto standard error
  
          av_dump_format(pFormatCtx, 0, filename, 0);
  
  
  
          // Find the first video stream
  
          int i = 0;
  
          for (i; i &lt; pFormatCtx->nb_streams; i++){
  
             if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO &amp;&amp; videoStream &lt; 0){
  
                videoStream = i;
  
             }
  
  
  
             if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO &amp;&amp; audioStream &lt; 0){
  
                audioStream = i;
  
             }
  
          }
  
  
  
          if (videoStream == -1){
  
             fprintf(stderr, "No video stream found in %s!\n", filename);
  
             exit(-1);
  
          }
  
  
  
          if (audioStream == -1){
  
             fprintf(stderr, "No audio stream found in %s!\n", filename);
  
             exit(-1);
  
          }
  
  
  
          // Get a pointer to the codec context for the audio stream
  
          aCodecCtx = pFormatCtx->streams[audioStream]->codec;
  
  
  
          // Set audio settings from codec info
  
          SDL_AudioSpec wanted_spec;
  
          wanted_spec.freq = aCodecCtx->sample_rate;
  
          wanted_spec.format = AUDIO_S16SYS;
  
          wanted_spec.channels = aCodecCtx->channels;
  
          wanted_spec.silence = 0;
  
          wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
  
          wanted_spec.callback = audio_callback;
  
          wanted_spec.userdata = aCodecCtx;
  
  
  
          SDL_AudioSpec spec;
  
          if (SDL_OpenAudio(&amp;wanted_spec, &amp;spec) &lt; 0){
  
             fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
  
             exit(-1);
  
          }
  
  
  
          AVCodec* aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
  
          if (!aCodec){
  
             fprintf(stderr, "Unsupported codec [1]!\n");
  
             exit(-1);
  
          }
  
          avcodec_open(aCodecCtx, aCodec);
  
  
  
          // audio_st = pFormatCtx->streams[index]
  
          packet_queue_init(&amp;audioq);
  
          SDL_PauseAudio(0);
  
  
  
          // Get a pointer to the codec context for the video stream
  
          pCodecCtx = pFormatCtx->streams[videoStream]->codec;
  
  
  
          // Find the decoder for the video stream
  
          AVCodec* pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
  
          if (pCodec == NULL){
  
             fprintf(stderr, "Unsupported codec [2]!\n");
  
             exit(-1); // Codec not found
  
          }
  
  
  
          // Open codec
  
          if (avcodec_open(pCodecCtx, pCodec) &lt; 0){
  
             fprintf(stderr, "Unsupported codec [3]!\n");
  
             exit(-1); // Could not open codec
  
          }
  
       }
  
  
  
  
  
       int main(int argc, char* argv[])
  
       {
  
          if (argc &lt; 2){
  
              std::cout &lt;&lt; "Usage: " &lt;&lt; argv[0] &lt;&lt; " <video>" &lt;&lt; std::endl;
  
              return -1;
  
          }
  
  
  
          av_register_all();
  
  
  
          // Init SDL
  
          if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))
  
          {
  
             fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
  
             return -1;
  
          }
  
  
  
          // Init ffmpeg and setup some SDL stuff related to Audio
  
          setup_ffmpeg(argv[1]);
  
  
  
          VideoCapture cap(argv[1]);
  
          if (!cap.isOpened()){
  
             std::cout &lt;&lt; "Failed to load file!" &lt;&lt; std::endl;
  
             return -1;
  
          }
  
  
  
          AVPacket packet;
  
          while (av_read_frame(pFormatCtx, &amp;packet) >= 0)
  
          {
  
             if (packet.stream_index == videoStream)
  
             {
  
                // Actually this is were SYNC between audio/video would happen.
  
                // Right now I assume that every VIDEO packet contains an entire video frame, and that&#39;s not true. A video frame can be made by multiple packets!
  
                // But for the time being, assume 1 video frame == 1 video packet,
  
                // so instead of reading the frame through ffmpeg, I read it through OpenCV.
  
  
  
                Mat frame;
  
                cap >> frame; // get a new frame from camera
  
  
  
                // do some processing on the frame, either as a Mat or as IplImage.
  
                // For educational purposes, applying a lame grayscale conversion
  
                IplImage ipl_frame = frame;
  
                for (int i = 0; i &lt; ipl_frame.width * ipl_frame.height * ipl_frame.nChannels; i += ipl_frame.nChannels)
  
                {
  
                   ipl_frame.imageData[i] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3;   //B
  
                   ipl_frame.imageData[i+1] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //G
  
                   ipl_frame.imageData[i+2] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //R
  
                }
  
  
  
                // Display it on SDL window
  
                show_frame(&amp;ipl_frame);
  
  
  
                av_free_packet(&amp;packet);
  
             }
  
             else if (packet.stream_index == audioStream)
  
             {
  
                packet_queue_put(&amp;audioq, &amp;packet);
  
             }
  
             else
  
             {
  
                av_free_packet(&amp;packet);
  
             }
  
  
  
             SDL_Event event;
  
             SDL_PollEvent(&amp;event);
  
             switch (event.type)
  
             {
  
             case SDL_QUIT:
  
                SDL_FreeSurface(surface);
  
                SDL_Quit();
  
                break;
  
  
  
             default:
  
                break;
  
             }
  
          }
  
  
  
          // the camera will be deinitialized automatically in VideoCapture destructor
  
  
  
          // Close the codec
  
          avcodec_close(pCodecCtx);
  
  
  
          // Close the video file
  
          av_close_input_file(pFormatCtx);
  
  
  
          return 0;
  
       }
  
  </video></iostream>

• fate : add tests for cdxl video

  21 février 2012, par Paul B Mahol

  fate : add tests for cdxl video

• avconv : saner output video timebase.

  7 février 2012, par Anton Khirnov

  avconv : saner output video timebase.