Recherche avancée

Médias (2)

Mot : - Tags -/documentation

Autres articles (43)

Sur d’autres sites (8557)

  • iPhone camera shooting video using the AVCaptureSession and using ffmpeg CMSampleBufferRef a change in h.264 format is the issue. please advice

    4 janvier 2012, par isaiah

    My goal is h.264/AAC , mpeg2-ts streaming to server from iphone device.

    Current my source is FFmpeg+libx264 compile success. I Know gnu License. I want the demo program.

    I'm want to know that

    1.CMSampleBufferRef to AVPicture data is success ?

    avpicture_fill((AVPicture*)pFrame, rawPixelBase, PIX_FMT_RGB32, width, height);
     pFrame linesize and data is not null but pst -9233123123 . outpic also .
    Because of this I have to guess 'non-strictly-monotonic PTS' message

    2.This log is repeat.

    encoding frame (size= 0)
    encoding frame = "" , 'avcodec_encode_video' return 0 is success but always 0 .

    I don't know what to do...

    2011-06-01 15:15:14.199 AVCam[1993:7303] pFrame = avcodec_alloc_frame();
    2011-06-01 15:15:14.207 AVCam[1993:7303] avpicture_fill = 1228800
    Video encoding
    2011-0601 15:5:14.215 AVCam[1993:7303] codec = 5841844
    [libx264 @ 0x1441e00] using cpu capabilities: ARMv6 NEON
    [libx264 @ 0x1441e00] profile Constrained Baseline, level 2.0[libx264 @ 0x1441e00] non-strictly-monotonic PTS
    encoding frame (size=    0)
    encoding frame
    [libx264 @ 0x1441e00] final ratefactor: 26.74

    3.I have to guess 'non-strictly-monotonic PTS' message is the cause of all problems.
    what is this 'non-strictly-monotonic PTS' .

     this is source 

    (void)        captureOutput:(AVCaptureOutput *)captureOutput
           didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
                  fromConnection:(AVCaptureConnection *)connection
    {

       if( !CMSampleBufferDataIsReady(sampleBuffer) )
       {
           NSLog( @"sample buffer is not ready. Skipping sample" );
           return;
       }


       if( [isRecordingNow isEqualToString:@"YES"] )
       {
           lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
           if( videoWriter.status != AVAssetWriterStatusWriting  )
           {
               [videoWriter startWriting];
               [videoWriter startSessionAtSourceTime:lastSampleTime];
           }

           if( captureOutput == videooutput )
           {
               [self newVideoSample:sampleBuffer];

               CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
               CVPixelBufferLockBaseAddress(pixelBuffer, 0);

               // access the data
               int width = CVPixelBufferGetWidth(pixelBuffer);
               int height = CVPixelBufferGetHeight(pixelBuffer);
               unsigned char *rawPixelBase = (unsigned char *)CVPixelBufferGetBaseAddress(pixelBuffer);

               AVFrame *pFrame;
               pFrame = avcodec_alloc_frame();
               pFrame->quality = 0;

               NSLog(@"pFrame = avcodec_alloc_frame(); ");

    //          int bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);

    //          int bytesSize = height * bytesPerRow ;  

    //          unsigned char *pixel = (unsigned char*)malloc(bytesSize);

    //          unsigned char *rowBase = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);

    //          memcpy (pixel, rowBase, bytesSize);


               int avpicture_fillNum = avpicture_fill((AVPicture*)pFrame, rawPixelBase, PIX_FMT_RGB32, width, height);//PIX_FMT_RGB32//PIX_FMT_RGB8
               //NSLog(@"rawPixelBase = %i , rawPixelBase -s = %s",rawPixelBase, rawPixelBase);
               NSLog(@"avpicture_fill = %i",avpicture_fillNum);
               //NSLog(@"width = %i,height = %i",width, height);



               // Do something with the raw pixels here

               CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);

               //avcodec_init();
               //avdevice_register_all();
               av_register_all();





               AVCodec *codec;
               AVCodecContext *c= NULL;
               int  out_size, size, outbuf_size;
               //FILE *f;
               uint8_t *outbuf;

               printf("Video encoding\n");

               /* find the mpeg video encoder */
               codec =avcodec_find_encoder(CODEC_ID_H264);//avcodec_find_encoder_by_name("libx264"); //avcodec_find_encoder(CODEC_ID_H264);//CODEC_ID_H264);
               NSLog(@"codec = %i",codec);
               if (!codec) {
                   fprintf(stderr, "codec not found\n");
                   exit(1);
               }

               c= avcodec_alloc_context();

               /* put sample parameters */
               c->bit_rate = 400000;
               c->bit_rate_tolerance = 10;
               c->me_method = 2;
               /* resolution must be a multiple of two */
               c->width = 352;//width;//352;
               c->height = 288;//height;//288;
               /* frames per second */
               c->time_base= (AVRational){1,25};
               c->gop_size = 10;//25; /* emit one intra frame every ten frames */
               //c->max_b_frames=1;
               c->pix_fmt = PIX_FMT_YUV420P;

               c ->me_range = 16;
               c ->max_qdiff = 4;
               c ->qmin = 10;
               c ->qmax = 51;
               c ->qcompress = 0.6f;

               /* open it */
               if (avcodec_open(c, codec) < 0) {
                   fprintf(stderr, "could not open codec\n");
                   exit(1);
               }


               /* alloc image and output buffer */
               outbuf_size = 100000;
               outbuf = malloc(outbuf_size);
               size = c->width * c->height;

               AVFrame* outpic = avcodec_alloc_frame();
               int nbytes = avpicture_get_size(PIX_FMT_YUV420P, c->width, c->height);

               //create buffer for the output image
               uint8_t* outbuffer = (uint8_t*)av_malloc(nbytes);

    #pragma mark -  

               fflush(stdout);

    <pre>//         int numBytes = avpicture_get_size(PIX_FMT_YUV420P, c->width, c->height);
    //          uint8_t *buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
    //          
    //          //UIImage *image = [UIImage imageNamed:[NSString stringWithFormat:@"10%d", i]];
    //          CGImageRef newCgImage = [self imageFromSampleBuffer:sampleBuffer];//[image CGImage];
    //          
    //          CGDataProviderRef dataProvider = CGImageGetDataProvider(newCgImage);
    //          CFDataRef bitmapData = CGDataProviderCopyData(dataProvider);
    //          buffer = (uint8_t *)CFDataGetBytePtr(bitmapData);  
    //          
    //          avpicture_fill((AVPicture*)pFrame, buffer, PIX_FMT_RGB8, c->width, c->height);
               avpicture_fill((AVPicture*)outpic, outbuffer, PIX_FMT_YUV420P, c->width, c->height);

               struct SwsContext* fooContext = sws_getContext(c->width, c->height,
                                                              PIX_FMT_RGB8,
                                                              c->width, c->height,
                                                              PIX_FMT_YUV420P,
                                                              SWS_FAST_BILINEAR, NULL, NULL, NULL);

               //perform the conversion
               sws_scale(fooContext, pFrame->data, pFrame->linesize, 0, c->height, outpic->data, outpic->linesize);
               // Here is where I try to convert to YUV

               /* encode the image */

               out_size = avcodec_encode_video(c, outbuf, outbuf_size, outpic);
               printf("encoding frame (size=%5d)\n", out_size);
               printf("encoding frame %s\n", outbuf);


               //fwrite(outbuf, 1, out_size, f);

               //              free(buffer);
               //              buffer = NULL;      



               /* add sequence end code to have a real mpeg file */
    //          outbuf[0] = 0x00;
    //          outbuf[1] = 0x00;
    //          outbuf[2] = 0x01;
    //          outbuf[3] = 0xb7;
               //fwrite(outbuf, 1, 4, f);
               //fclose(f);
               free(outbuf);

               avcodec_close(c);
               av_free(c);
               av_free(pFrame);
               printf("\n");
    </pre>
  • Decode audio and video and process both streams — ffmpeg, sdl, opencv

    6 mai 2013, par Eric

    My goal is to proceed on audio and video of mpeg-2 file independently, and to keep synchronicity on both flows. Duration of video is about 1 or 2 minutes maximum.

    1. First, following this post "opencv for reading videos (and do the process),ffmpeg for audio , and SDL used to play both" sounds perfect. I have done some modification on the code considering recent ffmpeg naming changes. Compilation with cmake on 64-bits machine is fine. I get an error "Unsupported codec [3]" when opening codec.
      The code is following.

    2. Second, I looking for code dealing with synchronicity on both flows.


    #include "opencv/highgui.h"
    #include "opencv/cv.h"

    #ifndef INT64_C
    #define INT64_C(c) (c ## LL)
    #define UINT64_C(c) (c ## ULL)
    #endif

    extern "C"{
    #include <sdl></sdl>SDL.h>
    #include <sdl></sdl>SDL_thread.h>
    #include <libavcodec></libavcodec>avcodec.h>
    #include <libavformat></libavformat>avformat.h>
    }

    #include <iostream>
    #include
    #include

    using namespace cv;

    #define SDL_AUDIO_BUFFER_SIZE 1024

    typedef struct PacketQueue
    {
      AVPacketList *first_pkt, *last_pkt;
      int nb_packets;
      int size;
      SDL_mutex *mutex;
      SDL_cond *cond;
    } PacketQueue;
    PacketQueue audioq;

    int audioStream = -1;
    int videoStream = -1;
    int quit = 0;

    SDL_Surface* screen = NULL;
    SDL_Surface* surface = NULL;

    AVFormatContext* pFormatCtx = NULL;
    AVCodecContext* aCodecCtx = NULL;
    AVCodecContext* pCodecCtx = NULL;

    void show_frame(IplImage* img){
      if (!screen){
         screen = SDL_SetVideoMode(img->width, img->height, 0, 0);
         if (!screen){
            fprintf(stderr, "SDL: could not set video mode - exiting\n");
            exit(1);
         }
      }
      // Assuming IplImage packed as BGR 24bits
      SDL_Surface* surface = SDL_CreateRGBSurfaceFrom((void*)img->imageData,
                                                      img->width,
                                                      img->height,
                                                      img->depth * img->nChannels,
                                                      img->widthStep,
                                                      0xff0000, 0x00ff00, 0x0000ff, 0
                                                     );

      SDL_BlitSurface(surface, 0, screen, 0);
      SDL_Flip(screen);
    }

    void packet_queue_init(PacketQueue *q){
      memset(q, 0, sizeof(PacketQueue));
      q->mutex = SDL_CreateMutex();
      q->cond = SDL_CreateCond();
    }

    int packet_queue_put(PacketQueue *q, AVPacket *pkt){
      AVPacketList *pkt1;
      if (av_dup_packet(pkt) &lt; 0){
         return -1;
      }

      pkt1 = (AVPacketList*) av_malloc(sizeof(AVPacketList));
      //pkt1 = (AVPacketList*) malloc(sizeof(AVPacketList));
      if (!pkt1) return -1;
      pkt1->pkt = *pkt;
      pkt1->next = NULL;

      SDL_LockMutex(q->mutex);

      if (!q->last_pkt)
         q->first_pkt = pkt1;
      else
         q->last_pkt->next = pkt1;

      q->last_pkt = pkt1;
      q->nb_packets++;
      q->size += pkt1->pkt.size;
      SDL_CondSignal(q->cond);

      SDL_UnlockMutex(q->mutex);
      return 0;
    }

    static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block){
      AVPacketList *pkt1;
      int ret;

      SDL_LockMutex(q->mutex);
      for (;;){
         if( quit){
            ret = -1;
            break;
         }

         pkt1 = q->first_pkt;
         if (pkt1){
            q->first_pkt = pkt1->next;
            if (!q->first_pkt)
               q->last_pkt = NULL;

            q->nb_packets--;
            q->size -= pkt1->pkt.size;
            *pkt = pkt1->pkt;
            av_free(pkt1);
            //free(pkt1);
            ret = 1;
            break;
         }

         else if (!block){
            ret = 0;
            break;
         }
         else{
            SDL_CondWait(q->cond, q->mutex);
         }
      }

      SDL_UnlockMutex(q->mutex);
      return ret;
    }

    int audio_decode_frame(AVCodecContext *aCodecCtx, uint8_t *audio_buf, int buf_size){
      static AVPacket pkt;
      static uint8_t *audio_pkt_data = NULL;
      static int audio_pkt_size = 0;

      int len1, data_size;

      for (;;){
         while (audio_pkt_size > 0){
            data_size = buf_size;
            len1 = avcodec_decode_audio3(aCodecCtx, (int16_t*)audio_buf, &amp;data_size, &amp;pkt);
            if (len1 &lt; 0){
               // if error, skip frame
               audio_pkt_size = 0;
               break;
            }
            audio_pkt_data += len1;
            audio_pkt_size -= len1;
            if (data_size &lt;= 0){
               // No data yet, get more frames
               continue;
            }
            // We have data, return it and come back for more later
            return data_size;
        }

        if (pkt.data)
           av_free_packet(&amp;pkt);
        if (quit) return -1;
        if (packet_queue_get(&amp;audioq, &amp;pkt, 1) &lt; 0) return -1;
        audio_pkt_data = pkt.data;
        audio_pkt_size = pkt.size;
     }
    }

    void audio_callback(void *userdata, Uint8 *stream, int len){
     AVCodecContext *aCodecCtx = (AVCodecContext *)userdata;
     int len1, audio_size;

     static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
     static unsigned int audio_buf_size = 0;
     static unsigned int audio_buf_index = 0;

     while (len > 0){
        if (audio_buf_index >= audio_buf_size){
           // We have already sent all our data; get more
           audio_size = audio_decode_frame(aCodecCtx, audio_buf, sizeof(audio_buf));
           if(audio_size &lt; 0){
              // If error, output silence
              audio_buf_size = 1024; // arbitrary?
              memset(audio_buf, 0, audio_buf_size);
           }
           else{
              audio_buf_size = audio_size;
           }
           audio_buf_index = 0;
       }

       len1 = audio_buf_size - audio_buf_index;
       if (len1 > len)
          len1 = len;
       memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);
       len -= len1;
       stream += len1;
       audio_buf_index += len1;
     }
    }

        void setup_ffmpeg(char* filename)
        {
           if (avformat_open_input(&amp;pFormatCtx, filename, NULL, NULL) != 0){
              fprintf(stderr, "FFmpeg failed to open file %s!\n", filename);
              exit(-1);
           }

           if (av_find_stream_info(pFormatCtx) &lt; 0){
              fprintf(stderr, "FFmpeg failed to retrieve stream info!\n");
              exit(-1);
           }

           // Dump information about file onto standard error
           av_dump_format(pFormatCtx, 0, filename, 0);

           // Find the first video stream
           int i = 0;
           for (i; i &lt; pFormatCtx->nb_streams; i++){
              if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO &amp;&amp; videoStream &lt; 0){
                 videoStream = i;
              }

              if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO &amp;&amp; audioStream &lt; 0){
                 audioStream = i;
              }
           }

           if (videoStream == -1){
              fprintf(stderr, "No video stream found in %s!\n", filename);
              exit(-1);
           }

           if (audioStream == -1){
              fprintf(stderr, "No audio stream found in %s!\n", filename);
              exit(-1);
           }

           // Get a pointer to the codec context for the audio stream
           aCodecCtx = pFormatCtx->streams[audioStream]->codec;

           // Set audio settings from codec info
           SDL_AudioSpec wanted_spec;
           wanted_spec.freq = aCodecCtx->sample_rate;
           wanted_spec.format = AUDIO_S16SYS;
           wanted_spec.channels = aCodecCtx->channels;
           wanted_spec.silence = 0;
           wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
           wanted_spec.callback = audio_callback;
           wanted_spec.userdata = aCodecCtx;

           SDL_AudioSpec spec;
           if (SDL_OpenAudio(&amp;wanted_spec, &amp;spec) &lt; 0){
              fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
              exit(-1);
           }

           AVCodec* aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
           if (!aCodec){
              fprintf(stderr, "Unsupported codec [1]!\n");
              exit(-1);
           }
           avcodec_open(aCodecCtx, aCodec);

           // audio_st = pFormatCtx->streams[index]
           packet_queue_init(&amp;audioq);
           SDL_PauseAudio(0);

           // Get a pointer to the codec context for the video stream
           pCodecCtx = pFormatCtx->streams[videoStream]->codec;

           // Find the decoder for the video stream
           AVCodec* pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
           if (pCodec == NULL){
              fprintf(stderr, "Unsupported codec [2]!\n");
              exit(-1); // Codec not found
           }

           // Open codec
           if (avcodec_open(pCodecCtx, pCodec) &lt; 0){
              fprintf(stderr, "Unsupported codec [3]!\n");
              exit(-1); // Could not open codec
           }
        }


        int main(int argc, char* argv[])
        {
           if (argc &lt; 2){
               std::cout &lt;&lt; "Usage: " &lt;&lt; argv[0] &lt;&lt; " <video>" &lt;&lt; std::endl;
               return -1;
           }

           av_register_all();

           // Init SDL
           if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))
           {
              fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
              return -1;
           }

           // Init ffmpeg and setup some SDL stuff related to Audio
           setup_ffmpeg(argv[1]);

           VideoCapture cap(argv[1]);
           if (!cap.isOpened()){
              std::cout &lt;&lt; "Failed to load file!" &lt;&lt; std::endl;
              return -1;
           }

           AVPacket packet;
           while (av_read_frame(pFormatCtx, &amp;packet) >= 0)
           {
              if (packet.stream_index == videoStream)
              {
                 // Actually this is were SYNC between audio/video would happen.
                 // Right now I assume that every VIDEO packet contains an entire video frame, and that&#39;s not true. A video frame can be made by multiple packets!
                 // But for the time being, assume 1 video frame == 1 video packet,
                 // so instead of reading the frame through ffmpeg, I read it through OpenCV.

                 Mat frame;
                 cap >> frame; // get a new frame from camera

                 // do some processing on the frame, either as a Mat or as IplImage.
                 // For educational purposes, applying a lame grayscale conversion
                 IplImage ipl_frame = frame;
                 for (int i = 0; i &lt; ipl_frame.width * ipl_frame.height * ipl_frame.nChannels; i += ipl_frame.nChannels)
                 {
                    ipl_frame.imageData[i] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3;   //B
                    ipl_frame.imageData[i+1] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //G
                    ipl_frame.imageData[i+2] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //R
                 }

                 // Display it on SDL window
                 show_frame(&amp;ipl_frame);

                 av_free_packet(&amp;packet);
              }
              else if (packet.stream_index == audioStream)
              {
                 packet_queue_put(&amp;audioq, &amp;packet);
              }
              else
              {
                 av_free_packet(&amp;packet);
              }

              SDL_Event event;
              SDL_PollEvent(&amp;event);
              switch (event.type)
              {
              case SDL_QUIT:
                 SDL_FreeSurface(surface);
                 SDL_Quit();
                 break;

              default:
                 break;
              }
           }

           // the camera will be deinitialized automatically in VideoCapture destructor

           // Close the codec
           avcodec_close(pCodecCtx);

           // Close the video file
           av_close_input_file(pFormatCtx);

           return 0;
        }
    </video></iostream>
  • fate : add tests for cdxl video

    21 février 2012, par Paul B Mahol

    fate : add tests for cdxl video