Recherche avancée

Sur d’autres sites (7877)

• Android sws_scale RGB0 Frame taking long time and cause video latency [duplicate]

  17 avril 2018, par AJit

  This question already has an answer here :

  • Hardware scaling the OpenGL surface with NativeActivity
    
    1 answer
    

  Reading frames from FFMPEG and try to directly draw on a surface window in Native android, If scale the image to exact size what we are getting from the camera and apply YUV420P to RGBA it take 1ms to scale through av_image_fill_arrays but if scale image surface size, then It took 25 to 30ms to scale a same frame.

  Low latency :[ 1ms by sws_scale]

  swsContext = sws_getContext(videoCodecContext->width,
  
                                      videoCodecContext->height,
  
                                      videoCodecContext->pix_fmt,
  
                                      videoCodecContext->width,
  
                                      videoCodecContext->height,
  
                                      AV_PIX_FMT_RGB0,
  
                                      SWS_FAST_BILINEAR, NULL, NULL, NULL);
  
  av_image_fill_arrays()
  
  av_read_frame()
  
  avcodec_decode_video2(
  
  sws_scale(swsContext,
  
            (const uint8_t *const *) videoFrame->data,
  
            videoFrame->linesize,
  
            0,
  
            videoCodecContext->height,
  
            pictureFrame->data,
  
            pictureFrame->linesize);
  
  ANativeWindow_lock()
  
  Write all buffer bytes to window.
  
  ANativeWindow_unlockAndPost()

  Low latency :[ 30ms by sws_scale]

  [videoContext Width: 848 Height: 608]
  
  swsContext = sws_getContext(videoCodecContext->width,
  
                                      videoCodecContext->height,
  
                                      videoCodecContext->pix_fmt,
  
                                      1080,
  
                                      608,
  
                                      AV_PIX_FMT_RGB0,
  
                                      SWS_FAST_BILINEAR, NULL, NULL, NULL);

  NOTE : we are getting YUV420P pixel format.

  Whenever we change context width and height other than videoContext it will take more than 30ms so result is video delaying.

  TRY 1 : Pass the buffer from JNI to Java and create bitmap there to scale later on but createBitmap itself take ~500ms which is not good enough.

  TRY 2 : Direct YUV420P to RGB conversion, but still sws_scale is greater.

  TRY 3 : Direct writes YUV to window bites, but it shows without color (If anyone has a solution here will might helpful).

  TRY 4 : yuvlib, still not worth it.

  TRY 5 : Different pixel formats and flags in swsContext.

  

  Any help would appreciated.

• How to get picture buffer data in ffmpeg ?

  5 juin 2014, par 4ntoine

  I’m trying to pass bitmap from ffmpeg to android.
  It already works but it’s displaying picture right on surface passed from java to native code.
  How can i get frame buffer bitmap data to pass it to java ?

  I’ve tried to save out_frame buffer data :

  unsigned char bmpFileHeader[14] = {'B', 'M', 0,0,0,0, 0,0, 0,0, 54, 0,0,0};
  
  unsigned char bmpInfoHeader[40] = {40,0,0,0, 0,0,0,0, 0,0,0,0, 1,0, 24,0};
  
  unsigned char bmpPad[3] = {0, 0, 0};
  
  
  
  void saveBuffer(int fileIndex, int width, int height, unsigned char *buffer, int buffer_size) {
  
  
  
      unsigned char filename[1024];
  
      sprintf(filename, "/storage/sdcard0/3d_player_%d.bmp", fileIndex);
  
  
  
      LOGI(10, "saving ffmpeg bitmap file: %d to %s", fileIndex, filename);
  
  
  
      FILE *bitmapFile = fopen(filename, "wb");
  
      if (!bitmapFile) {
  
          LOGE(10, "failed to create ffmpeg bitmap file");
  
          return;
  
      }
  
  
  
      unsigned char filesize = 54 + 3 * width * height; // 3 = (r,g,b)
  
  
  
      bmpFileHeader[2] = (unsigned char)(filesize);
  
      bmpFileHeader[3] = (unsigned char)(filesize >> 8);
  
      bmpFileHeader[4] = (unsigned char)(filesize >> 16);
  
      bmpFileHeader[5] = (unsigned char)(filesize >> 24);
  
  
  
      bmpInfoHeader[4] = (unsigned char)(width);
  
      bmpInfoHeader[5] = (unsigned char)(width >> 8);
  
      bmpInfoHeader[6] = (unsigned char)(width >> 16);
  
      bmpInfoHeader[7] = (unsigned char)(width >> 24);
  
      bmpInfoHeader[8] = (unsigned char)(height);
  
      bmpInfoHeader[9] = (unsigned char)(height >> 8);
  
      bmpInfoHeader[10] = (unsigned char)(height >> 16);
  
      bmpInfoHeader[11] = (unsigned char)(height >> 24);
  
  
  
      fwrite(bmpFileHeader, 1, 14, bitmapFile);
  
      fwrite(bmpInfoHeader, 1, 40, bitmapFile);
  
      int i;
  
      for (i=0; iplayer;
  
      int stream_no = decoder_data->stream_no;
  
      AVCodecContext * ctx = player->input_codec_ctxs[stream_no];
  
      AVFrame * frame = player->input_frames[stream_no];
  
      AVStream * stream = player->input_streams[stream_no];
  
      int interrupt_ret;
  
      int to_write;
  
      int err = 0;
  
      AVFrame *rgb_frame = player->rgb_frame;
  
      ANativeWindow_Buffer buffer;
  
      ANativeWindow * window;
  
  
  
  #ifdef MEASURE_TIME
  
      struct timespec timespec1, timespec2, diff;
  
  #endif // MEASURE_TIME
  
      LOGI(10, "player_decode_video decoding");
  
      int frameFinished;
  
  
  
  #ifdef MEASURE_TIME
  
      clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec1);
  
  #endif // MEASURE_TIME
  
      int ret = avcodec_decode_video2(ctx, frame, &frameFinished,
  
              packet_data->packet);
  
  
  
  #ifdef MEASURE_TIME
  
      clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec2);
  
      diff = timespec_diff(timespec1, timespec2);
  
      LOGI(3, "decode_video timediff: %d.%9ld", diff.tv_sec, diff.tv_nsec);
  
  #endif // MEASURE_TIME
  
  
  
      if (ret < 0) {
  
          LOGE(1, "player_decode_video Fail decoding video %d\n", ret);
  
          return -ERROR_WHILE_DECODING_VIDEO;
  
      }
  
      if (!frameFinished) {
  
          LOGI(10, "player_decode_video Video frame not finished\n");
  
          return 0;
  
      }
  
  
  
      // saving in buffer converted video frame
  
      LOGI(7, "player_decode_video copy wait");
  
  
  
  #ifdef MEASURE_TIME
  
      clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec1);
  
  #endif // MEASURE_TIME
  
  
  
      pthread_mutex_lock(&player->mutex_queue);
  
      window = player->window;
  
      if (window == NULL) {
  
          pthread_mutex_unlock(&player->mutex_queue);
  
          goto skip_frame;
  
      }
  
      ANativeWindow_setBuffersGeometry(window, ctx->width, ctx->height,
  
              WINDOW_FORMAT_RGBA_8888);
  
      if (ANativeWindow_lock(window, &buffer, NULL) != 0) {
  
          pthread_mutex_unlock(&player->mutex_queue);
  
          goto skip_frame;
  
      }
  
      pthread_mutex_unlock(&player->mutex_queue);
  
  
  
      int format = buffer.format;
  
      if (format < 0) {
  
          LOGE(1, "Could not get window format")
  
      }
  
      enum PixelFormat out_format;
  
      if (format == WINDOW_FORMAT_RGBA_8888) {
  
          out_format = PIX_FMT_RGBA;
  
          LOGI(6, "Format: WINDOW_FORMAT_RGBA_8888");
  
      } else if (format == WINDOW_FORMAT_RGBX_8888) {
  
          out_format = PIX_FMT_RGB0;
  
          LOGE(1, "Format: WINDOW_FORMAT_RGBX_8888 (not supported)");
  
      } else if (format == WINDOW_FORMAT_RGB_565) {
  
          out_format = PIX_FMT_RGB565;
  
          LOGE(1, "Format: WINDOW_FORMAT_RGB_565 (not supported)");
  
      } else {
  
          LOGE(1, "Unknown window format");
  
      }
  
  
  
      avpicture_fill((AVPicture *) rgb_frame, buffer.bits, out_format,
  
              buffer.width, buffer.height);
  
      rgb_frame->data[0] = buffer.bits;
  
      if (format == WINDOW_FORMAT_RGBA_8888) {
  
          rgb_frame->linesize[0] = buffer.stride * 4;
  
      } else {
  
          LOGE(1, "Unknown window format");
  
      }
  
      LOGI(6,
  
              "Buffer: width: %d, height: %d, stride: %d",
  
              buffer.width, buffer.height, buffer.stride);
  
      int i = 0;
  
  
  
  #ifdef MEASURE_TIME
  
      clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec2);
  
      diff = timespec_diff(timespec1, timespec2);
  
      LOGI(1,
  
              "lockPixels and fillimage timediff: %d.%9ld", diff.tv_sec, diff.tv_nsec);
  
  #endif // MEASURE_TIME
  
  #ifdef MEASURE_TIME
  
      clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &timespec1);
  
  #endif // MEASURE_TIME
  
      LOGI(7, "player_decode_video copying...");
  
      AVFrame * out_frame;
  
      int rescale;
  
      if (ctx->width == buffer.width && ctx->height == buffer.height) {
  
          // This always should be true
  
          out_frame = rgb_frame;
  
          rescale = FALSE;
  
      } else {
  
          out_frame = player->tmp_frame2;
  
          rescale = TRUE;
  
      }
  
  
  
      if (ctx->pix_fmt == PIX_FMT_YUV420P) {
  
          __I420ToARGB(frame->data[0], frame->linesize[0], frame->data[2],
  
                  frame->linesize[2], frame->data[1], frame->linesize[1],
  
                  out_frame->data[0], out_frame->linesize[0], ctx->width,
  
                  ctx->height);
  
      } else if (ctx->pix_fmt == PIX_FMT_NV12) {
  
          __NV21ToARGB(frame->data[0], frame->linesize[0], frame->data[1],
  
                  frame->linesize[1], out_frame->data[0], out_frame->linesize[0],
  
                  ctx->width, ctx->height);
  
      } else {
  
          LOGI(3, "Using slow conversion: %d ", ctx->pix_fmt);
  
          struct SwsContext *sws_context = player->sws_context;
  
          sws_context = sws_getCachedContext(sws_context, ctx->width, ctx->height,
  
                  ctx->pix_fmt, ctx->width, ctx->height, out_format,
  
                  SWS_FAST_BILINEAR, NULL, NULL, NULL);
  
          player->sws_context = sws_context;
  
          if (sws_context == NULL) {
  
              LOGE(1, "could not initialize conversion context from: %d"
  
              ", to :%d\n", ctx->pix_fmt, out_format);
  
              // TODO some error
  
          }
  
          sws_scale(sws_context, (const uint8_t * const *) frame->data,
  
                  frame->linesize, 0, ctx->height, out_frame->data,
  
                  out_frame->linesize);
  
      }
  
  
  
      if (rescale) {
  
          // Never occurs
  
          __ARGBScale(out_frame->data[0], out_frame->linesize[0], ctx->width,
  
                  ctx->height, rgb_frame->data[0], rgb_frame->linesize[0],
  
                  buffer.width, buffer.height, __kFilterNone);
  
          out_frame = rgb_frame;
  
      }
  
  
  
      // TODO: (4ntoine) frame decoded and rescaled, ready to call callback with frame picture from buffer
  
      int bufferSize = buffer.width * buffer.height * 3; // 3 = (r,g,b);  
  
  
  
      static int bitmapCounter = 0;
  
      if (bitmapCounter < 10) {
  
          saveBuffer(bitmapCounter++, buffer.width, buffer.height, (unsigned char *)out_frame->data, bufferSize);
  
      }

  but out_frame is empty and file has header and 0x00 bytes body.

  How to get picture buffer data in ffmpeg ?

• phantomjs screenshots and ffmpeg frame missing

  20 octobre 2015, par Bussiness Way

  I have problem making video from website screenshots taken from phantomjs.

  the phantomjs did not make screenshots for all frames within the same second and even not all seconds there , there is huge missing frames .

  the result is high speed video playing with many jumps in video effects .

  test.js :

  var page = require('webpage').create(),
  
      address = 'http://raphaeljs.com/polar-clock.html',
  
      duration = 5, // duration of the video, in seconds
  
      framerate = 24, // number of frames per second. 24 is a good value.
  
      counter = 0,
  
      width = 1024,
  
      height = 786;
  
          frame = 10001;
  
  
  
  page.viewportSize = { width: width, height: height };
  
  
  
  page.open(address, function(status) {
  
      if (status !== 'success') {
  
          console.log('Unable to load the address!');
  
          phantom.exit(1);
  
      } else {
  
          window.setTimeout(function () {
  
              page.clipRect = { top: 0, left: 0, width: width, height: height };
  
  
  
              window.setInterval(function () {
  
                  counter++;
  
                  page.render('newtest/image'+(frame++)+'.png', { format: 'png' });
  
                  if (counter > duration * framerate) {
  
                      phantom.exit();
  
                  }
  
              }, 1/framerate);
  
          }, 200);
  
      }
  
  });

  this will create 120 image , this is correct count , but when you see the images one by one you will see many duplicate the same contents and many missing frames

  ffmpeg :

  fmpeg -start_number 10001 -i newtest/image%05d.png -c:v libx264 -r 24 -pix_fmt yuv420p out.mp4

  I know this script and ffmpeg command not perfect , because I did hundred of changes without lucky, and I lost the correct setting understanding .

  an anyone guide me to fix this ?.

  thank you all