Recherche avancée

Sur d’autres sites (10869)

• iOS : How to fill audioUnit IOdata from avFrame

  4 septembre 2012, par Michelle Cannon

  I'm a little confused, I understand the basis of audio units in general but I can't seem to come up with an easy approach to my problem.

  I have an rtsp player application using ffmpeg and AudioQueues , this works very well for some formats but the audio queue api seems to have issues with mu-law (g.711) audio.
  I did some preliminary testing with another player that uses SDL audio which is a wrapper around audio units and playback was fairly smooth with little latency. I'd like to duplicate this without using the SDL middleware.

  What I am having problems with is how to take the decoded avFrame, I assume using audio units I have to decode using avformat_decode_audio4 since audio units can only handle uncompressed formats unlike audioQueue api that can play compressed formats.

  My audio callback and fillAudio method for audioQueues is like the following
  
  
  
     {{
  
      void audioQueueOutputCallback(void *info, AudioQueueRef unused, AudioQueueBufferRef    buffer) {
  
      // NSLog(@"buffer size on callback %lu ",sizeof(buffer));
  
      [(__bridge FLPlayer *)info fillAudioBuffer:buffer];
  
  }
  
  
  
  
  
  - (void)fillAudioBuffer:(AudioQueueBufferRef)buffer {
  
      AudioTimeStamp bufferStartTime;
  
  
  
      AudioQueueGetCurrentTime(audioQueue, NULL, &bufferStartTime, NULL);
  
  
  
      buffer->mAudioDataByteSize = 0;
  
      buffer->mPacketDescriptionCount = 0;
  
  
  
      if (audioPacketQueue.count <= 0) {
  
          //NSLog(@"Warning: No audio packets in queue  %d ",audioPacketQueue.count);
  
          emptyAudioBuffer = buffer;
  
          return;
  
      }
  
      //NSLog(@" audio packets in queue  %d ",audioPacketQueue.count);
  
      emptyAudioBuffer = nil;
  
  
  
      while (audioPacketQueue.count && buffer->mPacketDescriptionCount < buffer->mPacketDescriptionCapacity) {
  
          NSMutableData *packetData = [audioPacketQueue objectAtIndex:0];
  
          AVPacket *packet = [packetData mutableBytes];
  
  
  
          if (buffer->mAudioDataBytesCapacity - buffer->mAudioDataByteSize >= packet->size) {
  
              if (buffer->mPacketDescriptionCount == 0) {
  
                  bufferStartTime.mSampleTime = (Float64)packet->dts * avfContext->streams[audio_index]->codec->frame_size;
  
                  bufferStartTime.mFlags = kAudioTimeStampSampleTimeValid;
  
              }
  
  
  
              memcpy((uint8_t *)buffer->mAudioData + buffer->mAudioDataByteSize, packet->data, packet->size);
  
              buffer->mPacketDescriptions[buffer->mPacketDescriptionCount].mStartOffset = buffer->mAudioDataByteSize;
  
              buffer->mPacketDescriptions[buffer->mPacketDescriptionCount].mDataByteSize = packet->size;
  
              buffer->mPacketDescriptions[buffer->mPacketDescriptionCount].mVariableFramesInPacket = avfContext->streams[audio_index]->codec->frame_size;
  
  
  
              buffer->mAudioDataByteSize += packet->size;
  
              buffer->mPacketDescriptionCount++;
  
  
  
              [audioPacketQueueLock lock];
  
              audioPacketQueueSize -= packet->size;
  
              [audioPacketQueue removeObjectAtIndex:0];
  
              [audioPacketQueueLock unlock];
  
  
  
              av_free_packet(packet);
  
          }
  
          else {
  
              break;
  
          }
  
      }
  
  
  
      if (buffer->mPacketDescriptionCount > 0) {
  
          OSStatus err;
  
  
  
          err = AudioQueueEnqueueBufferWithParameters(audioQueue, buffer, 0, NULL, 0, 0, 0, NULL, &bufferStartTime, NULL) ;
  
  
  
          if (err != noErr ) {
  
              NSLog(@"Error enqueuing audio buffer: %d", err);
  
          }
  
  
  
          [decodeDoneLock lock];
  
          if (decodeDone && audioPacketQueue.count == 0) {
  
              err = AudioQueueStop(audioQueue, false); 
  
  
  
              if (err != noErr) {
  
                  NSLog(@"Error: Failed to stop audio queue: %d", err);
  
              }
  
              else {
  
                  NSLog(@"Stopped audio queue");
  
              }
  
          }
  
          [decodeDoneLock unlock];
  
      }
  
  }
  
  }}

  but the audioUnit callback looks like the following, how do I fill iodata from my buffers

  #pragma mark Playback callback
  
  
  
  static OSStatus playbackCallback(void *inRefCon, 
  
                                   AudioUnitRenderActionFlags *ioActionFlags, 
  
                                   const AudioTimeStamp *inTimeStamp, 
  
                                   UInt32 inBusNumber, 
  
                                   UInt32 inNumberFrames, 
  
                                   AudioBufferList *ioData) {    

  edited-update *

  Wow this really is a lot more difficult than I thought, why I am not sure, but anyway I think I have most of the pieces.

  So to get my 8bit samples from ffmpeg int sint16 lpcm we can do something like this

  UInt8* packetPtr = packet->data;
  
      int packetSize = packet->size;
  
      int16_t audioBuf[AVCODEC_MAX_AUDIO_FRAME_SIZE];
  
      int dataSize, decodedSize, pts, nextPts;
  
      double decodedAudioTime;
  
      BOOL newPacket = true;
  
      while (0 < packetSize) {
  
          dataSize = AVCODEC_MAX_AUDIO_FRAME_SIZE;
  
          decodedSize = avcodec_decode_audio2(_audioContext(trackId),
  
                                              audioBuf, &dataSize,
  
                                              packetPtr, packetSize);

  now audioBuf has our 16bit samples but we need 32 bit samples so we can do something like this

  - (OSStatus)getSamples:(AudioBufferList*)ioData  
  
  {  
  
      for(UInt32 i = 0; i < ioData->mNumberBuffers; ++i)  
  
      {  
  
          SInt32 *outBuffer = (SInt32 *) (ioData->mBuffers[i].mData);  
  
  
  
          const int mDataByteSize = ioData->mBuffers[i].mDataByteSize;  
  
  
  
          numSamples = mDataByteSize / 4;  
  
  
  
          for(UInt32 j=0; j/do I get more than 1 sample from ffmpeg?? audiobuf needs to feed my ring buffer 
  
          {  
  
              outBuffer[j] = (SInt16) (ringbuf[j] * 32767.0);  
  
          }  
  
      }  
  
  
  
      return noErr;  
  
  }  

• android ffmpeg bad video output

  20 août 2014, par Sujith Manjavana

  I’m following this tutorial to create my first ffmpeg app. I have successfully build the shared libs and compiled the project without any errors. But when i run the app on my nexus 5 the output is this

  Here is the native code

  #include <libavcodec></libavcodec>avcodec.h>
  
  #include <libavformat></libavformat>avformat.h>
  
  #include <libswscale></libswscale>swscale.h>
  
  #include <libavutil></libavutil>pixfmt.h>
  
  
  
  #include 
  
  #include 
  
  
  
  #include 
  
  #include <android></android>native_window.h>
  
  #include <android></android>native_window_jni.h>
  
  
  
  #define LOG_TAG "android-ffmpeg-tutorial02"
  
  #define LOGI(...) __android_log_print(4, LOG_TAG, __VA_ARGS__);
  
  #define LOGE(...) __android_log_print(6, LOG_TAG, __VA_ARGS__);
  
  
  
  ANativeWindow*      window;
  
  char                *videoFileName;
  
  AVFormatContext     *formatCtx = NULL;
  
  int                 videoStream;
  
  AVCodecContext      *codecCtx = NULL;
  
  AVFrame             *decodedFrame = NULL;
  
  AVFrame             *frameRGBA = NULL;
  
  jobject             bitmap;
  
  void*               buffer;
  
  struct SwsContext   *sws_ctx = NULL;
  
  int                 width;
  
  int                 height;
  
  int                 stop;
  
  
  
  jint naInit(JNIEnv *pEnv, jobject pObj, jstring pFileName) {
  
      AVCodec         *pCodec = NULL;
  
      int             i;
  
      AVDictionary    *optionsDict = NULL;
  
  
  
      videoFileName = (char *)(*pEnv)->GetStringUTFChars(pEnv, pFileName, NULL);
  
      LOGI("video file name is %s", videoFileName);
  
      // Register all formats and codecs
  
      av_register_all();
  
      // Open video file
  
      if(avformat_open_input(&amp;formatCtx, videoFileName, NULL, NULL)!=0)
  
          return -1; // Couldn't open file
  
      // Retrieve stream information
  
      if(avformat_find_stream_info(formatCtx, NULL)&lt;0)
  
          return -1; // Couldn't find stream information
  
      // Dump information about file onto standard error
  
      av_dump_format(formatCtx, 0, videoFileName, 0);
  
      // Find the first video stream
  
      videoStream=-1;
  
      for(i=0; inb_streams; i++) {
  
          if(formatCtx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO) {
  
              videoStream=i;
  
              break;
  
          }
  
      }
  
      if(videoStream==-1)
  
          return -1; // Didn't find a video stream
  
      // Get a pointer to the codec context for the video stream
  
      codecCtx=formatCtx->streams[videoStream]->codec;
  
      // Find the decoder for the video stream
  
      pCodec=avcodec_find_decoder(codecCtx->codec_id);
  
      if(pCodec==NULL) {
  
          fprintf(stderr, "Unsupported codec!\n");
  
          return -1; // Codec not found
  
      }
  
      // Open codec
  
      if(avcodec_open2(codecCtx, pCodec, &amp;optionsDict)&lt;0)
  
          return -1; // Could not open codec
  
      // Allocate video frame
  
      decodedFrame=avcodec_alloc_frame();
  
      // Allocate an AVFrame structure
  
      frameRGBA=avcodec_alloc_frame();
  
      if(frameRGBA==NULL)
  
          return -1;
  
      return 0;
  
  }
  
  
  
  jobject createBitmap(JNIEnv *pEnv, int pWidth, int pHeight) {
  
      int i;
  
      //get Bitmap class and createBitmap method ID
  
      jclass javaBitmapClass = (jclass)(*pEnv)->FindClass(pEnv, "android/graphics/Bitmap");
  
      jmethodID mid = (*pEnv)->GetStaticMethodID(pEnv, javaBitmapClass, "createBitmap", "(IILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;");
  
      //create Bitmap.Config
  
      //reference: https://forums.oracle.com/thread/1548728
  
      const wchar_t* configName = L"ARGB_8888";
  
      int len = wcslen(configName);
  
      jstring jConfigName;
  
      if (sizeof(wchar_t) != sizeof(jchar)) {
  
          //wchar_t is defined as different length than jchar(2 bytes)
  
          jchar* str = (jchar*)malloc((len+1)*sizeof(jchar));
  
          for (i = 0; i &lt; len; ++i) {
  
              str[i] = (jchar)configName[i];
  
          }
  
          str[len] = 0;
  
          jConfigName = (*pEnv)->NewString(pEnv, (const jchar*)str, len);
  
      } else {
  
          //wchar_t is defined same length as jchar(2 bytes)
  
          jConfigName = (*pEnv)->NewString(pEnv, (const jchar*)configName, len);
  
      }
  
      jclass bitmapConfigClass = (*pEnv)->FindClass(pEnv, "android/graphics/Bitmap$Config");
  
      jobject javaBitmapConfig = (*pEnv)->CallStaticObjectMethod(pEnv, bitmapConfigClass,
  
              (*pEnv)->GetStaticMethodID(pEnv, bitmapConfigClass, "valueOf", "(Ljava/lang/String;)Landroid/graphics/Bitmap$Config;"), jConfigName);
  
      //create the bitmap
  
      return (*pEnv)->CallStaticObjectMethod(pEnv, javaBitmapClass, mid, pWidth, pHeight, javaBitmapConfig);
  
  }
  
  
  
  jintArray naGetVideoRes(JNIEnv *pEnv, jobject pObj) {
  
      jintArray lRes;
  
      if (NULL == codecCtx) {
  
          return NULL;
  
      }
  
      lRes = (*pEnv)->NewIntArray(pEnv, 2);
  
      if (lRes == NULL) {
  
          LOGI(1, "cannot allocate memory for video size");
  
          return NULL;
  
      }
  
      jint lVideoRes[2];
  
      lVideoRes[0] = codecCtx->width;
  
      lVideoRes[1] = codecCtx->height;
  
      (*pEnv)->SetIntArrayRegion(pEnv, lRes, 0, 2, lVideoRes);
  
      return lRes;
  
  }
  
  
  
  void naSetSurface(JNIEnv *pEnv, jobject pObj, jobject pSurface) {
  
      if (0 != pSurface) {
  
          // get the native window reference
  
          window = ANativeWindow_fromSurface(pEnv, pSurface);
  
          // set format and size of window buffer
  
          ANativeWindow_setBuffersGeometry(window, 0, 0, WINDOW_FORMAT_RGBA_8888);
  
      } else {
  
          // release the native window
  
          ANativeWindow_release(window);
  
      }
  
  }
  
  
  
  jint naSetup(JNIEnv *pEnv, jobject pObj, int pWidth, int pHeight) {
  
      width = pWidth;
  
      height = pHeight;
  
      //create a bitmap as the buffer for frameRGBA
  
      bitmap = createBitmap(pEnv, pWidth, pHeight);
  
      if (AndroidBitmap_lockPixels(pEnv, bitmap, &amp;buffer) &lt; 0)
  
          return -1;
  
      //get the scaling context
  
      sws_ctx = sws_getContext (
  
              codecCtx->width,
  
              codecCtx->height,
  
              codecCtx->pix_fmt,
  
              pWidth,
  
              pHeight,
  
              AV_PIX_FMT_RGBA,
  
              SWS_BILINEAR,
  
              NULL,
  
              NULL,
  
              NULL
  
      );
  
      // Assign appropriate parts of bitmap to image planes in pFrameRGBA
  
      // Note that pFrameRGBA is an AVFrame, but AVFrame is a superset
  
      // of AVPicture
  
      avpicture_fill((AVPicture *)frameRGBA, buffer, AV_PIX_FMT_RGBA,
  
              pWidth, pHeight);
  
      return 0;
  
  }
  
  
  
  void finish(JNIEnv *pEnv) {
  
      //unlock the bitmap
  
      AndroidBitmap_unlockPixels(pEnv, bitmap);
  
      av_free(buffer);
  
      // Free the RGB image
  
      av_free(frameRGBA);
  
      // Free the YUV frame
  
      av_free(decodedFrame);
  
      // Close the codec
  
      avcodec_close(codecCtx);
  
      // Close the video file
  
      avformat_close_input(&amp;formatCtx);
  
  }
  
  
  
  void decodeAndRender(JNIEnv *pEnv) {
  
      ANativeWindow_Buffer    windowBuffer;
  
      AVPacket                packet;
  
      int                     i=0;
  
      int                     frameFinished;
  
      int                     lineCnt;
  
      while(av_read_frame(formatCtx, &amp;packet)>=0 &amp;&amp; !stop) {
  
          // Is this a packet from the video stream?
  
          if(packet.stream_index==videoStream) {
  
              // Decode video frame
  
              avcodec_decode_video2(codecCtx, decodedFrame, &amp;frameFinished,
  
                 &amp;packet);
  
              // Did we get a video frame?
  
              if(frameFinished) {
  
                  // Convert the image from its native format to RGBA
  
                  sws_scale
  
                  (
  
                      sws_ctx,
  
                      (uint8_t const * const *)decodedFrame->data,
  
                      decodedFrame->linesize,
  
                      0,
  
                      codecCtx->height,
  
                      frameRGBA->data,
  
                      frameRGBA->linesize
  
                  );
  
                  // lock the window buffer
  
                  if (ANativeWindow_lock(window, &amp;windowBuffer, NULL) &lt; 0) {
  
                      LOGE("cannot lock window");
  
                  } else {
  
                      // draw the frame on buffer
  
                      LOGI("copy buffer %d:%d:%d", width, height, width*height*4);
  
                      LOGI("window buffer: %d:%d:%d", windowBuffer.width,
  
                              windowBuffer.height, windowBuffer.stride);
  
                      memcpy(windowBuffer.bits, buffer,  width * height * 4);
  
                      // unlock the window buffer and post it to display
  
                      ANativeWindow_unlockAndPost(window);
  
                      // count number of frames
  
                      ++i;
  
                  }
  
              }
  
          }
  
          // Free the packet that was allocated by av_read_frame
  
          av_free_packet(&amp;packet);
  
      }
  
      LOGI("total No. of frames decoded and rendered %d", i);
  
      finish(pEnv);
  
  }
  
  
  
  /**
  
   * start the video playback
  
   */
  
  void naPlay(JNIEnv *pEnv, jobject pObj) {
  
      //create a new thread for video decode and render
  
      pthread_t decodeThread;
  
      stop = 0;
  
      pthread_create(&amp;decodeThread, NULL, decodeAndRender, NULL);
  
  }
  
  
  
  /**
  
   * stop the video playback
  
   */
  
  void naStop(JNIEnv *pEnv, jobject pObj) {
  
      stop = 1;
  
  }
  
  
  
  jint JNI_OnLoad(JavaVM* pVm, void* reserved) {
  
      JNIEnv* env;
  
      if ((*pVm)->GetEnv(pVm, (void **)&amp;env, JNI_VERSION_1_6) != JNI_OK) {
  
           return -1;
  
      }
  
      JNINativeMethod nm[8];
  
      nm[0].name = "naInit";
  
      nm[0].signature = "(Ljava/lang/String;)I";
  
      nm[0].fnPtr = (void*)naInit;
  
  
  
      nm[1].name = "naSetSurface";
  
      nm[1].signature = "(Landroid/view/Surface;)V";
  
      nm[1].fnPtr = (void*)naSetSurface;
  
  
  
      nm[2].name = "naGetVideoRes";
  
      nm[2].signature = "()[I";
  
      nm[2].fnPtr = (void*)naGetVideoRes;
  
  
  
      nm[3].name = "naSetup";
  
      nm[3].signature = "(II)I";
  
      nm[3].fnPtr = (void*)naSetup;
  
  
  
      nm[4].name = "naPlay";
  
      nm[4].signature = "()V";
  
      nm[4].fnPtr = (void*)naPlay;
  
  
  
      nm[5].name = "naStop";
  
      nm[5].signature = "()V";
  
      nm[5].fnPtr = (void*)naStop;
  
  
  
      jclass cls = (*env)->FindClass(env, "roman10/tutorial/android_ffmpeg_tutorial02/MainActivity");
  
      //Register methods with env->RegisterNatives.
  
      (*env)->RegisterNatives(env, cls, nm, 6);
  
      return JNI_VERSION_1_6;
  
  }

  Here is the build.sh

  #!/bin/bash
  
  NDK=$HOME/Desktop/adt/android-ndk-r9
  
  SYSROOT=$NDK/platforms/android-9/arch-arm/
  
  TOOLCHAIN=$NDK/toolchains/arm-linux-androideabi-4.8/prebuilt/linux-x86_64
  
  function build_one
  
  {
  
  ./configure \
  
      --prefix=$PREFIX \
  
      --enable-shared \
  
      --disable-static \
  
      --disable-doc \
  
      --disable-ffmpeg \
  
      --disable-ffplay \
  
      --disable-ffprobe \
  
      --disable-ffserver \
  
      --disable-avdevice \
  
      --disable-doc \
  
      --disable-symver \
  
      --cross-prefix=$TOOLCHAIN/bin/arm-linux-androideabi- \
  
      --target-os=linux \
  
      --arch=arm \
  
      --enable-cross-compile \
  
      --sysroot=$SYSROOT \
  
      --extra-cflags="-Os -fpic $ADDI_CFLAGS" \
  
      --extra-ldflags="$ADDI_LDFLAGS" \
  
      $ADDITIONAL_CONFIGURE_FLAG
  
  make clean
  
  make
  
  make install
  
  }
  
  CPU=arm
  
  PREFIX=$(pwd)/android/$CPU 
  
  ADDI_CFLAGS="-marm"
  
  build_one

  It works on the Galaxy tab2. what can i do to make it work on all devices ?? Please help me..

• Merge commit ’30e50c50274f88f0f5ae829f401cd3c7f5266719’

  14 août 2014, par Michael Niedermayer

  Merge commit ’30e50c50274f88f0f5ae829f401cd3c7f5266719’
  * commit ’30e50c50274f88f0f5ae829f401cd3c7f5266719’ :
  
    lavf : eliminate ff_get_audio_frame_size()
  Conflicts :
  
  	libavformat/utils.c
  Not merged as the removed code is needed for some bugfixes and
  
  would break the fate tests
  Merged-by : Michael Niedermayer <michaelni@gmx.at>