Recherche avancée

Sur d’autres sites (7468)

• iOS FFMPEG encode images to video

  10 avril 2013, par brad.roush

  I am trying to take a set of UIImages and create a video file out of them with FFMPEG. Seems there are lots of questions about this topic but non have been able to get this working correctly for me. This one was particularly helpful in giving me a starting point. This iFrameExtractor example was also very helpful but I want to do this in reverse, then add audio.

  This is the closest I have gotten and it creates a short silent video with flashing colors and no images :

  // Register all formats and codecs
  
  av_register_all();
  
  
  
  AVCodec *codec;
  
  AVCodecContext *c= NULL;
  
  
  
  int i, out_size, size, outbuf_size;
  
  FILE *file;
  
  AVFrame *picture;
  
  uint8_t *outbuf;
  
  
  
  NSLog(@"Video encoding");
  
  
  
  /* find the mpeg video encoder */
  
  codec = avcodec_find_encoder(CODEC_ID_MPEG2VIDEO);
  
  if (!codec) {
  
      fprintf(stderr, "codec not found\n");
  
      exit(1);
  
  }
  
  
  
  c= avcodec_alloc_context3(codec);
  
  picture= avcodec_alloc_frame();
  
  
  
  /* put sample parameters */
  
  c->bit_rate = 400000;
  
  /* resolution must be a multiple of two */
  
  c->width = 352;
  
  c->height = 288;
  
  /* frames per second */
  
  c->time_base= (AVRational){1,25};
  
  c->gop_size = 10; /* emit one intra frame every ten frames */
  
  c->max_b_frames=1;
  
  c->pix_fmt = PIX_FMT_YUV420P;
  
  
  
  /* open it */
  
  if (avcodec_open2(c, codec, nil) < 0) {
  
      fprintf(stderr, "could not open codec\n");
  
      exit(1);
  
  }
  
  
  
  // Put file in place
  
  
  
  NSArray *paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);
  
  NSString *docs_dir = [paths objectAtIndex:0];
  
  NSString *filePath = [docs_dir stringByAppendingPathComponent:@"test.mpeg"];
  
  
  
  // Seed file
  
  NSString *sourcePath = [[NSBundle mainBundle] pathForResource:@"test" ofType:@"mov"];
  
  NSError* error = nil;
  
  if (![[NSFileManager defaultManager] copyItemAtPath:sourcePath toPath:filePath error:&error]) {
  
      NSLog(@"Test Video creation failed:%@",[error userInfo]);
  
  } else NSLog(@"Test Video Created");
  
  
  
  const char *filename = [filePath UTF8String];
  
  file = fopen(filename, "wb");
  
  if (!file) {
  
      fprintf(stderr, "could not open %s\n", "filename");
  
      exit(1);
  
  }
  
  
  
  /* alloc image and output buffer */
  
  outbuf_size = 100000;
  
  outbuf = malloc(outbuf_size);
  
  size = c->width * c->height;
  
  
  
  //#pragma mark -
  
  AVFrame* outpic = avcodec_alloc_frame();
  
  int nbytes = avpicture_get_size(PIX_FMT_YUV420P, c->width, c->height);
  
  
  
  //create buffer for the output image
  
  uint8_t* outbuffer = (uint8_t*)av_malloc(nbytes);
  
  
  
  
  
  AVPacket packet;
  
  av_init_packet(&packet);
  
  
  
  
  
  //#pragma mark -
  
  for(i=1;i<50;i++) {
  
      fflush(stdout);
  
  
  
      int numBytes = avpicture_get_size(PIX_FMT_YUV420P, c->width, c->height);
  
      uint8_t *buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
  
  
  
      UIImage *image = [UIImage imageWithContentsOfFile:[docs_dir stringByAppendingPathComponent:[NSString stringWithFormat:@"%d.png",i]]];
  
      CGImageRef newCgImage = [image CGImage];
  
  
  
      CGDataProviderRef dataProvider = CGImageGetDataProvider(newCgImage);
  
      CFDataRef bitmapData = CGDataProviderCopyData(dataProvider);
  
      buffer = (uint8_t *)CFDataGetBytePtr(bitmapData);
  
  
  
      avpicture_fill((AVPicture*)picture, buffer, PIX_FMT_RGB8, c->width, c->height);
  
      avpicture_fill((AVPicture*)outpic, outbuffer, PIX_FMT_YUV420P, c->width, c->height);
  
  
  
      struct SwsContext* fooContext = sws_getContext(c->width, c->height,
  
                                                     PIX_FMT_RGB8,
  
                                                     c->width, c->height,
  
                                                     PIX_FMT_YUV420P,
  
                                                     SWS_FAST_BILINEAR, NULL, NULL, NULL);
  
  
  
      //perform the conversion
  
  
  
      sws_scale(fooContext, outpic->data, outpic->linesize,
  
                0, c->height, outpic->data, outpic->linesize);
  
      // Tried This but it didn't work
  
      //sws_scale(fooContext, picture->data, picture->linesize, 0, c->height, outpic->data, outpic->linesize);
  
  
  
  
  
      out_size = avcodec_encode_video(c, outbuf, outbuf_size, outpic);
  
      // Tried this but it didn't work
  
      //int test = 0;
  
      //out_size = avcodec_encode_video2(c, &packet, outpic, &test);
  
  
  
  
  
  
  
      printf("encoding frame %3d (size=%5d)\n", i, out_size);
  
      fwrite(outbuf, 1, out_size, file);
  
  
  
      free(buffer);
  
      buffer = NULL;
  
  
  
  }
  
  
  
  /* get the delayed frames */
  
  /*
  
  for(; out_size; i++) {
  
      fflush(stdout);
  
  
  
      out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
  
      printf("write frame %3d (size=%5d)\n", i, out_size);
  
      fwrite(outbuf, 1, outbuf_size, file);
  
  }
  
  */
  
  
  
  /* add sequence end code to have a real mpeg file */
  
  outbuf[0] = 0x00;
  
  outbuf[1] = 0x00;
  
  outbuf[2] = 0x01;
  
  outbuf[3] = 0xb7;
  
  fwrite(outbuf, 1, 4, file);
  
  fclose(file);
  
  free(outbuf);
  
  
  
  avcodec_close(c);
  
  av_free(c);
  
  av_free(picture);
  
  printf("\n");

  Any ideas will be helpful here. If anyone knows of any other good objective-c examples, this would also be great.

• Twilio Real-Time Media Streaming to WebSocket Receives Only Noise Instead of Speech

  21 février, par dannym25

  I'm setting up a Twilio Voice call with real-time media streaming to a WebSocket server for speech-to-text processing using Google Cloud Speech-to-Text. The connection is established successfully, and I receive a continuous stream of audio data from Twilio. However, when I play back the received audio, all I hear is a rapid clicking/jackhammering noise instead of the actual speech spoken during the call.

  


  Setup :

  


  

  • Twilio sends inbound audio to my WebSocket server.

  


  • WebSocket receives and saves the raw mulaw-encoded audio data from Twilio.

  


  • The audio is processed via Google Speech-to-Text for transcription.

  


  • When I attempt to play back the audio, it sounds like machine-gun-like noise instead of spoken words.

  


  


  1. Confirmed WebSocket Receives Data

  


  • The WebSocket successfully logs incoming audio chunks from Twilio :

  


  🔊 Received 379 bytes of audio from Twilio
🔊 Received 379 bytes of audio from Twilio


  


  • This suggests Twilio is sending audio data, but it's not being interpreted correctly.

  


  2. Saving and Playing Raw Audio

  


  • I save the incoming raw mulaw (8000Hz) audio from Twilio to a file :

  


  fs.appendFileSync('twilio-audio.raw', message);


  


  • Then, I convert it to a .wav file using FFmpeg :

  


  ffmpeg -f mulaw -ar 8000 -ac 1 -i twilio-audio.raw twilio-audio.wav


  


  • Problem : When I play the audio using ffplay, it contains no speech, only rapid clicking sounds.

  


  3. Ensured Correct Audio Encoding

  


  • Twilio sends mulaw 8000Hz mono format.
• Verified that my ffmpeg conversion is using the same settings.
• Tried different conversion methods :

  


  ffmpeg -f mulaw -ar 8000 -ac 1 -i twilio-audio.raw -c:a pcm_s16le twilio-audio-fixed.wav


  


  → Same issue.

  


  4. Checked Google Speech-to-Text Input Format

  


  • Google STT requires proper encoding configuration :

  


  const request = {
    config: {
        encoding: 'MULAW',
        sampleRateHertz: 8000,
        languageCode: 'en-US',
    },
    interimResults: false,
};


  


  • No errors from Google STT, but it never detects speech, likely because the input audio is just noise.

  


  5. Confirmed That Raw Audio is Not a WAV File

  


  • Since Twilio sends raw audio, I checked whether I needed to strip the header before processing.
• Tried manually extracting raw bytes, but the issue persists.

  


  Current Theory :

  


  

  • The WebSocket server might be handling Twilio’s raw audio incorrectly before saving it.

  


  • There might be an additional header in the Twilio stream that needs to be removed before playback.

  


  • Twilio’s <stream></stream> tag expects a WebSocket connection starting with wss:// instead of https://, and switching to wss:// partially fixed some previous connection issues.

  


  


  Code Snippets :

  


  Twilio Setup in TwiML Response

  


  app.post('/voice-response', (req, res) => {
    console.log("📞 Incoming call from Twilio");

    const twiml = new twilio.twiml.VoiceResponse();
    twiml.say("Hello! Welcome to the service. How can I help you?");
    
    // Prevent Twilio from hanging up too early
    twiml.pause({ length: 5 });

    twiml.connect().stream({
        url: `wss://your-ngrok-url/ws`,
        track: "inbound_track"
    });

    console.log("🛠️ Twilio Stream URL:", `wss://your-ngrok-url/ws`);
    
    res.type('text/xml').send(twiml.toString());
});


  


  WebSocket Server Handling Twilio Audio Stream

  


  wss.on('connection', (ws) => {
    console.log("🔗 WebSocket Connected! Waiting for audio input...");

    ws.on('message', (message) => {
        console.log(`🔊 Received ${message.length} bytes of audio from Twilio`);

        // Save raw audio data for debugging
        fs.appendFileSync('twilio-audio.raw', message);

        // Check if audio is non-empty but contains only noise
        if (message.length < 100) {
            console.warn("⚠️ Warning: Audio data from Twilio is very small. Might be silent.");
        }
    });

    ws.on('close', () => {
        console.log("❌ WebSocket Disconnected!");
        
        // Convert Twilio audio for debugging
        exec(`ffmpeg -f mulaw -ar 8000 -ac 1 -i twilio-audio.raw twilio-audio.wav`, (err) => {
            if (err) console.error("❌ FFmpeg Conversion Error:", err);
            else console.log("✅ Twilio Audio Saved as `twilio-audio.wav`");
        });
    });

    ws.on('error', (error) => console.error("⚠️ WebSocket Error:", error));
});


  


  Questions :

  


  

  • Why is the audio from Twilio being received as a clicking noise instead of actual speech ?

  


  • Do I need to strip any additional metadata from the raw bytes before saving ?

  


  • Is there a known issue with Twilio’s mulaw format when streaming audio over WebSockets ?

  


  • How can I confirm that Google STT is receiving properly formatted audio ?

  


  


  Additional Context :

  


  

  • Twilio <stream></stream> is connected and receiving data (confirmed by logs).

  


  • WebSocket successfully receives and saves audio, but it only plays noise.

  


  • Tried multiple ffmpeg conversions, Google STT configurations, and raw data inspection.

  


  • Still no recognizable speech in the audio output.

  


  


  Any help is greatly appreciated ! 🙏

  


• VLC Ubuntu : Problems converting rstp streaming into http streaming

  16 janvier 2019, par Gorka Sanz Monllor

  I am trying to convert in a Linux Ubuntu Machine a rstp stream into a http stream, so I execute the following comand :

  vlc -vvv -Idummy rtsp://**************************************transportmode=unicast --sout #transcode{vcodec=MJPG,venc=ffmpeg{strict=1}}:standard{access=http{mime=multipart/x-mixed-replace;boundary=--7b3cc56e5f51db803f790dad720ed50a},mux=mpjpeg,dst=:8080/} --run-time= 300 

  vlc ://quit

  and it doesn’t work. The output is the following :

  root@f139da580fd9:/# vlc -vvv -Idummy rtsp://*********Streaming/Channels/101?transportmode=unicast --sout #transcode{vcodec=MJPG,venc=ffmpeg{strict=1}}:standard{access=ht                  tp{mime=multipart/x-mixed-replace;boundary=--7b3cc56e5f51db803f790dad720ed50a},mux=mpjpeg,dst=:8080/} --run-time= 300 vlc://quit
  
  
  
  VLC media player 2.2.2 Weatherwax (revision 2.2.2-0-g6259d80)
  
  [0000000000a9c178] core libvlc debug: VLC media player - 2.2.2 Weatherwax
  
  [0000000000a9c178] core libvlc debug: Copyright © 1996-2016 the VideoLAN team
  
  [0000000000a9c178] core libvlc debug: revision 2.2.2-0-g6259d80
  
  [0000000000a9c178] core libvlc debug: configured with ./configure  '--build=x86_64-linux-gnu' '--prefix=/usr' '--includedir=${prefix}/include' '--mandir=${prefix}/share/man' '--infodir=${pr                  efix}/share/info' '--sysconfdir=/etc' '--localstatedir=/var' '--disable-silent-rules' '--libdir=${prefix}/lib/x86_64-linux-gnu' '--libexecdir=${prefix}/lib/x86_64-linux-gnu' '--disable-main                  tainer-mode' '--disable-dependency-tracking' '--config-cache' '--disable-update-check' '--enable-fast-install' '--docdir=/usr/share/doc/vlc-data' '--libdir=/usr/lib' '--with-binary-version=                  2.2.2-5ubuntu0.16.04.4' '--enable-a52' '--enable-aa' '--enable-bluray' '--enable-bonjour' '--enable-caca' '--enable-chromaprint' '--enable-dbus' '--enable-dca' '--enable-directfb' '--enable                  -dvbpsi' '--enable-dvdnav' '--enable-faad' '--enable-flac' '--enable-fluidsynth' '--enable-freerdp' '--enable-freetype' '--enable-fribidi' '--disable-gles1' '--enable-gles2' '--enable-gnutl                  s' '--enable-jack' '--enable-kate' '--enable-libass' '--enable-libmpeg2' '--enable-libxml2' '--enable-lirc' '--enable-live555' '--enable-mad' '--enable-mkv' '--enable-mod' '--enable-mpc' '-                  -enable-mtp' '--enable-mux_ogg' '--enable-ncurses' '--enable-notify' '--enable-ogg' '--enable-opus' '--enable-pulse' '--enable-qt' '--enable-realrtsp' '--enable-samplerate' '--enable-schroe                  dinger' '--enable-sdl' '--enable-sdl-image' '--enable-sftp' '--enable-shine' '--enable-shout' '--enable-skins2' '--enable-speex' '--enable-svg' '--enable-svgdec' '--enable-taglib' '--enable                  -theora' '--enable-twolame' '--enable-upnp' '--enable-vcdx' '--enable-vdpau' '--enable-vnc' '--enable-vorbis' '--enable-x264' '--enable-x265' '--enable-zvbi' '--with-kde-solid=/usr/share/kd                  e4/apps/solid/actions/' '--disable-decklink' '--disable-dxva2' '--disable-fdkaac' '--disable-gnomevfs' '--disable-goom' '--disable-libtar' '--disable-mfx' '--disable-opencv' '--disable-proj                  ectm' '--disable-sndio' '--disable-telx' '--disable-vpx' '--disable-vsxu' '--disable-wasapi' '--enable-alsa' '--enable-atmo' '--enable-dc1394' '--enable-dv1394' '--enable-linsys' '--enable-                  omxil' '--enable-udev' '--enable-v4l2' '--enable-libva' '--enable-vcd' '--enable-smbclient' '--disable-oss' '--enable-crystalhd' '--enable-mmx' '--enable-sse' '--disable-neon' '--disable-al                  tivec' 'build_alias=x86_64-linux-gnu' 'CFLAGS=-g -O2 -fstack-protector-strong -Wformat -Werror=format-security ' 'LDFLAGS=-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,--as-needed' 'CPPFLAGS=-W                  date-time -D_FORTIFY_SOURCE=2' 'CXXFLAGS=-g -O2 -fstack-protector-strong -Wformat -Werror=format-security ' 'OBJCFLAGS=-g -O2 -fstack-protector-strong -Wformat -Werror=format-security'
  
  [0000000000a9c178] core libvlc debug: searching plug-in modules
  
  [0000000000a9c178] core libvlc debug: loading plugins cache file /usr/lib/vlc/plugins/plugins.dat
  
  [0000000000a9c178] core libvlc debug: recursively browsing `/usr/lib/vlc/plugins'
  
  [0000000000a9c178] core libvlc debug: saving plugins cache /usr/lib/vlc/plugins/plugins.dat
  
  [0000000000a9c178] core libvlc debug: plug-ins loaded: 459 modules
  
  [0000000000a9c178] core libvlc debug: opening config file (/root/.config/vlc/vlcrc)

  I am trying to connect to it from an external browser as

  http://********:8080/

  I have checked in the Linux Server that there is not any server running. Any idea on why it does not work ?