Recherche avancée

Sur d’autres sites (10009)

• Seeking to every 30th frame in .mp4 file with ffmpeg

  6 mars 2020, par MTRNS

  I’m trying to seek to every 30th frame and read motion vectors from packets.

  // Move the video 1 frame
  
  av_read_frame(fmt_ctx, &pkt);
  
  if (pkt.stream_index == video_stream_idx)
  
      ret = decode_packet(&pkt);
  
  av_packet_unref(&pkt);
  
  
  
  // Read motion vectors until end of video
  
  while (av_read_frame(fmt_ctx, &pkt) >= 0) {
  
      if (pkt.stream_index == video_stream_idx)
  
          ret = decode_packet(&pkt);
  
  
  
      // Find packet frame
  
      int pcktPts = av_rescale_q(pkt.pts,
  
              fmt_ctx->streams[video_stream_idx]->time_base,
  
              fmt_ctx->streams[video_stream_idx]->codec->time_base);
  
      pcktPts = (pcktPts/video_dec_ctx->ticks_per_frame);
  
  
  
      // Find target position from packet (30 frames ahead)
  
      int target = (pcktPts+30) *
  
                   (fmt_ctx->streams[video_stream_idx]->time_base.den /
  
                    fmt_ctx->streams[video_stream_idx]->time_base.num) /
  
                   (fmt_ctx->streams[video_stream_idx]->codec->time_base.den /
  
                    fmt_ctx->streams[video_stream_idx]->codec->time_base.num )*
  
                   video_dec_ctx->ticks_per_frame;
  
  
  
      // Seek to 30 frames ahead
  
      avformat_seek_file(fmt_ctx, video_stream_idx, 0, target, target, AVSEEK_FLAG_ANY);
  
  
  
      av_packet_unref(&pkt);
  
      if (ret < 0)
  
          break;
  
  }

  Running through this, it runs well at first, but starts to hang at around the 6th or 7th frame analyzed. Here’s my output of what frame I’m decoding :

  1
  
  31
  
  61
  
  92
  
  123
  
  155
  
  187
  
  220
  
  237
  
  237
  
  237
  
  ...

  where it just stalls on frame 237. So a few things are wrong : the program is hanging, and I’m not increasing by 30 frames every time, but by 30 + some random number. I must be using some function wrong. This is specifically for .mp4 files.

  Essentially, I want to read motion vectors from every frame following an I-frame. My entire program is here :

  #include <libavutil></libavutil>motion_vector.h>
  
  #include <libavformat></libavformat>avformat.h>
  
  
  
  static AVFormatContext *fmt_ctx = NULL;
  
  static AVCodecContext *video_dec_ctx = NULL;
  
  static AVStream *video_stream = NULL;
  
  static const char *src_filename = NULL;
  
  
  
  static int video_stream_idx = -1;
  
  static AVFrame *frame = NULL;
  
  static int video_frame_count = 0;
  
  
  
  static int decode_packet(const AVPacket *pkt)
  
  {
  
      char szFileName[255] = {0};
  
      FILE *file = NULL;
  
      int ret = avcodec_send_packet(video_dec_ctx, pkt);
  
      if (ret &lt; 0) {
  
          fprintf(stderr, "Error while sending a packet to the decoder: %s\n", av_err2str(ret));
  
          return ret;
  
      }
  
  
  
      while (ret >= 0)  {
  
          ret = avcodec_receive_frame(video_dec_ctx, frame);
  
          if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
  
              break;
  
          } else if (ret &lt; 0) {
  
              fprintf(stderr, "Error while receiving a frame from the decoder: %s\n", av_err2str(ret));
  
              return ret;
  
          }
  
  
  
          if (ret >= 0) {
  
              int i;
  
              AVFrameSideData *sd;
  
              video_frame_count++;
  
              sprintf(szFileName, "./mv/frame%d.txt", video_frame_count);
  
              file = fopen(szFileName, "w");
  
              sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS);
  
              if (sd) {
  
                  const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
  
                  for (i = 0; i &lt; sd->size / sizeof(*mvs); i++) {
  
                      const AVMotionVector *mv = &amp;mvs[i];
  
                      fprintf(file, "%4d %4d %4d %4d\n",
  
                          abs(mv->motion_x),
  
                          abs(mv->motion_y),
  
                          abs(mv->src_x - mv->dst_x),
  
                          abs(mv->src_y - mv->dst_y));
  
                  }
  
              }
  
              fclose(file);
  
              av_frame_unref(frame);
  
          }
  
      }
  
  
  
      return 0;
  
  }
  
  
  
  static int open_codec_context(AVFormatContext *fmt_ctx, enum AVMediaType type)
  
  {
  
      int ret;
  
      AVStream *st;
  
      AVCodecContext *dec_ctx = NULL;
  
      AVCodec *dec = NULL;
  
      AVDictionary *opts = NULL;
  
  
  
      ret = av_find_best_stream(fmt_ctx, type, -1, -1, &amp;dec, 0);
  
      if (ret &lt; 0) {
  
          fprintf(stderr, "Could not find %s stream in input file '%s'\n",
  
                  av_get_media_type_string(type), src_filename);
  
          return ret;
  
      } else {
  
          int stream_idx = ret;
  
          st = fmt_ctx->streams[stream_idx];
  
  
  
          dec_ctx = avcodec_alloc_context3(dec);
  
          if (!dec_ctx) {
  
              fprintf(stderr, "Failed to allocate codec\n");
  
              return AVERROR(EINVAL);
  
          }
  
  
  
          ret = avcodec_parameters_to_context(dec_ctx, st->codecpar);
  
          if (ret &lt; 0) {
  
              fprintf(stderr, "Failed to copy codec parameters to codec context\n");
  
              return ret;
  
          }
  
  
  
          // Initialize video decoder
  
          av_dict_set(&amp;opts, "flags2", "+export_mvs", 0);
  
          if ((ret = avcodec_open2(dec_ctx, dec, &amp;opts)) &lt; 0) {
  
              fprintf(stderr, "Failed to open %s codec\n",
  
                      av_get_media_type_string(type));
  
              return ret;
  
          }
  
  
  
          video_stream_idx = stream_idx;
  
          video_stream = fmt_ctx->streams[video_stream_idx];
  
          video_dec_ctx = dec_ctx;
  
      }
  
  
  
      return 0;
  
  }
  
  
  
  int main(int argc, char **argv)
  
  {
  
      int ret = 0;
  
      AVPacket pkt = { 0 };
  
  
  
      if (argc != 2) {
  
          fprintf(stderr, "Usage: %s <video>\n", argv[0]);
  
          exit(1);
  
      }
  
      src_filename = argv[1];
  
  
  
      if (avformat_open_input(&amp;fmt_ctx, src_filename, NULL, NULL) &lt; 0) {
  
          fprintf(stderr, "Could not open source file %s\n", src_filename);
  
          exit(1);
  
      }
  
  
  
      if (avformat_find_stream_info(fmt_ctx, NULL) &lt; 0) {
  
          fprintf(stderr, "Could not find stream information\n");
  
          exit(1);
  
      }
  
  
  
      open_codec_context(fmt_ctx, AVMEDIA_TYPE_VIDEO);
  
      av_dump_format(fmt_ctx, 0, src_filename, 0);
  
  
  
      if (!video_stream) {
  
          fprintf(stderr, "Could not find video stream in the input, aborting\n");
  
          ret = 1;
  
          goto end;
  
      }
  
  
  
      frame = av_frame_alloc();
  
      if (!frame) {
  
          fprintf(stderr, "Could not allocate frame\n");
  
          ret = AVERROR(ENOMEM);
  
          goto end;
  
      }
  
  
  
      // Move the video 1 frame
  
      av_read_frame(fmt_ctx, &amp;pkt);
  
      if (pkt.stream_index == video_stream_idx)
  
          ret = decode_packet(&amp;pkt);
  
      av_packet_unref(&amp;pkt);
  
  
  
      // Read motion vectors until end of video
  
      while (av_read_frame(fmt_ctx, &amp;pkt) >= 0) {
  
          if (pkt.stream_index == video_stream_idx)
  
              ret = decode_packet(&amp;pkt);
  
  
  
          int pcktPts = av_rescale_q(pkt.pts,
  
                  fmt_ctx->streams[video_stream_idx]->time_base,
  
                  fmt_ctx->streams[video_stream_idx]->codec->time_base);
  
          pcktPts = (pcktPts/video_dec_ctx->ticks_per_frame);
  
  
  
          int target = (pcktPts+30) *
  
                       (fmt_ctx->streams[video_stream_idx]->time_base.den /
  
                        fmt_ctx->streams[video_stream_idx]->time_base.num) /
  
                       (fmt_ctx->streams[video_stream_idx]->codec->time_base.den /
  
                        fmt_ctx->streams[video_stream_idx]->codec->time_base.num )*
  
                       video_dec_ctx->ticks_per_frame;
  
  
  
          avformat_seek_file(fmt_ctx, video_stream_idx, 0, target, target, AVSEEK_FLAG_ANY);
  
  
  
          av_packet_unref(&amp;pkt);
  
          if (ret &lt; 0)
  
              break;
  
      }
  
  
  
      // Flush cached frames
  
      decode_packet(NULL);
  
      fprintf(stderr, "Finished decoding.\n\n");
  
  
  
  end:
  
      avcodec_free_context(&amp;video_dec_ctx);
  
      avformat_close_input(&amp;fmt_ctx);
  
      av_frame_free(&amp;frame);
  
      return ret &lt; 0;
  
  }
  
  </video>

• Twilio Real-Time Media Streaming to WebSocket Receives Only Noise Instead of Speech

  21 février, par dannym25

  I'm setting up a Twilio Voice call with real-time media streaming to a WebSocket server for speech-to-text processing using Google Cloud Speech-to-Text. The connection is established successfully, and I receive a continuous stream of audio data from Twilio. However, when I play back the received audio, all I hear is a rapid clicking/jackhammering noise instead of the actual speech spoken during the call.

  


  Setup :

  


  

  • Twilio sends inbound audio to my WebSocket server.

  


  • WebSocket receives and saves the raw mulaw-encoded audio data from Twilio.

  


  • The audio is processed via Google Speech-to-Text for transcription.

  


  • When I attempt to play back the audio, it sounds like machine-gun-like noise instead of spoken words.

  


  


  1. Confirmed WebSocket Receives Data

  


  • The WebSocket successfully logs incoming audio chunks from Twilio :

  


  🔊 Received 379 bytes of audio from Twilio
🔊 Received 379 bytes of audio from Twilio


  


  • This suggests Twilio is sending audio data, but it's not being interpreted correctly.

  


  2. Saving and Playing Raw Audio

  


  • I save the incoming raw mulaw (8000Hz) audio from Twilio to a file :

  


  fs.appendFileSync('twilio-audio.raw', message);


  


  • Then, I convert it to a .wav file using FFmpeg :

  


  ffmpeg -f mulaw -ar 8000 -ac 1 -i twilio-audio.raw twilio-audio.wav


  


  • Problem : When I play the audio using ffplay, it contains no speech, only rapid clicking sounds.

  


  3. Ensured Correct Audio Encoding

  


  • Twilio sends mulaw 8000Hz mono format.
• Verified that my ffmpeg conversion is using the same settings.
• Tried different conversion methods :

  


  ffmpeg -f mulaw -ar 8000 -ac 1 -i twilio-audio.raw -c:a pcm_s16le twilio-audio-fixed.wav


  


  → Same issue.

  


  4. Checked Google Speech-to-Text Input Format

  


  • Google STT requires proper encoding configuration :

  


  const request = {
    config: {
        encoding: 'MULAW',
        sampleRateHertz: 8000,
        languageCode: 'en-US',
    },
    interimResults: false,
};


  


  • No errors from Google STT, but it never detects speech, likely because the input audio is just noise.

  


  5. Confirmed That Raw Audio is Not a WAV File

  


  • Since Twilio sends raw audio, I checked whether I needed to strip the header before processing.
• Tried manually extracting raw bytes, but the issue persists.

  


  Current Theory :

  


  

  • The WebSocket server might be handling Twilio’s raw audio incorrectly before saving it.

  


  • There might be an additional header in the Twilio stream that needs to be removed before playback.

  


  • Twilio’s <stream></stream> tag expects a WebSocket connection starting with wss:// instead of https://, and switching to wss:// partially fixed some previous connection issues.

  


  


  Code Snippets :

  


  Twilio Setup in TwiML Response

  


  app.post('/voice-response', (req, res) => {
    console.log("📞 Incoming call from Twilio");

    const twiml = new twilio.twiml.VoiceResponse();
    twiml.say("Hello! Welcome to the service. How can I help you?");
    
    // Prevent Twilio from hanging up too early
    twiml.pause({ length: 5 });

    twiml.connect().stream({
        url: `wss://your-ngrok-url/ws`,
        track: "inbound_track"
    });

    console.log("🛠️ Twilio Stream URL:", `wss://your-ngrok-url/ws`);
    
    res.type('text/xml').send(twiml.toString());
});


  


  WebSocket Server Handling Twilio Audio Stream

  


  wss.on('connection', (ws) => {
    console.log("🔗 WebSocket Connected! Waiting for audio input...");

    ws.on('message', (message) => {
        console.log(`🔊 Received ${message.length} bytes of audio from Twilio`);

        // Save raw audio data for debugging
        fs.appendFileSync('twilio-audio.raw', message);

        // Check if audio is non-empty but contains only noise
        if (message.length < 100) {
            console.warn("⚠️ Warning: Audio data from Twilio is very small. Might be silent.");
        }
    });

    ws.on('close', () => {
        console.log("❌ WebSocket Disconnected!");
        
        // Convert Twilio audio for debugging
        exec(`ffmpeg -f mulaw -ar 8000 -ac 1 -i twilio-audio.raw twilio-audio.wav`, (err) => {
            if (err) console.error("❌ FFmpeg Conversion Error:", err);
            else console.log("✅ Twilio Audio Saved as `twilio-audio.wav`");
        });
    });

    ws.on('error', (error) => console.error("⚠️ WebSocket Error:", error));
});


  


  Questions :

  


  

  • Why is the audio from Twilio being received as a clicking noise instead of actual speech ?

  


  • Do I need to strip any additional metadata from the raw bytes before saving ?

  


  • Is there a known issue with Twilio’s mulaw format when streaming audio over WebSockets ?

  


  • How can I confirm that Google STT is receiving properly formatted audio ?

  


  


  Additional Context :

  


  

  • Twilio <stream></stream> is connected and receiving data (confirmed by logs).

  


  • WebSocket successfully receives and saves audio, but it only plays noise.

  


  • Tried multiple ffmpeg conversions, Google STT configurations, and raw data inspection.

  


  • Still no recognizable speech in the audio output.

  


  


  Any help is greatly appreciated ! 🙏

  


• Executing an exe and reading the OutputStream

  30 septembre 2014, par vlatkozelka

  I have a program called FFprobe which probes media (files/live streams ... ) and outputs result in different formats , for example :

  ffprobe.exe -i test.ts -print_format xml -show_programs

  gives this output :

      <?xml version="1.0" encoding="UTF-8"?>
  
  <ffprobe>
  
      <programs>
  
          <program>
  
              <tag key="service_name" value="Arabica TV"></tag>
  
              <tag key="service_provider" value="Nilesat"></tag>
  
              <streams>
  
                  <stream index="10" profile="Main" width="720" height="576" level="8" timecode="08:28:54:09">
  
                      <disposition default="0" dub="0" original="0" comment="0" lyrics="0" karaoke="0" forced="0"></disposition>
  
                  </stream>
  
                  <stream index="4" channels="2">
  
                      <disposition default="0" dub="0" original="0" comment="0" lyrics="0" karaoke="0" forced="0"></disposition>
  
                  </stream>
  
              </streams>
  
          </program> 
  
  ... more programs 
  
  </programs></ffprobe>

  to retrieve this info in java i used ProcessBuilder and a scanner , and then id write to a file once the result is ok ... but it wasnt :

   Process proc = new ProcessBuilder("ffprobe.exe","-i", ... same arguments );
  
   Scanner sc = new Scanner (proc.getInputStream()) // im 100% sure its not errorStream
  
   while(sc.hasNext()){
  
     System.out.println(sc.nextLine());
  
   }

  the app just hangs with no output , i know its hanging cuz the process is still running and scanner has next , but , i don’t know why it would do that ?If i execute the same in cmd i would get good result and ofc i can write to file with ">"

  Ive tried it w/o the -print_format option , which gives the info in a plain text on the errorstream(i know its error cuz i was able to write with 2> not >> ) , and i was able to read the error stream in java , but its not meant for parsing cuz very very un-organized .

     Input #0, mpegts, from 'PBR_REC_20140426094852_486.ts':
  
    Duration: 00:13:34.30, start: 7791.344722, bitrate: 42154 kb/s
  
    Program 1201 
  
      Metadata:
  
        service_name    : Arabica TV
  
        service_provider: Nilesat
  
      Stream #0:19[0x7db]: Video: mpeg2video (Main) ([2][0][0][0] / 0x0002), yuv420p(tv), 720x576 [SAR 16:15 DAR 4:3], max. 2348 kb/s, 25 fps, 25 tbr, 90k tbn, 50 tbc
  
      Stream #0:3[0xbcf]: Audio: mp2, 48000 Hz, stereo, s16p, 384 kb/s
  
    Program 1202 
  
      Metadata:
  
        service_name    : NBN
  
        service_provider: NILESAT
  
      Stream #0:10[0x7d1]: Video: mpeg2video (Main) ([2][0][0][0] / 0x0002), yuv420p(tv), 720x576 [SAR 16:15 DAR 4:3], max. 2600 kb/s, 25 fps, 25 tbr, 90k tbn, 50 tbc
  
      Stream #0:11[0xbba](eng): Audio: mp2 ([3][0][0][0] / 0x0003), 48000 Hz, stereo, s16p, 125 kb/s
  
    Program 1203 
  
      Metadata:
  
        service_name    : Heya TV
  
        service_provider: NILESAT
  
      Stream #0:5[0x7d2]: Video: mpeg2video (Main) ([2][0][0][0] / 0x0002), yuv420p(tv), 720x576 [SAR 16:15 DAR 4:3], max. 2600 kb/s, 25 fps, 25 tbr, 90k tbn, 50 tbc
  
      Stream #0:6[0xbbc](eng): Audio: mp2 ([3][0][0][0] / 0x0003), 48000 Hz, mono, s16p, 125 kb/s
  
    Program 1204  ... more programs

  now it might seem organized and "parse-able" and i actually made a parser that worked to some point , but sometimes it doesnt stick to this structure and ruins the whole parsing, its why i need a xml/json ...

  The -print_format im sure outputs to outputstream.

  Im not asking for help on how to use FFprobe as thats not this place’s purpose , Im asking why am i not being able to get the output stream from java while it is definitely outputting if i execute in windows .

  I also tried apache commons-exec as i know processbuilder can be a pain ,it did execute perfectly and showed in System.in (black for intput and red for error), but getting the stream with apache is something i couldn’t understand , i tried this example

  The xml parser i already taken care of , simply put i just need to execute that first command from java and read the output , but for some reason its not working .