Recherche avancée

Sur d’autres sites (8923)

• Google speech to text api is partially converting .flac file into text

  12 juillet 2018, par Gemini Jain

  Steps followed :

  1. Converted .mp3 to .flac using ffmpeg.
  2. Ran this command
     gs://xxx/xxx.flac --language-code=en-US --async --encoding=FLAC --sample-rate=44100.
  3. After processing it, is showing result in JSON format, but its not relevant to audio file.

  JSON result looks like :

  {
  
    "@type": "xxx",
  
    "results": [
  
      {
  
        "alternatives": [
  
          {
  
            "confidence": 0.71890223,
  
            "transcript": "I reports everybody."
  
          }
  
        ]
  
      },
  
      {
  
        "alternatives": [
  
          {
  
            "confidence": 0.5876879,
  
            "transcript": "dear, it's your"
  
          }
  
        ]
  
      }
  
  ......
  
  }]}

  Can please someone help me in figuring out why it is not converting the audio files correctly ? Am I missing any tags ?.

• Play video using mse (media source extension) in google chrome

  23 août 2019, par liyuqihxc

  I’m working on a project that convert rtsp stream (ffmpeg) and play it on the web page (signalr + mse).

  So far it works pretty much as I expected on the latest version of edge and firefox, but not chrome.

  here’s the code

  public class WebmMediaStreamContext
  
  {
  
      private Process _ffProcess;
  
      private readonly string _cmd;
  
      private byte[] _initSegment;
  
      private Task _readMediaStreamTask;
  
      private CancellationTokenSource _cancellationTokenSource;
  
  
  
      private const string _CmdTemplate = "-i {0} -c:v libvpx -tile-columns 4 -frame-parallel 1 -keyint_min 90 -g 90 -f webm -dash 1 pipe:";
  
  
  
      public static readonly byte[] ClusterStart = { 0x1F, 0x43, 0xB6, 0x75, 0x01, 0x00, 0x00, 0x00 };
  
  
  
      public event EventHandler<clusterreadyeventargs> ClusterReadyEvent;
  
  
  
      public WebmMediaStreamContext(string rtspFeed)
  
      {
  
          _cmd = string.Format(_CmdTemplate, rtspFeed);
  
      }
  
  
  
      public async Task StartConverting()
  
      {
  
          if (_ffProcess != null)
  
              throw new InvalidOperationException();
  
  
  
          _ffProcess = new Process();
  
          _ffProcess.StartInfo = new ProcessStartInfo
  
          {
  
              FileName = "ffmpeg/ffmpeg.exe",
  
              Arguments = _cmd,
  
              UseShellExecute = false,
  
              CreateNoWindow = true,
  
              RedirectStandardOutput = true
  
          };
  
          _ffProcess.Start();
  
  
  
          _initSegment = await ParseInitSegmentAndStartReadMediaStream();
  
      }
  
  
  
      public byte[] GetInitSegment()
  
      {
  
          return _initSegment;
  
      }
  
  
  
      // Find the first cluster， and everything before it is the InitSegment
  
      private async Task ParseInitSegmentAndStartReadMediaStream()
  
      {
  
          Memory<byte> buffer = new byte[10 * 1024];
  
          int length = 0;
  
          while (length != buffer.Length)
  
          {
  
              length += await _ffProcess.StandardOutput.BaseStream.ReadAsync(buffer.Slice(length));
  
              int cluster = buffer.Span.IndexOf(ClusterStart);
  
              if (cluster >= 0)
  
              {
  
                  _cancellationTokenSource = new CancellationTokenSource();
  
                  _readMediaStreamTask = new Task(() => ReadMediaStreamProc(buffer.Slice(cluster, length - cluster).ToArray(), _cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning);
  
                  _readMediaStreamTask.Start();
  
                  return buffer.Slice(0, cluster).ToArray();
  
              }
  
          }
  
  
  
          throw new InvalidOperationException();
  
      }
  
  
  
      private void ReadMoreBytes(Span<byte> buffer)
  
      {
  
          int size = buffer.Length;
  
          while (size > 0)
  
          {
  
              int len = _ffProcess.StandardOutput.BaseStream.Read(buffer.Slice(buffer.Length - size));
  
              size -= len;
  
          }
  
      }
  
  
  
      // Parse every single cluster and fire ClusterReadyEvent
  
      private void ReadMediaStreamProc(byte[] bytesRead, CancellationToken cancel)
  
      {
  
          Span<byte> buffer = new byte[5 * 1024 * 1024];
  
          bytesRead.CopyTo(buffer);
  
          int bufferEmptyIndex = bytesRead.Length;
  
  
  
          do
  
          {
  
              if (bufferEmptyIndex &lt; ClusterStart.Length + 4)
  
              {
  
                  ReadMoreBytes(buffer.Slice(bufferEmptyIndex, 1024));
  
                  bufferEmptyIndex += 1024;
  
              }
  
  
  
              int clusterDataSize = BitConverter.ToInt32(
  
                  buffer.Slice(ClusterStart.Length, 4)
  
                  .ToArray()
  
                  .Reverse()
  
                  .ToArray()
  
              );
  
              int clusterSize = ClusterStart.Length + 4 + clusterDataSize;
  
              if (clusterSize > buffer.Length)
  
              {
  
                  byte[] newBuffer = new byte[clusterSize];
  
                  buffer.Slice(0, bufferEmptyIndex).CopyTo(newBuffer);
  
                  buffer = newBuffer;
  
              }
  
  
  
              if (bufferEmptyIndex &lt; clusterSize)
  
              {
  
                  ReadMoreBytes(buffer.Slice(bufferEmptyIndex, clusterSize - bufferEmptyIndex));
  
                  bufferEmptyIndex = clusterSize;
  
              }
  
  
  
              ClusterReadyEvent?.Invoke(this, new ClusterReadyEventArgs(buffer.Slice(0, bufferEmptyIndex).ToArray()));
  
  
  
              bufferEmptyIndex = 0;
  
          } while (!cancel.IsCancellationRequested);
  
      }
  
  }
  
  </byte></byte></byte></clusterreadyeventargs>

  I use ffmpeg to convert the rtsp stream to vp8 WEBM byte stream and parse it to "Init Segment" (ebml head、info、tracks...) and "Media Segment" (cluster)， then send it to browser via signalR

  $(function () {
  
  
  
      var mediaSource = new MediaSource();
  
      var mimeCodec = 'video/webm; codecs="vp8"';
  
  
  
      var video = document.getElementById('video');
  
  
  
      mediaSource.addEventListener('sourceopen', callback, false);
  
      function callback(e) {
  
          var sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
  
          var queue = [];
  
  
  
          sourceBuffer.addEventListener('updateend', function () {
  
              if (queue.length === 0) {
  
                  return;
  
              }
  
  
  
              var base64 = queue[0];
  
              if (base64.length === 0) {
  
                  mediaSource.endOfStream();
  
                  queue.shift();
  
                  return;
  
              } else {
  
                  var buffer = new Uint8Array(atob(base64).split("").map(function (c) {
  
                      return c.charCodeAt(0);
  
                  }));
  
                  sourceBuffer.appendBuffer(buffer);
  
                  queue.shift();
  
              }
  
          }, false);
  
  
  
          var connection = new signalR.HubConnectionBuilder()
  
              .withUrl("/signalr-video")
  
              .configureLogging(signalR.LogLevel.Information)
  
              .build();
  
          connection.start().then(function () {
  
              connection.stream("InitVideoReceive")
  
                  .subscribe({
  
                      next: function(item) {
  
                          if (queue.length === 0 && !!!sourceBuffer.updating) {
  
                              var buffer = new Uint8Array(atob(item).split("").map(function (c) {
  
                                  return c.charCodeAt(0);
  
                              }));
  
                              sourceBuffer.appendBuffer(buffer);
  
                              console.log(blockindex++ + " : " + buffer.byteLength);
  
                          } else {
  
                              queue.push(item);
  
                          }
  
                      },
  
                      complete: function () {
  
                          queue.push('');
  
                      },
  
                      error: function (err) {
  
                          console.error(err);
  
                      }
  
                  });
  
          });
  
      }
  
      video.src = window.URL.createObjectURL(mediaSource);
  
  })

  chrome just play the video for 3 5 seconds and then stop for buffering, even though there are plenty of cluster transfered and inserted into SourceBuffer.

  here’s the information in chrome ://media-internals/

  Player Properties :

  render_id: 217
  
  player_id: 1
  
  origin_url: http://localhost:52531/
  
  frame_url: http://localhost:52531/
  
  frame_title: Home Page
  
  url: blob:http://localhost:52531/dcb25d89-9830-40a5-ba88-33c13b5c03eb
  
  info: Selected FFmpegVideoDecoder for video decoding, config: codec: vp8 format: 1 profile: vp8 coded size: [1280,720] visible rect: [0,0,1280,720] natural size: [1280,720] has extra data? false encryption scheme: Unencrypted rotation: 0°
  
  pipeline_state: kSuspended
  
  found_video_stream: true
  
  video_codec_name: vp8
  
  video_dds: false
  
  video_decoder: FFmpegVideoDecoder
  
  duration: unknown
  
  height: 720
  
  width: 1280
  
  video_buffering_state: BUFFERING_HAVE_NOTHING
  
  for_suspended_start: false
  
  pipeline_buffering_state: BUFFERING_HAVE_NOTHING
  
  event: PAUSE

  Log

  Timestamp       Property            Value
  
  00:00:00 00     origin_url          http://localhost:52531/
  
  00:00:00 00     frame_url           http://localhost:52531/
  
  00:00:00 00     frame_title         Home Page
  
  00:00:00 00     url                 blob:http://localhost:52531/dcb25d89-9830-40a5-ba88-33c13b5c03eb
  
  00:00:00 00     info                ChunkDemuxer: buffering by DTS
  
  00:00:00 35     pipeline_state      kStarting
  
  00:00:15 213    found_video_stream  true
  
  00:00:15 213    video_codec_name    vp8
  
  00:00:15 216    video_dds           false
  
  00:00:15 216    video_decoder       FFmpegVideoDecoder
  
  00:00:15 216    info                Selected FFmpegVideoDecoder for video decoding, config: codec: vp8 format: 1 profile: vp8 coded size: [1280,720] visible rect: [0,0,1280,720] natural size: [1280,720] has extra data? false encryption scheme: Unencrypted rotation: 0°
  
  00:00:15 216    pipeline_state      kPlaying
  
  00:00:15 213    duration            unknown
  
  00:00:16 661    height              720
  
  00:00:16 661    width               1280
  
  00:00:16 665    video_buffering_state       BUFFERING_HAVE_ENOUGH
  
  00:00:16 665    for_suspended_start         false
  
  00:00:16 665    pipeline_buffering_state    BUFFERING_HAVE_ENOUGH
  
  00:00:16 667    pipeline_state      kSuspending
  
  00:00:16 670    pipeline_state      kSuspended
  
  00:00:52 759    info                Effective playback rate changed from 0 to 1
  
  00:00:52 759    event               PLAY
  
  00:00:52 759    pipeline_state      kResuming
  
  00:00:52 760    video_dds           false
  
  00:00:52 760    video_decoder       FFmpegVideoDecoder
  
  00:00:52 760    info                Selected FFmpegVideoDecoder for video decoding, config: codec: vp8 format: 1 profile: vp8 coded size: [1280,720] visible rect: [0,0,1280,720] natural size: [1280,720] has extra data? false encryption scheme: Unencrypted rotation: 0°
  
  00:00:52 760    pipeline_state      kPlaying
  
  00:00:52 793    height              720
  
  00:00:52 793    width               1280
  
  00:00:52 798    video_buffering_state       BUFFERING_HAVE_ENOUGH
  
  00:00:52 798    for_suspended_start         false
  
  00:00:52 798    pipeline_buffering_state    BUFFERING_HAVE_ENOUGH
  
  00:00:56 278    video_buffering_state       BUFFERING_HAVE_NOTHING
  
  00:00:56 295    for_suspended_start         false
  
  00:00:56 295    pipeline_buffering_state    BUFFERING_HAVE_NOTHING
  
  00:01:20 717    event               PAUSE
  
  00:01:33 538    event               PLAY
  
  00:01:35 94     event               PAUSE
  
  00:01:55 561    pipeline_state      kSuspending
  
  00:01:55 563    pipeline_state      kSuspended

  Can someone tell me what’s wrong with my code, or dose chrome require some magic configuration to work ?

  Thanks 

  Please excuse my english :)

• How to Stream Audio from Google Cloud Storage in Chunks and Convert Each Chunk to WAV for Whisper Transcription

  14 novembre 2024, par Douglas Landvik

  I'm working on a project where I need to transcribe audio stored in a Google Cloud Storage bucket using OpenAI's Whisper model. The audio is stored in WebM format with Opus encoding, and due to the file size, I'm streaming the audio in 30-second chunks.

  


  To convert each chunk to WAV (16 kHz, mono, 16-bit PCM) compatible with Whisper, I'm using FFmpeg. The first chunk converts successfully, but subsequent chunks fail to convert. I suspect this is because each chunk lacks the WebM container's header, which FFmpeg needs to interpret the Opus codec correctly.

  


  Here’s a simplified version of my approach :

  


  Download Chunk : I download each chunk from GCS as bytes.
Convert with FFmpeg : I pass the bytes to FFmpeg to convert each chunk from WebM/Opus to WAV.

  


  async def handle_transcription_and_notify(
    consultation_service: ConsultationService,
    consultation_processor: ConsultationProcessor,
    consultation: Consultation,
    language: str,
    notes: str,
    clinic_id: str,
    vet_email: str,
    trace_id: str,
    blob_path: str,
    max_retries: int = 3,
    retry_delay: int = 5,
    max_concurrent_tasks: int = 3
):
    """
    Handles the transcription process by streaming the file from GCS, converting to a compatible format, 
    and notifying the client via WebSocket.
    """
    chunk_duration_sec = 30  # 30 seconds per chunk
    logger.info(f"Starting transcription process for consultation {consultation.consultation_id}",
                extra={'trace_id': trace_id})

    # Initialize GCS client
    service_account_key = os.environ.get('SERVICE_ACCOUNT_KEY_BACKEND')
    if not service_account_key:
        logger.error("Service account key not found in environment variables", extra={'trace_id': trace_id})
        await send_discord_alert(
            f"Service account key not found for consultation {consultation.consultation_id}.\nTrace ID: {trace_id}"
        )
        return

    try:
        service_account_info = json.loads(service_account_key)
        credentials = service_account.Credentials.from_service_account_info(service_account_info)
    except Exception as e:
        logger.error(f"Error loading service account credentials: {str(e)}", extra={'trace_id': trace_id})
        await send_discord_alert(
            f"Error loading service account credentials for consultation {consultation.consultation_id}.\nError: {str(e)}\nTrace ID: {trace_id}"
        )
        return

    # Initialize GCS client
    service_account_key = os.environ.get('SERVICE_ACCOUNT_KEY_BACKEND')
    if not service_account_key:
        logger.error("Service account key not found in environment variables", extra={'trace_id': trace_id})
        await send_discord_alert(
            f"Service account key not found for consultation {consultation.consultation_id}.\nTrace ID: {trace_id}"
        )
        return

    try:
        service_account_info = json.loads(service_account_key)
        credentials = service_account.Credentials.from_service_account_info(service_account_info)
    except Exception as e:
        logger.error(f"Error loading service account credentials: {str(e)}", extra={'trace_id': trace_id})
        await send_discord_alert(
            f"Error loading service account credentials for consultation {consultation.consultation_id}.\nError: {str(e)}\nTrace ID: {trace_id}"
        )
        return

    storage_client = storage.Client(credentials=credentials)
    bucket_name = 'vetz_consultations'
    blob = storage_client.bucket(bucket_name).get_blob(blob_path)
    bytes_per_second = 16000 * 2  # 32,000 bytes per second
    chunk_size_bytes = 30 * bytes_per_second
    size = blob.size

    async def stream_blob_in_chunks(blob, chunk_size):
        loop = asyncio.get_running_loop()
        start = 0
        size = blob.size
        while start < size:
            end = min(start + chunk_size - 1, size - 1)
            try:
                logger.info(f"Requesting chunk from {start} to {end}", extra={'trace_id': trace_id})
                chunk = await loop.run_in_executor(
                    None, lambda: blob.download_as_bytes(start=start, end=end)
                )
                if not chunk:
                    break
                logger.info(f"Yielding chunk from {start} to {end}, size: {len(chunk)} bytes",
                            extra={'trace_id': trace_id})
                yield chunk
                start += chunk_size
            except Exception as e:
                logger.error(f"Error downloading chunk from {start} to {end}: {str(e)}", exc_info=True,
                             extra={'trace_id': trace_id})
                raise e

    async def convert_to_wav(chunk_bytes, chunk_idx):
        """
        Convert audio chunk to WAV format compatible with Whisper, ensuring it's 16 kHz, mono, and 16-bit PCM.
        """
        try:
            logger.debug(f"Processing chunk {chunk_idx}: size = {len(chunk_bytes)} bytes")

            detected_format = await detect_audio_format(chunk_bytes)
            logger.info(f"Detected audio format for chunk {chunk_idx}: {detected_format}")
            input_io = io.BytesIO(chunk_bytes)
            output_io = io.BytesIO()

            # ffmpeg command to convert webm/opus to WAV with 16 kHz, mono, and 16-bit PCM

            # ffmpeg command with debug information
            ffmpeg_command = [
                "ffmpeg",
                "-loglevel", "debug",
                "-f", "s16le",            # Treat input as raw PCM data
                "-ar", "48000",           # Set input sample rate
                "-ac", "1",               # Set input to mono
                "-i", "pipe:0",
                "-ar", "16000",           # Set output sample rate to 16 kHz
                "-ac", "1",               # Ensure mono output
                "-sample_fmt", "s16",     # Set output format to 16-bit PCM
                "-f", "wav",              # Output as WAV format
                "pipe:1"
            ]

            process = subprocess.Popen(
                ffmpeg_command,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )

            stdout, stderr = process.communicate(input=input_io.read())

            if process.returncode == 0:
                logger.info(f"FFmpeg conversion completed successfully for chunk {chunk_idx}")
                output_io.write(stdout)
                output_io.seek(0)

                # Save the WAV file locally for listening
                output_dir = "converted_chunks"
                os.makedirs(output_dir, exist_ok=True)
                file_path = os.path.join(output_dir, f"chunk_{chunk_idx}.wav")

                with open(file_path, "wb") as f:
                    f.write(stdout)
                logger.info(f"Chunk {chunk_idx} saved to {file_path}")

                return output_io
            else:
                logger.error(f"FFmpeg failed for chunk {chunk_idx} with return code {process.returncode}")
                logger.error(f"Chunk {chunk_idx} - FFmpeg stderr: {stderr.decode()}")
                return None

        except Exception as e:
            logger.error(f"Unexpected error in FFmpeg conversion for chunk {chunk_idx}: {str(e)}")
            return None

    async def transcribe_chunk(idx, chunk_bytes):
        for attempt in range(1, max_retries + 1):
            try:
                logger.info(f"Transcribing chunk {idx + 1} (attempt {attempt}).", extra={'trace_id': trace_id})

                # Convert to WAV format
                wav_io = await convert_to_wav(chunk_bytes, idx)
                if not wav_io:
                    logger.error(f"Failed to convert chunk {idx + 1} to WAV format.")
                    return ""

                wav_io.name = "chunk.wav"
                chunk_transcription = await consultation_processor.transcribe_audio_whisper(wav_io)
                logger.info(f"Chunk {idx + 1} transcribed successfully.", extra={'trace_id': trace_id})
                return chunk_transcription
            except Exception as e:
                logger.error(f"Error transcribing chunk {idx + 1} (attempt {attempt}): {str(e)}", exc_info=True,
                             extra={'trace_id': trace_id})
                if attempt < max_retries:
                    await asyncio.sleep(retry_delay)
                else:
                    await send_discord_alert(
                        f"Max retries reached for chunk {idx + 1} in consultation {consultation.consultation_id}.\nError: {str(e)}\nTrace ID: {trace_id}"
                    )
                    return ""  # Return empty string for failed chunk

    await notification_manager.send_personal_message(
        f"Consultation {consultation.consultation_id} is being transcribed.", vet_email
    )

    try:
        idx = 0
        full_transcription = []
        async for chunk in stream_blob_in_chunks(blob, chunk_size_bytes):
            transcription = await transcribe_chunk(idx, chunk)
            if transcription:
                full_transcription.append(transcription)
            idx += 1

        combined_transcription = " ".join(full_transcription)
        consultation.full_transcript = (consultation.full_transcript or "") + " " + combined_transcription
        consultation_service.save_consultation(clinic_id, vet_email, consultation)
        logger.info(f"Transcription saved for consultation {consultation.consultation_id}.",
                    extra={'trace_id': trace_id})

    except Exception as e:
        logger.error(f"Error during transcription process: {str(e)}", exc_info=True, extra={'trace_id': trace_id})
        await send_discord_alert(
            f"Error during transcription process for consultation {consultation.consultation_id}.\nError: {str(e)}\nTrace ID: {trace_id}"
        )
        return

    await notification_manager.send_personal_message(
        f"Consultation {consultation.consultation_id} has been transcribed.", vet_email
    )

    try:
        template_service = TemplateService()
        medical_record_template = template_service.get_template_by_name(
            consultation.medical_record_template_id).sections

        sections = await consultation_processor.extract_structured_sections(
            transcription=consultation.full_transcript,
            notes=notes,
            language=language,
            template=medical_record_template,
        )
        consultation.sections = sections
        consultation_service.save_consultation(clinic_id, vet_email, consultation)
        logger.info(f"Sections processed for consultation {consultation.consultation_id}.",
                    extra={'trace_id': trace_id})
    except Exception as e:
        logger.error(f"Error processing sections for consultation {consultation.consultation_id}: {str(e)}",
                     exc_info=True, extra={'trace_id': trace_id})
        await send_discord_alert(
            f"Error processing sections for consultation {consultation.consultation_id}.\nError: {str(e)}\nTrace ID: {trace_id}"
        )
        raise e

    await notification_manager.send_personal_message(
        f"Consultation {consultation.consultation_id} is fully processed.", vet_email
    )
    logger.info(f"Successfully processed consultation {consultation.consultation_id}.",
                extra={'trace_id': trace_id})