Recherche avancée

Sur d’autres sites (11968)

• Ffmpeg AVAudioFifo memory leak

  10 août 2020, par Expressingx

  I'm encoding audio to AAC with encoder. Because it requires I'm using AVAudioFifo following https://ffmpeg.org/doxygen/4.0/transcode_aac_8c-example.html

  


  Everything works, but I can see my memory slowly growing up. If I comment out PushToFifo() no memory leaks. And not sure why. I've profiled the app with ANTS Profiler and I can see a lot of unmanaged memory allocated by avutil-x.dll.

  


  while (ffmpeg.av_read_frame(_inputContext.InputFormatContext, pkt) >= 0)
{
   // decode
   int ret = ffmpeg.avcodec_send_packet(decCtx, pkt);

        if (ret < 0)
        {
            ffmpeg.av_packet_unref(pkt);
            return;
        }

        while (ret >= 0)
        {
            ret = ffmpeg.avcodec_receive_frame(decCtx, frame);

            if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN) || ret == ffmpeg.AVERROR_EOF)
            {
                return;
            }
            
            // push to fifo
            PushToFifo();
            TryFlushSamples();
        }
}


  


  PushToFifo()

  


              byte* samples = null;

            if (ffmpeg.av_samples_alloc(&samples, null, _outputContext->channels, inputFrame->nb_samples, _outputContext->sample_fmt, 0) < 0)
            {
              ffmpeg.av_freep(&samples[0]);
              ffmpeg.av_free(samples);
              // throw
            }

            if (ffmpeg.swr_convert(_resamplerCtx, &samples, inputFrame->nb_samples, inputFrame->extended_data, inputFrame->nb_samples) < 0)
            {
              throw new Exception();
            }

            if (ffmpeg.av_audio_fifo_realloc(AudioFifo, ffmpeg.av_audio_fifo_size(AudioFifo) + inputFrame->nb_samples) < 0)
            {
              throw new Exception();
            }

            if (ffmpeg.av_audio_fifo_write(AudioFifo, (void**)&samples, inputFrame->nb_samples) < 0)
            {
               throw new Exception();
            }


  


  And in TryFlushSamples();

  


          while (ffmpeg.av_audio_fifo_size(_audioFifo.AudioFifo) >= _outputContext.AudioEncodeContext->frame_size)
        {
            int fifoSize = ffmpeg.av_audio_fifo_size(_audioFifo.AudioFifo);
            int frameSize = fifoSize > _outputContext.AudioEncodeContext->frame_size
                ? _outputContext.AudioEncodeContext->frame_size
                : fifoSize;

            var outputContext = _outputContext.AudioEncodeContext;
            var frame = ffmpeg.av_frame_alloc();
            frame->nb_samples = frameSize;
            frame->channel_layout = outputContext->channel_layout;
            frame->format = (int)outputContext->sample_fmt;
            frame->sample_rate = outputContext->sample_rate;

            if (ffmpeg.av_frame_get_buffer(frame, 0) < 0)
                ffmpeg.av_frame_free(&frame);

            // read frame
            if (ffmpeg.av_audio_fifo_read(_audioFifo.AudioFifo, (void**)&frame->data, frameSize) < frameSize)
            {
                ffmpeg.av_frame_free(&frame);
                return;
            }

            frame->pts = _audioFrameCount;
            _audioFrameCount += frame->nb_samples;

            // send to encoder 

            ffmpeg.av_frame_free(&frame);
        }


  


• Detect volume via mic, start recording, end on silence, transcribe and sent to endpoint

  15 juin 2023, par alphadmon

  I have been attempting to get this to work in many ways but I can't seem to get it right. Most of the time I get a part of it to work and then when I try to make other parts work, I generally break other things.

  


  I am intercepting the volume coming from the mic and if it is louder than 50, I start a recording. I then keep recording until there is a silence, if the silence is equal to 5 seconds I then stop the recording.

  


  I then send the recording to be transcribed by whisper using OpenAI API.

  


  Once that is returned, I then want to send it to the open ai chat end point and get the response.

  


  After that, I would like to start listening again.

  


  Here is what I have that is sort of working so far, but the recording is an empty file always :

  


  // DETECT SPEECH
const recorder = require('node-record-lpcm16');

// TRANSCRIBE
const fs = require("fs");
const ffmpeg = require("fluent-ffmpeg");
const mic = require("mic");
const { Readable } = require("stream");
const ffmpegPath = require("@ffmpeg-installer/ffmpeg").path;
require('dotenv').config();

// CHAT
const { Configuration, OpenAIApi } = require("openai");

// OPEN AI
const configuration = new Configuration({
    organization: process.env.OPENAI_ORG,
    apiKey: process.env.OPENAI_API_KEY,
});
const openai = new OpenAIApi(configuration);

// SETUP
ffmpeg.setFfmpegPath(ffmpegPath);

// VARS
let isRecording = false;
const audioFilename = 'recorded_audio.wav';
const micInstance = mic({
    rate: '16000',
    channels: '1',
    fileType: 'wav',
});

// DETECT SPEECH
const file = fs.createWriteStream('determine_speech.wav', { encoding: 'binary' });
const recording = recorder.record();
recording.stream().pipe(file);


recording.stream().on('data', async (data) => {
    let volume = parseInt(calculateVolume(data));
    if (volume > 50 && !isRecording) {
        console.log('You are talking.');
        await recordAudio(audioFilename);
    } else {
        setTimeout(async () => {
            console.log('You are quiet.');
            micInstance.stop();
            console.log('Finished recording');
            const transcription = await transcribeAudio(audioFilename);
            console.log('Transcription:', transcription);
            setTimeout(async () => {
                await askAI(transcription);
            }, 5000);
        }, 5000);
    }
});

function calculateVolume(data) {
    let sum = 0;

    for (let i = 0; i < data.length; i += 2) {
        const sample = data.readInt16LE(i);
        sum += sample * sample;
    }

    const rms = Math.sqrt(sum / (data.length / 2));

    return rms;
}

// TRANSCRIBE
function recordAudio(filename) {
    const micInputStream = micInstance.getAudioStream();
    const output = fs.createWriteStream(filename);
    const writable = new Readable().wrap(micInputStream);

    console.log('Listening...');

    writable.pipe(output);

    micInstance.start();

    micInputStream.on('error', (err) => {
        console.error(err);
    });
}

// Transcribe audio
async function transcribeAudio(filename) {
    const transcript = await openai.createTranscription(
        fs.createReadStream(filename),
        "whisper-1",
    );
    return transcript.data.text;
}

// CHAT
async function askAI(text) {
    let completion = await openai.createChatCompletion({
        model: "gpt-4",
        temperature: 0.2,
        stream: false,
        messages: [
            { role: "user", content: text },
            { role: "system", content: "Act like you are a rude person." }
        ],
    });

    completion = JSON.stringify(completion.data, null, 2);
    console.log(completion);
}


  


• mp4 Vj Animation video lagging hi res video

  21 février 2020, par Ryan Stone

  I am trying to get a video to play inside a video tag at the top left hand corner of my page, it loads ok, the resolution is good and it seems to be looping but it is lagging very much, definatly not achieving 60fps it is in mp4 format and the resolution on the original mp4 is 1920x1080 it is a hi resolution vj free loop called GlassVein, you can see it if you search on youtube. On right clicking properties it comes up with the following inforamtion ;

  Bitrate:127kbs
  Data rate:11270kbps
  Total bitrate:11398kbs
  Audio sample rate is : 44khz
  filetype is:VLC media file(.mp4)
  (but i do not want or need the audio)

  & it also says 30fps, but I’m not sure i believe this as it runs smooth as butter on vlc media player no lagging, just smooth loop animation

  I have searched on :https://trac.ffmpeg.org/wiki/Encode/AAC for encoding information but it is complete gobbldygook to me, I don’t understand a word its saying

  My code is so far as follows ;

      <video src="GlassVeinColorful.mp4" autoplay="1" preload="auto" class="Vid" width="640" height="360" loop="1" viewport="" faststart="faststart" mpeg4="mpeg4" 320x240="320x240" 1080="1080" 128k="128k">  
  
      </video>

  Does anyone know why this is lagging so much, or what I could do about it.
  it is a quality animation and I don’t really want to loose an of its resolution or crispness.. the -s section was originally set to 1920x1080 as this is what the original file is but i have changed it to try and render it quicker...

  Any helpful sites, articles or answers would be great..

  2020 Update

  The Solution to this problem was to convert the Video to WebM, then use Javascript & a Html5 Canvas Element to render the Video to the page instead of using the video tag to embed the video.

  Html

  <section>
  
          <video src="Imgs/Vid/PurpGlassVein.webm" type="video/webm" width="684" height="auto" muted="muted" loop="loop" autoplay="autoplay">
  
                 <source>
  
                 <source>
  
                 <source>
  
          </source></source></source></video>
  
          <canvas style="filter:opacity(0);"></canvas>
  
  </section>

  Css

  video{
  
     display:none !important;
  
     visibility:hidden;
  
  }

  Javascript

      const Canv = document.querySelector("canvas");
  
      const Video = document.querySelector("video");
  
      const Ctx = Canv.getContext("2d");
  
  
  
      Video.addEventListener('play',()=>{
  
        function step() {
  
          Ctx.drawImage(Video, 0, 0, Canv.width, Canv.height)
  
          requestAnimationFrame(step)
  
        }
  
        requestAnimationFrame(step);
  
      })
  
  
  
      Canv.animate({
  
          filter: ['opacity(0) blur(5.28px)','opacity(1) blur(8.20px)']
  
      },{
  
          duration: 7288,
  
          fill: 'forwards',
  
          easing: 'ease-in',
  
          iterations: 1,
  
          delay: 728
  
      })

  I’ve Also Used the Vanilla Javascript .animate() API to fade the element into the page when the page loads. But one Caveat is that both the Canvas and the off-screen Video Tag must match the original videos resolution otherwise it starts to lag again, however you can use Css to scale it down via transform:scale(0.5) ; which doesn’t seem to effect performance at all.

  runs smooth as butter, and doesn’t loose any of the high resolution image.
  Added a slight blur 0.34px onto it aswell to smooth it even more.

  Possibly could of still used ffmpeg to get a better[Smaller File Size] WebM Output file but thats something I’ll have to look into at a later date.