Recherche avancée

Sur d’autres sites (7069)

• Decoding and playing audio with ffmpeg and XAudio2 - frequency ratio wrong

  9 mars, par Brent de Carteret

  I'm using ffmpeg to decode audio and output it using the XAudio2 API, it works and plays synced with the video output using the pts. But it's high pitched (i.e. sounds like chipmunks).

  


  Setting breakpoints I can see it has set the correct sample rate from the audio codec in CreateSourceVoice. I'm stumped.

  


  Any help would be much appreciated.

  


  CDVDAUDIO.cpp

  


  #include "DVDAudioDevice.h"
   
HANDLE m_hBufferEndEvent;

CDVDAudio::CDVDAudio()
{
    m_pXAudio2 = NULL;
    m_pMasteringVoice = NULL;
    m_pSourceVoice = NULL;
    m_pWfx  = NULL;
    m_VoiceCallback = NULL;    
    m_hBufferEndEvent = CreateEvent(NULL, false, false, "Buffer end event");
}
    
CDVDAudio::~CDVDAudio()
{
    m_pXAudio2 = NULL;
    m_pMasteringVoice = NULL;
    m_pSourceVoice = NULL;
    m_pWfx  = NULL;
    m_VoiceCallback = NULL;
    CloseHandle(m_hBufferEndEvent);
    m_hBufferEndEvent = NULL;
}
    
bool CDVDAudio::Create(int iChannels, int iBitrate, int iBitsPerSample, bool bPasstrough)
{
    CoInitializeEx(NULL, COINIT_MULTITHREADED);
    HRESULT hr = XAudio2Create( &m_pXAudio2, 0, XAUDIO2_DEFAULT_PROCESSOR);
    
    if (SUCCEEDED(hr))
    {
        m_pXAudio2->CreateMasteringVoice( &m_pMasteringVoice );
    }
    
    // Create source voice
    WAVEFORMATEXTENSIBLE wfx;
    memset(&wfx, 0, sizeof(WAVEFORMATEXTENSIBLE));
    
    wfx.Format.wFormatTag           = WAVE_FORMAT_PCM;
    wfx.Format.nSamplesPerSec       = iBitrate;//pFFMpegData->pAudioCodecCtx->sample_rate;//48000 by default
    wfx.Format.nChannels            = iChannels;//pFFMpegData->pAudioCodecCtx->channels;
    wfx.Format.wBitsPerSample       = 16;
    wfx.Format.nBlockAlign          = wfx.Format.nChannels*16/8;
    wfx.Format.nAvgBytesPerSec      = wfx.Format.nSamplesPerSec * wfx.Format.nBlockAlign;
    wfx.Format.cbSize               = sizeof(WAVEFORMATEXTENSIBLE)-sizeof(WAVEFORMATEX);
    wfx.Samples.wValidBitsPerSample = wfx.Format.wBitsPerSample;
    
    if(wfx.Format.nChannels == 1)
    {
        wfx.dwChannelMask = SPEAKER_MONO;
    }
    else if(wfx.Format.nChannels == 2)
    {
        wfx.dwChannelMask = SPEAKER_STEREO;
    }
    else if(wfx.Format.nChannels == 5)
    {
        wfx.dwChannelMask = SPEAKER_5POINT1;
    }
    
    wfx.SubFormat = KSDATAFORMAT_SUBTYPE_PCM;
    
    unsigned int flags = 0;//XAUDIO2_VOICE_NOSRC;// | XAUDIO2_VOICE_NOPITCH;
        
    //Source voice
    m_VoiceCallback = new StreamingVoiceCallback(this);
    hr = m_pXAudio2->CreateSourceVoice(&m_pSourceVoice,(WAVEFORMATEX*)&wfx, 0 , 1.0f, m_VoiceCallback);
        
    if (!SUCCEEDED(hr))
        return false;
    
    // Start sound
    hr = m_pSourceVoice->Start(0);
    
    if(!SUCCEEDED(hr))
        return false;
    
    return true;
}
    
DWORD CDVDAudio::AddPackets(unsigned char* data, DWORD len)
{  
    memset(&m_SoundBuffer,0,sizeof(XAUDIO2_BUFFER));
    m_SoundBuffer.AudioBytes = len;
    m_SoundBuffer.pAudioData = data;
    m_SoundBuffer.pContext = NULL;//(VOID*)data;
    XAUDIO2_VOICE_STATE state;
    
    while (m_pSourceVoice->GetState( &state ), state.BuffersQueued > 60)
    {
        WaitForSingleObject( m_hBufferEndEvent, INFINITE );
    }
    
    m_pSourceVoice->SubmitSourceBuffer( &m_SoundBuffer );
    return 0;
}
    
void CDVDAudio::Destroy()
{
    m_pMasteringVoice->DestroyVoice();
    m_pXAudio2->Release();
    m_pSourceVoice->DestroyVoice();
    delete m_VoiceCallback;
    m_VoiceCallback = NULL;
}


  


  CDVDAUdioCodecFFmpeg.cpp

  


  #include "DVDAudioCodecFFmpeg.h"
#include "Log.h"
    
CDVDAudioCodecFFmpeg::CDVDAudioCodecFFmpeg() : CDVDAudioCodec()
{
    m_iBufferSize = 0;
    m_pCodecContext = NULL;
    m_bOpenedCodec = false;
}
    
CDVDAudioCodecFFmpeg::~CDVDAudioCodecFFmpeg()
{
    Dispose();
}
    
bool CDVDAudioCodecFFmpeg::Open(AVCodecID codecID, int iChannels, int iSampleRate)
{
    AVCodec* pCodec;
    m_bOpenedCodec = false;
    av_register_all();
    pCodec = avcodec_find_decoder(codecID);
    m_pCodecContext = avcodec_alloc_context3(pCodec);//avcodec_alloc_context();
    avcodec_get_context_defaults3(m_pCodecContext, pCodec);
    
    if (!pCodec)
    {
        CLog::Log(LOGERROR, "CDVDAudioCodecFFmpeg::Open() Unable to find codec");
        return false;
    }
    
    m_pCodecContext->debug_mv = 0;
    m_pCodecContext->debug = 0;
    m_pCodecContext->workaround_bugs = 1;
    
    if (pCodec->capabilities & CODEC_CAP_TRUNCATED)
        m_pCodecContext->flags |= CODEC_FLAG_TRUNCATED;
    
    m_pCodecContext->channels = iChannels;
    m_pCodecContext->sample_rate = iSampleRate;
    //m_pCodecContext->bits_per_sample = 24;
     
    /* //FIXME BRENT
        if( ExtraData && ExtraSize > 0 )
        {
            m_pCodecContext->extradata_size = ExtraSize;
            m_pCodecContext->extradata = m_dllAvCodec.av_mallocz(ExtraSize + FF_INPUT_BUFFER_PADDING_SIZE);
            memcpy(m_pCodecContext->extradata, ExtraData, ExtraSize);
        }
    */
    
    // set acceleration
    //m_pCodecContext->dsp_mask = FF_MM_FORCE | FF_MM_MMX | FF_MM_MMXEXT | FF_MM_SSE; //BRENT
    
    if (avcodec_open2(m_pCodecContext, pCodec, NULL) < 0)
    {
        CLog::Log(LOGERROR, "CDVDAudioCodecFFmpeg::Open() Unable to open codec");
        Dispose();
        return false;
    }
    
    m_bOpenedCodec = true;
    return true;
}
    
void CDVDAudioCodecFFmpeg::Dispose()
{
    if (m_pCodecContext)
    {
        if (m_bOpenedCodec)
            avcodec_close(m_pCodecContext);
        m_bOpenedCodec = false;
        av_free(m_pCodecContext);
        m_pCodecContext = NULL;
    }
    m_iBufferSize = 0;
}

int CDVDAudioCodecFFmpeg::Decode(BYTE* pData, int iSize)
{
    int iBytesUsed;
    if (!m_pCodecContext) return -1;
    
    //Copy into a FFMpeg AVPAcket again
    AVPacket packet;
    av_init_packet(&packet);
    
    packet.data=pData;
    packet.size=iSize;
    
    int iOutputSize = AVCODEC_MAX_AUDIO_FRAME_SIZE; //BRENT
    
    iBytesUsed = avcodec_decode_audio3(m_pCodecContext, (int16_t *)m_buffer, &iOutputSize/*m_iBufferSize*/, &packet);

    m_iBufferSize = iOutputSize;//BRENT

    return iBytesUsed;
}

int CDVDAudioCodecFFmpeg::GetData(BYTE** dst)
{
    *dst = m_buffer;
    return m_iBufferSize;
}

void CDVDAudioCodecFFmpeg::Reset()
{
    if (m_pCodecContext)
        avcodec_flush_buffers(m_pCodecContext);
}

int CDVDAudioCodecFFmpeg::GetChannels()
{
    if (m_pCodecContext)
        return m_pCodecContext->channels;
    return 0;
}

int CDVDAudioCodecFFmpeg::GetSampleRate()
{
    if (m_pCodecContext)
        return m_pCodecContext->sample_rate;
    return 0;
}
    
int CDVDAudioCodecFFmpeg::GetBitsPerSample()
{
    if (m_pCodecContext)
        return 16;
    return 0;
}


  


  CDVDPlayerAudio.cpp

  


  #include "DVDPlayerAudio.h"
#include "DVDDemuxUtils.h"
#include "Log.h"
    
#include 
#include "DVDAudioCodecFFmpeg.h" //FIXME Move to a codec factory!!
    
CDVDPlayerAudio::CDVDPlayerAudio(CDVDClock* pClock) : CThread()
{
    m_pClock = pClock;
    m_pAudioCodec = NULL;
    m_bInitializedOutputDevice = false;
    m_iSourceChannels = 0;
    m_audioClock = 0;
    
    //  m_currentPTSItem.pts = DVD_NOPTS_VALUE;
    //  m_currentPTSItem.timestamp = 0;
    
    SetSpeed(DVD_PLAYSPEED_NORMAL);
      
    InitializeCriticalSection(&m_critCodecSection);
    m_messageQueue.SetMaxDataSize(10 * 16 * 1024);
    //  g_dvdPerformanceCounter.EnableAudioQueue(&m_packetQueue);
}

CDVDPlayerAudio::~CDVDPlayerAudio()
{
    //  g_dvdPerformanceCounter.DisableAudioQueue();

    // close the stream, and don't wait for the audio to be finished
    CloseStream(true);
    DeleteCriticalSection(&m_critCodecSection);
}

bool CDVDPlayerAudio::OpenStream( CDemuxStreamAudio *pDemuxStream )
{
    // should always be NULL!!!!, it will probably crash anyway when deleting m_pAudioCodec here.
    if (m_pAudioCodec)
    {
        CLog::Log(LOGFATAL, "CDVDPlayerAudio::OpenStream() m_pAudioCodec != NULL");
        return false;
    }
    
    AVCodecID codecID = pDemuxStream->codec;
    
    CLog::Log(LOGNOTICE, "Finding audio codec for: %i", codecID);
    //m_pAudioCodec = CDVDFactoryCodec::CreateAudioCodec( pDemuxStream ); 
    m_pAudioCodec = new CDVDAudioCodecFFmpeg; //FIXME BRENT Codec Factory needed!
    
    if (!m_pAudioCodec->Open(pDemuxStream->codec, pDemuxStream->iChannels, pDemuxStream->iSampleRate))
    {
        m_pAudioCodec->Dispose();
        delete m_pAudioCodec;
        m_pAudioCodec = NULL;
        return false;
    }
    
    if ( !m_pAudioCodec )
    {
        CLog::Log(LOGERROR, "Unsupported audio codec");
        return false;
    }
    
    m_codec = pDemuxStream->codec;
    m_iSourceChannels = pDemuxStream->iChannels;
    m_messageQueue.Init();
    
    CLog::Log(LOGNOTICE, "Creating audio thread");
    Create();
    
    return true;
}

void CDVDPlayerAudio::CloseStream(bool bWaitForBuffers)
{
    // wait until buffers are empty
    if (bWaitForBuffers)
        m_messageQueue.WaitUntilEmpty();
    
    // send abort message to the audio queue
    m_messageQueue.Abort();
    
    CLog::Log(LOGNOTICE, "waiting for audio thread to exit");
    
    // shut down the audio_decode thread and wait for it
    StopThread(); // will set this->m_bStop to true
    this->WaitForThreadExit(INFINITE);
    
    // uninit queue
    m_messageQueue.End();
    
    CLog::Log(LOGNOTICE, "Deleting audio codec");
    if (m_pAudioCodec)
    {
        m_pAudioCodec->Dispose();
        delete m_pAudioCodec;
        m_pAudioCodec = NULL;
    }
    
    // flush any remaining pts values
    //FlushPTSQueue(); //FIXME BRENT
}

void CDVDPlayerAudio::OnStartup()
{
    CThread::SetName("CDVDPlayerAudio");
    pAudioPacket = NULL;
    m_audioClock = 0;
    audio_pkt_data = NULL;
    audio_pkt_size = 0;
  
    //  g_dvdPerformanceCounter.EnableAudioDecodePerformance(ThreadHandle());
}

void CDVDPlayerAudio::Process()
{
    CLog::Log(LOGNOTICE, "running thread: CDVDPlayerAudio::Process()");

    int result;
    
    // silence data
    BYTE silence[1024];
    memset(silence, 0, 1024);
    
    DVDAudioFrame audioframe;
    
    __int64 iClockDiff=0;
    while (!m_bStop)
    {
        //Don't let anybody mess with our global variables
        EnterCriticalSection(&m_critCodecSection);
        result = DecodeFrame(audioframe, m_speed != DVD_PLAYSPEED_NORMAL); // blocks if no audio is available, but leaves critical section before doing so
        LeaveCriticalSection(&m_critCodecSection);
    
        if ( result & DECODE_FLAG_ERROR ) 
        {      
            CLog::Log(LOGERROR, "CDVDPlayerAudio::Process - Decode Error. Skipping audio frame");
            continue;
        }
    
        if ( result & DECODE_FLAG_ABORT )
        {
            CLog::Log(LOGDEBUG, "CDVDPlayerAudio::Process - Abort received, exiting thread");
            break;
        }
    
        if ( result & DECODE_FLAG_DROP ) //FIXME BRENT
        {
            /*  //frame should be dropped. Don't let audio move ahead of the current time thou
                //we need to be able to start playing at any time
                //when playing backwards, we try to keep as small buffers as possible
    
                // set the time at this delay
                AddPTSQueue(audioframe.pts, m_dvdAudio.GetDelay());
            */
            if (m_speed > 0)
            {
                __int64 timestamp = m_pClock->GetAbsoluteClock() + (audioframe.duration * DVD_PLAYSPEED_NORMAL) / m_speed;
                while ( !m_bStop && timestamp > m_pClock->GetAbsoluteClock() )
                    Sleep(1);
            }
            continue;
        }
    
        if ( audioframe.size > 0 ) 
        {
            // we have successfully decoded an audio frame, open up the audio device if not already done
            if (!m_bInitializedOutputDevice)
            {
                m_bInitializedOutputDevice = InitializeOutputDevice();
            }
    
            //Add any packets play
            m_dvdAudio.AddPackets(audioframe.data, audioframe.size);
    
            // store the delay for this pts value so we can calculate the current playing
            //AddPTSQueue(audioframe.pts, m_dvdAudio.GetDelay() - audioframe.duration);//BRENT
        }
    
        // if we where asked to resync on this packet, do so here
        if ( result & DECODE_FLAG_RESYNC )
        {
            CLog::Log(LOGDEBUG, "CDVDPlayerAudio::Process - Resync recieved.");
            //while (!m_bStop && (unsigned int)m_dvdAudio.GetDelay() > audioframe.duration ) Sleep(5); //BRENT
            m_pClock->Discontinuity(CLOCK_DISC_NORMAL, audioframe.pts);
        }
    
        #ifdef USEOLDSYNC
        //Clock should be calculated after packets have been added as m_audioClock points to the 
        //time after they have been played
    
        const __int64 iCurrDiff = (m_audioClock - m_dvdAudio.GetDelay()) - m_pClock->GetClock();
        const __int64 iAvDiff = (iClockDiff + iCurrDiff)/2;
    
        //Check for discontinuity in the stream, use a moving average to
        //eliminate highfreq fluctuations of large packet sizes
        if ( ABS(iAvDiff) > 5000 ) // sync clock if average diff is bigger than 5 msec 
        {
            //Wait until only the new audio frame which triggered the discontinuity is left
            //then set disc state
            while (!m_bStop && (unsigned int)m_dvdAudio.GetBytesInBuffer() > audioframe.size )
                Sleep(5);
    
            m_pClock->Discontinuity(CLOCK_DISC_NORMAL, m_audioClock - m_dvdAudio.GetDelay());
            CLog::("CDVDPlayer:: Detected Audio Discontinuity, syncing clock. diff was: %I64d, %I64d, av: %I64d", iClockDiff, iCurrDiff, iAvDiff);
            iClockDiff = 0;
        }
        else
        {
            //Do gradual adjustments (not working yet)
            //m_pClock->AdjustSpeedToMatch(iClock + iAvDiff);
            iClockDiff = iCurrDiff;
        }
        #endif
    }
}

void CDVDPlayerAudio::OnExit()
{
    //g_dvdPerformanceCounter.DisableAudioDecodePerformance();
  
    // destroy audio device
    CLog::Log(LOGNOTICE, "Closing audio device");
    m_dvdAudio.Destroy();
    m_bInitializedOutputDevice = false;

    CLog::Log(LOGNOTICE, "thread end: CDVDPlayerAudio::OnExit()");
}

// decode one audio frame and returns its uncompressed size
int CDVDPlayerAudio::DecodeFrame(DVDAudioFrame &audioframe, bool bDropPacket)
{
    CDVDDemux::DemuxPacket* pPacket = pAudioPacket;
    int n=48000*2*16/8, len;
    
    //Store amount left at this point, and what last pts was
    unsigned __int64 first_pkt_pts = 0;
    int first_pkt_size = 0; 
    int first_pkt_used = 0;
    int result = 0;
    
    // make sure the sent frame is clean
    memset(&audioframe, 0, sizeof(DVDAudioFrame));
    
    if (pPacket)
    {
        first_pkt_pts = pPacket->pts;
        first_pkt_size = pPacket->iSize;
        first_pkt_used = first_pkt_size - audio_pkt_size;
    }
     
    for (;;)
    {
        /* NOTE: the audio packet can contain several frames */
        while (audio_pkt_size > 0)
        {
            len = m_pAudioCodec->Decode(audio_pkt_data, audio_pkt_size);
            if (len < 0)
            {
                /* if error, we skip the frame */
                audio_pkt_size=0;
                m_pAudioCodec->Reset();
                break;
            }
    
            // fix for fucked up decoders //FIXME BRENT
            if( len > audio_pkt_size )
            {        
                CLog::Log(LOGERROR, "CDVDPlayerAudio:DecodeFrame - Codec tried to consume more data than available. Potential memory corruption");        
                audio_pkt_size=0;
                m_pAudioCodec->Reset();
                assert(0);
            }
    
            // get decoded data and the size of it
            audioframe.size = m_pAudioCodec->GetData(&audioframe.data);
            audio_pkt_data += len;
            audio_pkt_size -= len;
    
            if (audioframe.size <= 0)
                continue;
    
            audioframe.pts = m_audioClock;
    
            // compute duration.
            n = m_pAudioCodec->GetChannels() * m_pAudioCodec->GetBitsPerSample() / 8 * m_pAudioCodec->GetSampleRate();
            if (n > 0)
            {
                // safety check, if channels == 0, n will result in 0, and that will result in a nice divide exception
                audioframe.duration = (unsigned int)(((__int64)audioframe.size * DVD_TIME_BASE) / n);
    
                // increase audioclock to after the packet
                m_audioClock += audioframe.duration;
            }
    
            //If we are asked to drop this packet, return a size of zero. then it won't be played
            //we currently still decode the audio.. this is needed since we still need to know it's 
            //duration to make sure clock is updated correctly.
            if ( bDropPacket )
            {
                result |= DECODE_FLAG_DROP;
            }
            return result;
        }
    
        // free the current packet
        if (pPacket)
        {
            CDVDDemuxUtils::FreeDemuxPacket(pPacket); //BRENT FIXME
            pPacket = NULL;
            pAudioPacket = NULL;
        }
    
        if (m_messageQueue.RecievedAbortRequest())
            return DECODE_FLAG_ABORT;
    
        // read next packet and return -1 on error
        LeaveCriticalSection(&m_critCodecSection); //Leave here as this might stall a while
    
        CDVDMsg* pMsg;
        MsgQueueReturnCode ret = m_messageQueue.Get(&pMsg, INFINITE);
        EnterCriticalSection(&m_critCodecSection);
            
        if (MSGQ_IS_ERROR(ret) || ret == MSGQ_ABORT)
            return DECODE_FLAG_ABORT;
    
        if (pMsg->IsType(CDVDMsg::DEMUXER_PACKET))
        {
            CDVDMsgDemuxerPacket* pMsgDemuxerPacket = (CDVDMsgDemuxerPacket*)pMsg;
            pPacket = pMsgDemuxerPacket->GetPacket();
            pMsgDemuxerPacket->m_pPacket = NULL; // XXX, test
            pAudioPacket = pPacket;
            audio_pkt_data = pPacket->pData;
            audio_pkt_size = pPacket->iSize;
        }
        else
        {
            // other data is not used here, free if
            // msg itself will still be available
            pMsg->Release();
        }
 
        // if update the audio clock with the pts
        if (pMsg->IsType(CDVDMsg::DEMUXER_PACKET) || pMsg->IsType(CDVDMsg::GENERAL_RESYNC))
        {
            if (pMsg->IsType(CDVDMsg::GENERAL_RESYNC))
            { 
                //player asked us to sync on this package
                CDVDMsgGeneralResync* pMsgGeneralResync = (CDVDMsgGeneralResync*)pMsg;
                result |= DECODE_FLAG_RESYNC;
                m_audioClock = pMsgGeneralResync->GetPts();
            }
            else if (pPacket->pts != DVD_NOPTS_VALUE) // CDVDMsg::DEMUXER_PACKET, pPacket is already set above
            {
                if (first_pkt_size == 0) 
                { 
                    //first package
                    m_audioClock = pPacket->pts;        
                }
                else if (first_pkt_pts > pPacket->pts)
                { 
                    //okey first packet in this continous stream, make sure we use the time here        
                    m_audioClock = pPacket->pts;        
                }
                else if ((unsigned __int64)m_audioClock < pPacket->pts || (unsigned __int64)m_audioClock > pPacket->pts)
                {
                    //crap, moved outsided correct pts
                    //Use pts from current packet, untill we find a better value for it.
                    //Should be ok after a couple of frames, as soon as it starts clean on a packet
                    m_audioClock = pPacket->pts;
                }
                else if (first_pkt_size == first_pkt_used)
                {
                    //Nice starting up freshly on the start of a packet, use pts from it
                    m_audioClock = pPacket->pts;
                }
            }
        }
        pMsg->Release();
    }
}

void CDVDPlayerAudio::SetSpeed(int speed)
{ 
    m_speed = speed;
  
    //if (m_speed == DVD_PLAYSPEED_PAUSE) m_dvdAudio.Pause(); //BRENT FIXME
    //else m_dvdAudio.Resume();
}
    
bool CDVDPlayerAudio::InitializeOutputDevice()
{
    int iChannels = m_pAudioCodec->GetChannels();
    int iSampleRate = m_pAudioCodec->GetSampleRate();
    int iBitsPerSample = m_pAudioCodec->GetBitsPerSample();
    //bool bPasstrough = m_pAudioCodec->NeedPasstrough(); //BRENT
    
    if (iChannels == 0 || iSampleRate == 0 || iBitsPerSample == 0)
    {
        CLog::Log(LOGERROR, "Unable to create audio device, (iChannels == 0 || iSampleRate == 0 || iBitsPerSample == 0)");
        return false;
    }
    
    CLog::Log(LOGNOTICE, "Creating audio device with codec id: %i, channels: %i, sample rate: %i", m_codec, iChannels, iSampleRate);
    if (m_dvdAudio.Create(iChannels, iSampleRate, iBitsPerSample, /*bPasstrough*/0)) // always 16 bit with ffmpeg ? //BRENT Passthrough needed?
    {
        return true;
    }
    
    CLog::Log(LOGERROR, "Failed Creating audio device with codec id: %i, channels: %i, sample rate: %i", m_codec, iChannels, iSampleRate);
    return false;
}


  


• How to stop ffmpeg when there's no incoming rtmp stream

  5 juillet 2016, par M. Irich

  I use ffmpeg together with nginx-rtmp.
  The thing is ffmpeg doesn’t finish the process when the stream’s finished

  I use the following command :

  ffmpeg  -i 'rtmp://localhost:443/live/test' -loglevel debug  -c:a libfdk_aac -b:a 192k -c:v libx264 -profile baseline -preset superfast -tune zerolatency -b:v 2500k -maxrate 4500k -minrate 1500k -bufsize 9000k -keyint_min 15 -g 15 -f dash -use_timeline 1 -use_template 1 -min_seg_duration 5000 -y /tmp/dash/test/test.mpd

  but even the stream’s not running ffmpeg still can’t finish the process and is waiting for the rtmp stream

  Successfully parsed a group of options.
  
  Opening an input file: rtmp://localhost:443/live/test.
  
  [rtmp @ 0x2ba2160] No default whitelist set
  
  [tcp @ 0x2ba2720] No default whitelist set
  
  [rtmp @ 0x2ba2160] Handshaking...
  
  [rtmp @ 0x2ba2160] Type answer 3
  
  [rtmp @ 0x2ba2160] Server version 13.14.10.13
  
  [rtmp @ 0x2ba2160] Proto = rtmp, path = /live/test, app = live, fname = test
  
  [rtmp @ 0x2ba2160] Server bandwidth = 5000000
  
  [rtmp @ 0x2ba2160] Client bandwidth = 5000000
  
  [rtmp @ 0x2ba2160] New incoming chunk size = 4096
  
  [rtmp @ 0x2ba2160] Creating stream...
  
  [rtmp @ 0x2ba2160] Sending play command for 'test'

  Is it possible to limit the latency time to several seconds ?

  Sorry for any possible mistakes - English’s not my native language.

• How to Stream RTP (IP camera) Into React App setup

  10 novembre 2024, par sharon2469

  I am trying to transfer a live broadcast from an IP camera or any other broadcast coming from an RTP/RTSP source to my REACT application. BUT MUST BE LIVE

  


  My setup at the moment is :

  


  IP Camera -> (RTP) -> FFmpeg -> (udp) -> Server(nodeJs) -> (WebRTC) -> React app

  


  In the current situation, There is almost no delay, but there are some things here that I can't avoid and I can't understand why, and here is my question :

  


  1) First, is the SETUP even correct and this is the only way to Stream RTP video in Web app ?

  


  2) Is it possible to avoid re-encode the stream , RTP transmission necessarily comes in H.264, hence I don't really need to execute the following command :

  


      return spawn('ffmpeg', [
    '-re',                              // Read input at its native frame rate Important for live-streaming
    '-probesize', '32',                 // Set probing size to 32 bytes (32 is minimum)
    '-analyzeduration', '1000000',      // An input duration of 1 second
    '-c:v', 'h264',                     // Video codec of input video
    '-i', 'rtp://238.0.0.2:48888',      // Input stream URL
    '-map', '0:v?',                     // Select video from input stream
    '-c:v', 'libx264',                  // Video codec of output stream
    '-preset', 'ultrafast',             // Faster encoding for lower latency
    '-tune', 'zerolatency',             // Optimize for zero latency
    // '-s', '768x480',                    // Adjust the resolution (experiment with values)
    '-f', 'rtp', `rtp://127.0.0.1:${udpPort}` // Output stream URL
]);


  


  As you can se in this command I re-encode to libx264, But if I set FFMPEG a parameter '-c:v' :'copy' instead of '-c:v', 'libx264' then FFMPEG throw an error says : that it doesn't know how to encode h264 and only knows what is libx264-> Basically, I want to stop the re-encode because there is really no need for it, because the stream is already encoded to H264. Are there certain recommendations that can be made ?

  


  3) I thought about giving up the FFMPEG completely, but the RTP packets arrive at a size of 1200+ BYTES when WEBRTC is limited to up to 1280 BYTE. Is there a way to manage these sabotages without damaging the video and is it to enter this world ? I guess there is the whole story with the JITTER BUFFER here

  


  This is my server side code (THIS IS JUST A TEST CODE)

  


  import {
    MediaStreamTrack,
    randomPort,
    RTCPeerConnection,
    RTCRtpCodecParameters,
    RtpPacket,
} from 'werift'
import {Server} from "ws";
import {createSocket} from "dgram";
import {spawn} from "child_process";
import LoggerFactory from "./logger/loggerFactory";

//

const log = LoggerFactory.getLogger('ServerMedia')

// Websocket server -> WebRTC
const serverPort = 8888
const server = new Server({port: serverPort});
log.info(`Server Media start om port: ${serverPort}`);

// UDP server -> ffmpeg
const udpPort = 48888
const udp = createSocket("udp4");
// udp.bind(udpPort, () => {
//     udp.addMembership("238.0.0.2");
// })
udp.bind(udpPort)
log.info(`UDP port: ${udpPort}`)


const createFFmpegProcess = () => {
    log.info(`Start ffmpeg process`)
    return spawn('ffmpeg', [
        '-re',                              // Read input at its native frame rate Important for live-streaming
        '-probesize', '32',                 // Set probing size to 32 bytes (32 is minimum)
        '-analyzeduration', '1000000',      // An input duration of 1 second
        '-c:v', 'h264',                     // Video codec of input video
        '-i', 'rtp://238.0.0.2:48888',      // Input stream URL
        '-map', '0:v?',                     // Select video from input stream
        '-c:v', 'libx264',                  // Video codec of output stream
        '-preset', 'ultrafast',             // Faster encoding for lower latency
        '-tune', 'zerolatency',             // Optimize for zero latency
        // '-s', '768x480',                    // Adjust the resolution (experiment with values)
        '-f', 'rtp', `rtp://127.0.0.1:${udpPort}` // Output stream URL
    ]);

}

let ffmpegProcess = createFFmpegProcess();


const attachFFmpegListeners = () => {
    // Capture standard output and print it
    ffmpegProcess.stdout.on('data', (data) => {
        log.info(`FFMPEG process stdout: ${data}`);
    });

    // Capture standard error and print it
    ffmpegProcess.stderr.on('data', (data) => {
        console.error(`ffmpeg stderr: ${data}`);
    });

    // Listen for the exit event
    ffmpegProcess.on('exit', (code, signal) => {
        if (code !== null) {
            log.info(`ffmpeg process exited with code ${code}`);
        } else if (signal !== null) {
            log.info(`ffmpeg process killed with signal ${signal}`);
        }
    });
};


attachFFmpegListeners();


server.on("connection", async (socket) => {
    const payloadType = 96; // It is a numerical value that is assigned to each codec in the SDP offer/answer exchange -> for H264
    // Create a peer connection with the codec parameters set in advance.
    const pc = new RTCPeerConnection({
        codecs: {
            audio: [],
            video: [
                new RTCRtpCodecParameters({
                    mimeType: "video/H264",
                    clockRate: 90000, // 90000 is the default value for H264
                    payloadType: payloadType,
                }),
            ],
        },
    });

    const track = new MediaStreamTrack({kind: "video"});


    udp.on("message", (data) => {
        console.log(data)
        const rtp = RtpPacket.deSerialize(data);
        rtp.header.payloadType = payloadType;
        track.writeRtp(rtp);
    });

    udp.on("error", (err) => {
        console.log(err)

    });

    udp.on("close", () => {
        console.log("close")
    });

    pc.addTransceiver(track, {direction: "sendonly"});

    await pc.setLocalDescription(await pc.createOffer());
    const sdp = JSON.stringify(pc.localDescription);
    socket.send(sdp);

    socket.on("message", (data: any) => {
        if (data.toString() === 'resetFFMPEG') {
            ffmpegProcess.kill('SIGINT');
            log.info(`FFMPEG process killed`)
            setTimeout(() => {
                ffmpegProcess = createFFmpegProcess();
                attachFFmpegListeners();
            }, 5000)
        } else {
            pc.setRemoteDescription(JSON.parse(data));
        }
    });
});


  


  And this fronted :

  


  &#xA;&#xA;&#xA;    &#xA;    &#xA;    <code class="echappe-js">&lt;script&amp;#xA;            crossorigin&amp;#xA;            src=&quot;https://unpkg.com/react@16/umd/react.development.js&quot;&amp;#xA;    &gt;&lt;/script&gt;&#xA;    &lt;script&amp;#xA;            crossorigin&amp;#xA;            src=&quot;https://unpkg.com/react-dom@16/umd/react-dom.development.js&quot;&amp;#xA;    &gt;&lt;/script&gt;&#xA;    &lt;script&amp;#xA;            crossorigin&amp;#xA;            src=&quot;https://cdnjs.cloudflare.com/ajax/libs/babel-core/5.8.34/browser.min.js&quot;&amp;#xA;    &gt;&lt;/script&gt;&#xA;    &lt;script src=&quot;https://cdn.jsdelivr.net/npm/babel-regenerator-runtime@6.5.0/runtime.min.js&quot;&gt;&lt;/script&gt;&#xA;&#xA;&#xA;
  &#xA;    
  
  &#xA;
  
  &#xA;&lt;script type=&quot;text/babel&quot;&gt;&amp;#xA;    let rtc;&amp;#xA;&amp;#xA;    const App = () =&gt; {&amp;#xA;        const [log, setLog] = React.useState([]);&amp;#xA;        const videoRef = React.useRef();&amp;#xA;        const socket = new WebSocket(&quot;ws://localhost:8888&quot;);&amp;#xA;        const [peer, setPeer] = React.useState(null); // Add state to keep track of the peer connection&amp;#xA;&amp;#xA;        React.useEffect(() =&gt; {&amp;#xA;            (async () =&gt; {&amp;#xA;                await new Promise((r) =&gt; (socket.onopen = r));&amp;#xA;                console.log(&quot;open websocket&quot;);&amp;#xA;&amp;#xA;                const handleOffer = async (offer) =&gt; {&amp;#xA;                    console.log(&quot;new offer&quot;, offer.sdp);&amp;#xA;&amp;#xA;                    const updatedPeer = new RTCPeerConnection({&amp;#xA;                        iceServers: [],&amp;#xA;                        sdpSemantics: &quot;unified-plan&quot;,&amp;#xA;                    });&amp;#xA;&amp;#xA;                    updatedPeer.onicecandidate = ({ candidate }) =&gt; {&amp;#xA;                        if (!candidate) {&amp;#xA;                            const sdp = JSON.stringify(updatedPeer.localDescription);&amp;#xA;                            console.log(sdp);&amp;#xA;                            socket.send(sdp);&amp;#xA;                        }&amp;#xA;                    };&amp;#xA;&amp;#xA;                    updatedPeer.oniceconnectionstatechange = () =&gt; {&amp;#xA;                        console.log(&amp;#xA;                            &quot;oniceconnectionstatechange&quot;,&amp;#xA;                            updatedPeer.iceConnectionState&amp;#xA;                        );&amp;#xA;                    };&amp;#xA;&amp;#xA;                    updatedPeer.ontrack = (e) =&gt; {&amp;#xA;                        console.log(&quot;ontrack&quot;, e);&amp;#xA;                        videoRef.current.srcObject = e.streams[0];&amp;#xA;                    };&amp;#xA;&amp;#xA;                    await updatedPeer.setRemoteDescription(offer);&amp;#xA;                    const answer = await updatedPeer.createAnswer();&amp;#xA;                    await updatedPeer.setLocalDescription(answer);&amp;#xA;&amp;#xA;                    setPeer(updatedPeer);&amp;#xA;                };&amp;#xA;&amp;#xA;                socket.onmessage = (ev) =&gt; {&amp;#xA;                    const data = JSON.parse(ev.data);&amp;#xA;                    if (data.type === &quot;offer&quot;) {&amp;#xA;                        handleOffer(data);&amp;#xA;                    } else if (data.type === &quot;resetFFMPEG&quot;) {&amp;#xA;                        // Handle the resetFFMPEG message&amp;#xA;                        console.log(&quot;FFmpeg reset requested&quot;);&amp;#xA;                    }&amp;#xA;                };&amp;#xA;            })();&amp;#xA;        }, []); // Added socket as a dependency to the useEffect hook&amp;#xA;&amp;#xA;        const sendRequestToResetFFmpeg = () =&gt; {&amp;#xA;            socket.send(&quot;resetFFMPEG&quot;);&amp;#xA;        };&amp;#xA;&amp;#xA;        return (&amp;#xA;            &lt;div&gt;&amp;#xA;                Video: &amp;#xA;                &lt;video ref={videoRef} autoPlay muted /&gt;&amp;#xA;                &lt;button onClick={() =&gt; sendRequestToResetFFmpeg()}&gt;Reset FFMPEG&lt;/button&gt;&amp;#xA;            &lt;/div&gt;&amp;#xA;        );&amp;#xA;    };&amp;#xA;&amp;#xA;    ReactDOM.render(&lt;App /&gt;, document.getElementById(&quot;app1&quot;));&amp;#xA;&lt;/script&gt;&#xA;&#xA;&#xA;