Recherche avancée

Sur d’autres sites (5416)

• FFMPEG throws error while converting WEBM/MP4 audio files from MediaRecorder to MP3

  27 septembre 2022, par Sahil Malik

  I use MediaRecorder to record the audio from the browser and then upload it to my server (ARM-based Linux machine - AWS Lambda Function, if it matters). Based on the browser type, I get either an MP4 file (for the Safari browser) OR a WEBM file (for every other browser) from the MediaRecorder. The audio is converted to Base64 string and posted to my server with FFMPEG.

  


  MediaRecorder implementation

  


  const audioStream = await navigator.mediaDevices.getUserMedia(mediaConstraints);
const MediaRecorder = window['MediaRecorder'];
const mimeType = MediaRecorder.isTypeSupported('audio/webm') ? 'audio/webm' : 'audio/mp4';
const recordingFileExtension = MediaRecorder.isTypeSupported('audio/webm') ? 'webm' : 'mp4';
mediaRecorder = new MediaRecorder(audioStream, { mimeType });
mediaRecorder.ondataavailable = convertBlobAndUploadChunk;
mediaRecorder.start(30 * 1000); // timeslice needs to be in ms
// When user stops recording
mediaRecorder.stop();
audioStream.getTracks().forEach( t => { t.stop(); });
// To convert the audio blob to string
function convertBlobToBase64(blob) {
    return new Promise((resolve, reject) => {
        const reader = new FileReader();
        reader.readAsDataURL(blob);
        reader.onload = () => {
            const tmpStr = reader.result.toString();
            resolve(tmpStr.substring(tmpStr.indexOf('base64,') + 7));
        };
        reader.onerror = error => reject(error);
    });
}

let blobCount = 0;
async function convertBlobAndUploadChunk(blobEvent) {
    if (!blobEvent.data || blobEvent.data.size === 0) return;

    blobCount++;

    const recordData = JSON.stringify({
        M: {
            blobCount,
            //some other meta data
        },
        D: await convertBlobToBase64(audioBlob),
    });

    await angularHttpClient.post(apiUrl, recordData, new HttpHeaders({
        'Content-Type': 'application/json',
        'x-api-key': apiKey,
    })).toPromise();
}


  


  On my server, I convert the WEBM/MP4 file to an MP3 file for better cross-browser compatibility and to enable scrubbing.

  


  Backend Lambda Implementation

  


  const FFMpegCommand = require('fluent-ffmpeg'); // v2.1.2
new FFMpegCommand()
        .input(originalFile)
        .on('end', (error, stdOut, stdError) => {
            if (error) {
                console.error(error);
            }
        })
        .save(convertedFile);


  


  This works perfectly fine 99.9% of the time but FFMPEG throws one of the following errors for 0.1% of the time :

  


  Error 1 : Invalid data found when processing input

  


  error reading header

  


  ffmpeg version 4.4-static https://johnvansickle.com/ffmpeg/ Copyright (c) 2000-2021 the 
FFmpeg developers
built with gcc 8 (Debian 8.3.0-6)
configuration: --enable-gpl --enable-version3 --enable-static --disable-debug --disable-ffplay --disable-indev=sndio --disable-outdev=sndio --cc=gcc --enable-fontconfig --enable-frei0r --enable-gnutls --enable-gmp --enable-libgme --enable-gray --enable-libaom --enable-libfribidi --enable-libass --enable-libvmaf --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librubberband --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libvorbis --enable-libopus --enable-libtheora --enable-libvidstab --enable-libvo-amrwbenc --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libdav1d --enable-libxvid --enable-libzvbi --enable-libzimg
libavutil 56. 70.100 / 56. 70.100
libavcodec 58.134.100 / 58.134.100
libavformat 58. 76.100 / 58. 76.100
libavdevice 58. 13.100 / 58. 13.100
libavfilter 7.110.100 / 7.110.100
libswscale 5. 9.100 / 5. 9.100
libswresample 3. 9.100 / 3. 9.100
libpostproc 55. 9.100 / 55. 9.100
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7232f40] could not find corresponding trex (id 1)
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7232f40] could not find corresponding track id 0
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7232f40] trun track id unknown, no tfhd was found
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7232f40] error reading header
/tmp/long-filename-of-140chars-to-keep-files-unique.mp4: Invalid data found when processing input


  


  EBML Header parsing failed

  


  ffmpeg version 4.4-static https://johnvansickle.com/ffmpeg/ Copyright (c) 2000-2021 the FFmpeg developers
built with gcc 8 (Debian 8.3.0-6)
configuration: --enable-gpl --enable-version3 --enable-static --disable-debug --disable-ffplay --disable-indev=sndio --disable-outdev=sndio --cc=gcc --enable-fontconfig --enable-frei0r --enable-gnutls --enable-gmp --enable-libgme --enable-gray --enable-libaom --enable-libfribidi --enable-libass --enable-libvmaf --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librubberband --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libvorbis --enable-libopus --enable-libtheora --enable-libvidstab --enable-libvo-amrwbenc --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libdav1d --enable-libxvid --enable-libzvbi --enable-libzimg
libavutil 56. 70.100 / 56. 70.100
libavcodec 58.134.100 / 58.134.100
libavformat 58. 76.100 / 58. 76.100
libavdevice 58. 13.100 / 58. 13.100
libavfilter 7.110.100 / 7.110.100
libswscale 5. 9.100 / 5. 9.100
libswresample 3. 9.100 / 3. 9.100
libpostproc 55. 9.100 / 55. 9.100
[matroska,webm @ 0x7595f40] Format matroska,webm detected only with low score of 1, misdetection possible!
[matroska,webm @ 0x7595f40] EBML header parsing failed
/tmp/long-filename-of-140chars-to-keep-files-unique.webm: Invalid data found when processing input


  


  No specific details

  


  ffmpeg version 4.4-static https://johnvansickle.com/ffmpeg/ Copyright (c) 2000-2021 the FFmpeg developers
built with gcc 8 (Debian 8.3.0-6)
configuration: --enable-gpl --enable-version3 --enable-static --disable-debug --disable-ffplay --disable-indev=sndio --disable-outdev=sndio --cc=gcc --enable-fontconfig --enable-frei0r --enable-gnutls --enable-gmp --enable-libgme --enable-gray --enable-libaom --enable-libfribidi --enable-libass --enable-libvmaf --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librubberband --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libvorbis --enable-libopus --enable-libtheora --enable-libvidstab --enable-libvo-amrwbenc --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libdav1d --enable-libxvid --enable-libzvbi --enable-libzimg
libavutil 56. 70.100 / 56. 70.100
libavcodec 58.134.100 / 58.134.100
libavformat 58. 76.100 / 58. 76.100
libavdevice 58. 13.100 / 58. 13.100
libavfilter 7.110.100 / 7.110.100
libswscale 5. 9.100 / 5. 9.100
libswresample 3. 9.100 / 3. 9.100
libpostproc 55. 9.100 / 55. 9.100
/tmp/long-filename-of-140chars-to-keep-files-unique.webm: Invalid data found when processing input


  


  Error 2 : Output file #0 does not contain any stream

  


  ffmpeg version 4.4-static https://johnvansickle.com/ffmpeg/ Copyright (c) 2000-2021 the FFmpeg developers
built with gcc 8 (Debian 8.3.0-6)
configuration: --enable-gpl --enable-version3 --enable-static --disable-debug --disable-ffplay --disable-indev=sndio --disable-outdev=sndio --cc=gcc --enable-fontconfig --enable-frei0r --enable-gnutls --enable-gmp --enable-libgme --enable-gray --enable-libaom --enable-libfribidi --enable-libass --enable-libvmaf --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librubberband --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libvorbis --enable-libopus --enable-libtheora --enable-libvidstab --enable-libvo-amrwbenc --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libdav1d --enable-libxvid --enable-libzvbi --enable-libzimg
libavutil 56. 70.100 / 56. 70.100
libavcodec 58.134.100 / 58.134.100
libavformat 58. 76.100 / 58. 76.100
libavdevice 58. 13.100 / 58. 13.100
libavfilter 7.110.100 / 7.110.100
libswscale 5. 9.100 / 5. 9.100
libswresample 3. 9.100 / 3. 9.100
libpostproc 55. 9.100 / 55. 9.100
[mpegts @ 0x5d19f40] Format mpegts detected only with low score of 2, misdetection possible!
[mpegts @ 0x5d19f40] Could not detect TS packet size, defaulting to non-FEC/DVHS
Input #0, mpegts, from '/tmp/long-filename-of-140chars-to-keep-files-unique.webm':
Duration: N/A, bitrate: N/A
Output #0, mp3, to '/tmp/long-filename-of-140chars-to-keep-files-unique.mp3':
Output file #0 does not contain any stream


  


  Thanks for reading, any help/suggestion is highly appreciated.

  


• How to improve web camera streaming latency to v4l2loopback device with ffmpeg ?

  11 mars, par Made by Moses

  I'm trying to stream my iPhone camera to my PC on LAN.

  


  What I've done :

  


  


  1. HTTP server with html page and streaming script :

     


     I use WebSockets here and maybe WebRTC is better choice but it seems like network latency is good enough

     


  


  


  async function beginCameraStream() {
  const mediaStream = await navigator.mediaDevices.getUserMedia({
    video: { facingMode: "user" },
  });

  websocket = new WebSocket(SERVER_URL);

  websocket.onopen = () => {
    console.log("WS connected");

    const options = { mimeType: "video/mp4", videoBitsPerSecond: 1_000_000 };
    mediaRecorder = new MediaRecorder(mediaStream, options);

    mediaRecorder.ondataavailable = async (event) => {
      // to measure latency I prepend timestamp to the actual video bytes chunk
      const timestamp = Date.now();
      const timestampBuffer = new ArrayBuffer(8);
      const dataView = new DataView(timestampBuffer);
      dataView.setBigUint64(0, BigInt(timestamp), true);
      const data = await event.data.bytes();

      const result = new Uint8Array(data.byteLength + 8);
      result.set(new Uint8Array(timestampBuffer), 0);
      result.set(data, 8);

      websocket.send(result);
    };

    mediaRecorder.start(100); // Collect 100ms chunks
  };
}


  


  


  2. Server to process video chunks

     


  


  


  import { serve } from "bun";
import { Readable } from "stream";

const V4L2LOOPBACK_DEVICE = "/dev/video10";

export const setupFFmpeg = (v4l2device) => {
  // prettier-ignore
  return spawn("ffmpeg", [
    '-i', 'pipe:0',           // Read from stdin
    '-pix_fmt', 'yuv420p',    // Pixel format
    '-r', '30',               // Target 30 fps
    '-f', 'v4l2',             // Output format
    v4l2device, // Output to v4l2loopback device
  ]);
};

export class FfmpegStream extends Readable {
  _read() {
    // This is called when the stream wants more data
    // We push data when we get chunks
  }
}

function main() {
  const ffmpeg = setupFFmpeg(V4L2LOOPBACK_DEVICE);
  serve({
    port: 8000,
    fetch(req, server) {
      if (server.upgrade(req)) {
        return; // Upgraded to WebSocket
      }
    },
    websocket: {
      open(ws) {
        console.log("Client connected");
        const stream = new FfmpegStream();
        stream.pipe(ffmpeg?.stdin);

        ws.data = {
          stream,
          received: 0,
        };
      },
      async message(ws, message) {
        const view = new DataView(message.buffer, 0, 8);
        const ts = Number(view.getBigUint64(0, true));
        ws.data.received += message.byteLength;
        const chunk = new Uint8Array(message.buffer, 8, message.byteLength - 8);

        ws.data.stream.push(chunk);

        console.log(
          [
            `latency: ${Date.now() - ts} ms`,
            `chunk: ${message.byteLength}`,
            `total: ${ws.data.received}`,
          ].join(" | "),
        );
      },
    },
  });
}

main();


  


  After I try to open the v4l2loopback device

  


  cvlc v4l2:///dev/video10


  


  picture is delayed for at least 1.5 sec which is unacceptable for my project.

  


  Thoughts :

  


  

  • Problem doesn't seems to be with network latency

  


  


  latency: 140 ms | chunk: 661 Bytes | total: 661 Bytes
latency: 206 ms | chunk: 16.76 KB | total: 17.41 KB
latency: 141 ms | chunk: 11.28 KB | total: 28.68 KB
latency: 141 ms | chunk: 13.05 KB | total: 41.74 KB
latency: 199 ms | chunk: 11.39 KB | total: 53.13 KB
latency: 141 ms | chunk: 16.94 KB | total: 70.07 KB
latency: 139 ms | chunk: 12.67 KB | total: 82.74 KB
latency: 142 ms | chunk: 13.14 KB | total: 95.88 KB


  


   150ms is actually too much for 15KB on LAN but there can some issue with my router

  


  

  • As far as I can tell it neither ties to ffmpeg throughput :

  


  


  Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'pipe:0':
  Metadata:
    major_brand     : iso5
    minor_version   : 1
    compatible_brands: isomiso5hlsf
    creation_time   : 2025-03-09T17:16:49.000000Z
  Duration: 00:00:01.38, start:
0.000000, bitrate: N/A
    Stream #0:0(und): Video: h264 (Baseline) (avc1 / 0x31637661), yuvj420p(pc), 1280x720, 4012 kb/s, 57.14 fps, 29.83 tbr, 600 tbn, 1200 tbc (default)
    Metadata:
      rotate          : 90
      creation_time   : 2025-03-09T17:16:49.000000Z
      handler_name    : Core Media Video
    Side data:
      displaymatrix: rotation of -90.00 degrees

Stream mapping:
  Stream #0:0 -> #0:0 (h264 (native) -> rawvideo (native))

[swscaler @ 0x55d8d0b83100] deprecated pixel format used, make sure you did set range correctly

Output #0, video4linux2,v4l2, to '/dev/video10':
  Metadata:
    major_brand     : iso5
    minor_version   : 1
    compatible_brands: isomiso5hlsf
    encoder         : Lavf58.45.100

Stream #0:0(und): Video: rawvideo (I420 / 0x30323449), yuv420p, 720x1280, q=2-31, 663552 kb/s, 60 fps, 60 tbn, 60 tbc (default)
    Metadata:
      encoder         : Lavc58.91.100 rawvideo
      creation_time   : 2025-03-09T17:16:49.000000Z
      handler_name    : Core Media Video
    Side data:
      displaymatrix: rotation of -0.00 degrees

frame=   99 fps=0.0 q=-0.0 size=N/A time=00:00:01.65 bitrate=N/A dup=50 drop=0 speed=2.77x
frame=  137 fps=114 q=-0.0 size=N/A time=00:00:02.28 bitrate=N/A dup=69 drop=0 speed=1.89x
frame=  173 fps= 98 q=-0.0 size=N/A time=00:00:02.88 bitrate=N/A dup=87 drop=0 speed=1.63x
frame=  210 fps= 86 q=-0.0 size=N/A time=00:00:03.50 bitrate=N/A dup=105 drop=0 speed=1.44x
frame=  249 fps= 81 q=-0.0 size=N/A time=00:00:04.15 bitrate=N/A dup=125 drop=0 speed=1.36
frame=  279 fps= 78 q=-0.0 size=N/A time=00:00:04.65 bitrate=N/A dup=139 drop=0 speed=1.31x


  


  


  • I also tried to write the video stream directly to video.mp4 file and immediately open it with vlc but it only can be successfully opened after 1.5 sec.

    


  


  • I've tried to use OBS v4l2 input source instead of vlc but the latency is the same

    


  


  


  Update №1

  


  When i try to stream actual .mp4 file to ffmpeg it works almost immediately with 0.2sec delay to spin up the ffmpeg itself :

  


  cat video.mp4 | ffmpeg -re -i pipe:0 -pix_fmt yuv420p -f v4l2 /dev/video10 & ; sleep 0.2 && cvlc v4l2:///dev/video10


  


  So the problem is apparently with streaming process

  


• how to add audio using ffmpeg when recording video from browser and streaming to Youtube/Twitch ?

  26 juillet 2021, par Tosh Velaga

  I have a web application I am working on that allows the user to stream video from their browser and simultaneously livestream to both Youtube and Twitch using ffmpeg. The application works fine when I don't need to send any of the audio. Currently I am getting the error below when I try to record video and audio. I am new to using ffmpeg and so any help would be greatly appreciated. Here is also my repo if needed : https://github.com/toshvelaga/livestream

  


  Here is my node.js server with ffmpeg

  


  const child_process = require('child_process') // To be used later for running FFmpeg
const express = require('express')
const http = require('http')
const WebSocketServer = require('ws').Server
const NodeMediaServer = require('node-media-server')
const app = express()
const cors = require('cors')
const path = require('path')
const logger = require('morgan')
require('dotenv').config()

app.use(logger('dev'))
app.use(cors())

app.use(express.json({ limit: '200mb', extended: true }))
app.use(
  express.urlencoded({ limit: '200mb', extended: true, parameterLimit: 50000 })
)

var authRouter = require('./routes/auth')
var compareCodeRouter = require('./routes/compareCode')

app.use('/', authRouter)
app.use('/', compareCodeRouter)

if (process.env.NODE_ENV === 'production') {
  // serve static content
  // npm run build
  app.use(express.static(path.join(__dirname, 'client/build')))

  app.get('*', (req, res) => {
    res.sendFile(path.join(__dirname, 'client/build', 'index.html'))
  })
}

const PORT = process.env.PORT || 8080

app.listen(PORT, () => {
  console.log(`Server is starting on port ${PORT}`)
})

const server = http.createServer(app).listen(3000, () => {
  console.log('Listening on PORT 3000...')
})


const wss = new WebSocketServer({
  server: server,
})

wss.on('connection', (ws, req) => {
  const ffmpeg = child_process.spawn('ffmpeg', [
    // works fine when I use this but when I need audio problems arise
    // '-f',
    // 'lavfi',
    // '-i',
    // 'anullsrc',

    '-i',
    '-',

    '-f',
    'flv',
    '-c',
    'copy',
    `${process.env.TWITCH_STREAM_ADDRESS}`,
    '-f',
    'flv',
    '-c',
    'copy',
    `${process.env.YOUTUBE_STREAM_ADDRESS}`,
    // '-f',
    // 'flv',
    // '-c',
    // 'copy',
    // `${process.env.FACEBOOK_STREAM_ADDRESS}`,
  ])

  ffmpeg.on('close', (code, signal) => {
    console.log(
      'FFmpeg child process closed, code ' + code + ', signal ' + signal
    )
    ws.terminate()
  })

  ffmpeg.stdin.on('error', (e) => {
    console.log('FFmpeg STDIN Error', e)
  })

  ffmpeg.stderr.on('data', (data) => {
    console.log('FFmpeg STDERR:', data.toString())
  })

  ws.on('message', (msg) => {
    console.log('DATA', msg)
    ffmpeg.stdin.write(msg)
  })

  ws.on('close', (e) => {
    console.log('kill: SIGINT')
    ffmpeg.kill('SIGINT')
  })
})

const config = {
  rtmp: {
    port: 1935,
    chunk_size: 60000,
    gop_cache: true,
    ping: 30,
    ping_timeout: 60,
  },
  http: {
    port: 8000,
    allow_origin: '*',
  },
}

var nms = new NodeMediaServer(config)
nms.run()


  


  Here is my frontend code that records the video/audio and sends to server :

  


  import React, { useState, useEffect, useRef } from &#x27;react&#x27;&#xA;import Navbar from &#x27;../../components/Navbar/Navbar&#x27;&#xA;import &#x27;./Dashboard.css&#x27;&#xA;&#xA;const CAPTURE_OPTIONS = {&#xA;  audio: true,&#xA;  video: true,&#xA;}&#xA;&#xA;function Dashboard() {&#xA;  const [mute, setMute] = useState(false)&#xA;  const videoRef = useRef()&#xA;  const ws = useRef()&#xA;  const mediaStream = useUserMedia(CAPTURE_OPTIONS)&#xA;&#xA;  let liveStream&#xA;  let liveStreamRecorder&#xA;&#xA;  if (mediaStream &amp;&amp; videoRef.current &amp;&amp; !videoRef.current.srcObject) {&#xA;    videoRef.current.srcObject = mediaStream&#xA;  }&#xA;&#xA;  const handleCanPlay = () => {&#xA;    videoRef.current.play()&#xA;  }&#xA;&#xA;  useEffect(() => {&#xA;    ws.current = new WebSocket(&#xA;      window.location.protocol.replace(&#x27;http&#x27;, &#x27;ws&#x27;) &#x2B;&#xA;        &#x27;//&#x27; &#x2B; // http: -> ws:, https: -> wss:&#xA;        &#x27;localhost:3000&#x27;&#xA;    )&#xA;&#xA;    ws.current.onopen = () => {&#xA;      console.log(&#x27;WebSocket Open&#x27;)&#xA;    }&#xA;&#xA;    return () => {&#xA;      ws.current.close()&#xA;    }&#xA;  }, [])&#xA;&#xA;  const startStream = () => {&#xA;    liveStream = videoRef.current.captureStream(30) // 30 FPS&#xA;    liveStreamRecorder = new MediaRecorder(liveStream, {&#xA;      mimeType: &#x27;video/webm;codecs=h264&#x27;,&#xA;      videoBitsPerSecond: 3 * 1024 * 1024,&#xA;    })&#xA;    liveStreamRecorder.ondataavailable = (e) => {&#xA;      ws.current.send(e.data)&#xA;      console.log(&#x27;send data&#x27;, e.data)&#xA;    }&#xA;    // Start recording, and dump data every second&#xA;    liveStreamRecorder.start(1000)&#xA;  }&#xA;&#xA;  const stopStream = () => {&#xA;    liveStreamRecorder.stop()&#xA;    ws.current.close()&#xA;  }&#xA;&#xA;  const toggleMute = () => {&#xA;    setMute(!mute)&#xA;  }&#xA;&#xA;  return (&#xA;    &lt;>&#xA;      <navbar></navbar>&#xA;      <div style="{{" classname="&#x27;main&#x27;">&#xA;        <div>&#xA;          &#xA;        </div>&#xA;        <div classname="&#x27;button-container&#x27;">&#xA;          <button>Go Live</button>&#xA;          <button>Stop Recording</button>&#xA;          <button>Share Screen</button>&#xA;          <button>Mute</button>&#xA;        </div>&#xA;      </div>&#xA;    >&#xA;  )&#xA;}&#xA;&#xA;const useUserMedia = (requestedMedia) => {&#xA;  const [mediaStream, setMediaStream] = useState(null)&#xA;&#xA;  useEffect(() => {&#xA;    async function enableStream() {&#xA;      try {&#xA;        const stream = await navigator.mediaDevices.getUserMedia(requestedMedia)&#xA;        setMediaStream(stream)&#xA;      } catch (err) {&#xA;        console.log(err)&#xA;      }&#xA;    }&#xA;&#xA;    if (!mediaStream) {&#xA;      enableStream()&#xA;    } else {&#xA;      return function cleanup() {&#xA;        mediaStream.getVideoTracks().forEach((track) => {&#xA;          track.stop()&#xA;        })&#xA;      }&#xA;    }&#xA;  }, [mediaStream, requestedMedia])&#xA;&#xA;  return mediaStream&#xA;}&#xA;&#xA;export default Dashboard&#xA;