Recherche avancée

Sur d’autres sites (10079)

• How to minimize latency in ffmpeg stream Java ?

  13 juillet 2022, par Taavi Sõerd

  I need to stream ffmpeg video feed in android studio and need minimal latency. Code below has achieved that when playing on galaxy s21 ultra but when I play it on galaxy tab then it's like in slow motion. When i set buffer size to 0 I get minimal latency but can't actually even see the video as it's all corrupted (all gray and colored noise).

  


  public class Decode implements Runnable {
public Activity activity;
AVFrame pFrameRGB;
SwsContext sws_ctx;
ByteBuffer bitmapBuffer;
Bitmap bmp;
byte[] array;
int imageViewWidth = 0;
int imageViewHeight = 0;
boolean imageChanged = true;
int v_stream_idx = -1;
int klv_stream_idx = -1;

boolean imageDrawMutex = false;

boolean imageIsSet = false;
ImageView imageView =  MainActivity.getmInstanceActivity().findViewById(R.id.imageView);

String mFilename = "udp://@" + MainActivity.connectionIP;;
UasDatalinkLocalSet mLatestDls;

public Decode(Activity _activity) {
    this.activity = _activity;
}

public void create_decoder(AVCodecContext codec_ctx) {
    imageChanged = true;

    // Determine required buffer size and allocate buffer
    int numBytes =av_image_get_buffer_size(AV_PIX_FMT_RGBA, codec_ctx.width(),
            codec_ctx.height(), 1);
    BytePointer buffer = new BytePointer(av_malloc(numBytes));

    bmp = Bitmap.createBitmap(codec_ctx.width(), codec_ctx.height(), Bitmap.Config.ARGB_8888);

    array = new byte[codec_ctx.width() * codec_ctx.height() * 4];
    bitmapBuffer = ByteBuffer.wrap(array);

    sws_ctx = sws_getContext(
            codec_ctx.width(),
            codec_ctx.height(),
            codec_ctx.pix_fmt(),
            codec_ctx.width(),
            codec_ctx.height(),
            AV_PIX_FMT_RGBA,
            SWS_POINT,
            null,
            null,
            (DoublePointer) null
    );

    if (sws_ctx == null) {
        Log.d("app", "Can not use sws");
        throw new IllegalStateException();
    }

    av_image_fill_arrays(pFrameRGB.data(), pFrameRGB.linesize(),
            buffer, AV_PIX_FMT_RGBA, codec_ctx.width(), codec_ctx.height(), 1);
}

@Override
public void run() {
    Log.d("app", "Start decoder");

    int ret = -1, i = 0;
    String vf_path = mFilename;

    AVFormatContext fmt_ctx = new AVFormatContext(null);
    AVPacket pkt = new AVPacket();


    AVDictionary multicastDict = new AVDictionary();

    av_dict_set(multicastDict, "rtsp_transport", "udp_multicast", 0);

    av_dict_set(multicastDict, "localaddr", getIPAddress(true), 0);
    av_dict_set(multicastDict, "reuse", "1", 0);

    av_dict_set(multicastDict, "buffer_size", "0.115M", 0);

    ret = avformat_open_input(fmt_ctx, vf_path, null, multicastDict);
    if (ret < 0) {
        Log.d("app", String.format("Open video file %s failed \n", vf_path));
        byte[] error_message = new byte[1024];
        int elen = av_strerror(ret, error_message, 1024);
        String s = new String(error_message, 0, 20);
        Log.d("app", String.format("Return: %d", ret));
        Log.d("app", String.format("Message: %s", s));
        throw new IllegalStateException();
    }
    
    if (avformat_find_stream_info(fmt_ctx, (PointerPointer) null) < 0) {
        //System.exit(-1);
        Log.d("app", "Stream info not found");
    }


    avformat.av_dump_format(fmt_ctx, 0, mFilename, 0);

    int nstreams = fmt_ctx.nb_streams();

    for (i = 0; i < fmt_ctx.nb_streams(); i++) {
        if (fmt_ctx.streams(i).codecpar().codec_type() == AVMEDIA_TYPE_VIDEO) {
            v_stream_idx = i;
        }
        if (fmt_ctx.streams(i).codecpar().codec_type() == AVMEDIA_TYPE_DATA) {
            klv_stream_idx = i;
        }
    }
    if (v_stream_idx == -1) {
        Log.d("app", "Cannot find video stream");
        throw new IllegalStateException();
    } else {
        Log.d("app", String.format("Video stream %d with resolution %dx%d\n", v_stream_idx,
                fmt_ctx.streams(v_stream_idx).codecpar().width(),
                fmt_ctx.streams(v_stream_idx).codecpar().height()));
    }

    AVCodecContext codec_ctx = avcodec_alloc_context3(null);
    avcodec_parameters_to_context(codec_ctx, fmt_ctx.streams(v_stream_idx).codecpar());


    AVCodec codec = avcodec_find_decoder(codec_ctx.codec_id());


    AVDictionary avDictionary = new AVDictionary();

    av_dict_set(avDictionary, "fflags", "nobuffer", 0);


    if (codec == null) {
        Log.d("app", "Unsupported codec for video file");
        throw new IllegalStateException();
    }
    ret = avcodec_open2(codec_ctx, codec, avDictionary);
    if (ret < 0) {
        Log.d("app", "Can not open codec");
        throw new IllegalStateException();
    }

    AVFrame frm = av_frame_alloc();

    // Allocate an AVFrame structure
    pFrameRGB = av_frame_alloc();
    if (pFrameRGB == null) {
        //System.exit(-1);
        Log.d("app", "unable to init pframergb");
    }

    create_decoder(codec_ctx);

    int width = codec_ctx.width();
    int height = codec_ctx.height();

    double fps = 15;
    

    while (true) {
        try {
            Thread.sleep(1);
        } catch (Exception e) {

        }

        try {
            if (av_read_frame(fmt_ctx, pkt) >= 0) {
                if (pkt.stream_index() == v_stream_idx) {
                    avcodec_send_packet(codec_ctx, pkt);

                    if (codec_ctx.width() != width || codec_ctx.height() != height) {
                        create_decoder(codec_ctx);
                        width = codec_ctx.width();
                        height = codec_ctx.height();
                    }
                }

                if (pkt.stream_index() == klv_stream_idx) {

                    byte[] klvDataBuffer = new byte[pkt.size()];

                    for (int j = 0; j < pkt.size(); j++) {
                        klvDataBuffer[j] = pkt.data().get(j);
                    }

                    try {
                        KLV k = new KLV(klvDataBuffer, KLV.KeyLength.SixteenBytes, KLV.LengthEncoding.BER);
                        byte[] main_payload = k.getValue();

                        // decode the Uas Datalink Local Set from main_payload binary blob.
                        mLatestDls = new UasDatalinkLocalSet(main_payload);

                        if (mLatestDls != null) {

                            MainActivity.getmInstanceActivity().runOnUiThread(new Runnable() {
                                @RequiresApi(api = Build.VERSION_CODES.Q)
                                @Override
                                public void run() {
                                    MainActivity.getmInstanceActivity().updateKlv(mLatestDls);
                                }
                            });
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    
                }

                int wasFrameDecoded = 0;
                while (wasFrameDecoded >= 0) {
                    wasFrameDecoded = avcodec_receive_frame(codec_ctx, frm);

                    if (wasFrameDecoded >= 0) {
                        // get clip fps
                        fps = 15; //av_q2d(fmt_ctx.streams(v_stream_idx).r_frame_rate());

                        sws_scale(
                                sws_ctx,
                                frm.data(),
                                frm.linesize(),
                                0,
                                codec_ctx.height(),
                                pFrameRGB.data(),
                                pFrameRGB.linesize()
                        );

                        if(!imageDrawMutex) {
                            MainActivity.getmInstanceActivity().runOnUiThread(new Runnable() {
                                @Override
                                public void run() {
                                    if (imageIsSet) {
                                        imageDrawMutex = true;
                                        pFrameRGB.data(0).position(0).get(array);
                                        bitmapBuffer.rewind();
                                        bmp.copyPixelsFromBuffer(bitmapBuffer);

                                        if (imageChanged) {
                                            (imageView).setImageBitmap(bmp);
                                            imageChanged = false;
                                        }

                                        (imageView).invalidate();
                                        imageDrawMutex = false;
                                    } else {
                                        (imageView).setImageBitmap(bmp);
                                        imageIsSet = true;
                                    }
                                }
                            });
                        }
                    }
                }
                av_packet_unref(pkt);

            }
        } catch (Exception e) {
            e.printStackTrace();
        }

        if (false) {
            Log.d("threads", "false");

            av_frame_free(frm);

            avcodec_close(codec_ctx);
            avcodec_free_context(codec_ctx);

            avformat_close_input(fmt_ctx);
        }
    }
}


  


  This code is running in Android Studio with Java. I'm quite new on this topic so not really sure even where to start.
What could be the cause of that ?

  


• Looking for a general purpose way to merge audio and video file and make an output have stereo audio with FFmpeg

  21 février 2020, par Pineapple Joe

  So have have the following FFmpeg command

  ffmpeg -i vidab.mp4 -i recab.webm  -filter_complex "[0:a]volume=0.2,apad[A];[1:a][A]amerge[Aout]" -map 0:v -map [Aout] -y mergeab.mp4

  When the input files are played separately their audio is played equally through both headphones.

  But after being merged the audio from the video is on the right side and the audio from the webm file is on the left side.

  I see this in the output when I run the cmd

  [Parsed_amerge_2 @ 0x7fa0faf01bc0] No channel layout for input 1
  
  [Parsed_amerge_2 @ 0x7fa0faf01bc0] Input channel layouts overlap: output layout will be determined by the number of distinct input channels

  How can I adjust this so that the output file has the audio from both the inputs in equally in the left and right sides ?

  Also is there a generic way to do this such that even different configurations (video has stereo and audio has mono ... or both have stereo) result in the Audio being split evenly.

  I tried using the pan filter but there is something I am not understanding about how it works because either nothing happens or I flip left and right. But I am not getting even stereo.

  fwiw here is the full output.

  ffmpeg -i vidab.mp4 -i recab.webm  -filter_complex "[0:a]volume=0.2,apad[A];[1:a][A]amerge[Aout]" -map 0:v -map [Aout] -y mergeab.mp4
  
  ffmpeg version 4.2.1 Copyright (c) 2000-2019 the FFmpeg developers
  
    built with Apple clang version 11.0.0 (clang-1100.0.33.8)
  
    configuration: --prefix=/usr/local/Cellar/ffmpeg/4.2.1_2 --enable-shared --enable-pthreads --enable-version3 --enable-avresample --cc=clang --host-cflags='-I/Library/Java/JavaVirtualMachines/adoptopenjdk-13.jdk/Contents/Home/include -I/Library/Java/JavaVirtualMachines/adoptopenjdk-13.jdk/Contents/Home/include/darwin -fno-stack-check' --host-ldflags= --enable-ffplay --enable-gnutls --enable-gpl --enable-libaom --enable-libbluray --enable-libmp3lame --enable-libopus --enable-librubberband --enable-libsnappy --enable-libtesseract --enable-libtheora --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libx265 --enable-libxvid --enable-lzma --enable-libfontconfig --enable-libfreetype --enable-frei0r --enable-libass --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librtmp --enable-libspeex --enable-libsoxr --enable-videotoolbox --disable-libjack --disable-indev=jack
  
    libavutil      56. 31.100 / 56. 31.100
  
    libavcodec     58. 54.100 / 58. 54.100
  
    libavformat    58. 29.100 / 58. 29.100
  
    libavdevice    58.  8.100 / 58.  8.100
  
    libavfilter     7. 57.100 /  7. 57.100
  
    libavresample   4.  0.  0 /  4.  0.  0
  
    libswscale      5.  5.100 /  5.  5.100
  
    libswresample   3.  5.100 /  3.  5.100
  
    libpostproc    55.  5.100 / 55.  5.100
  
  Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'vidab.mp4':
  
    Metadata:
  
      major_brand     : qt  
  
      minor_version   : 0
  
      compatible_brands: qt  
  
      creation_time   : 2020-02-21T00:35:25.000000Z
  
    Duration: 00:00:10.65, start: 0.000000, bitrate: 2539 kb/s
  
      Stream #0:0(und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, mono, fltp, 96 kb/s (default)
  
      Metadata:
  
        creation_time   : 2020-02-21T00:35:25.000000Z
  
        handler_name    : Core Media Audio
  
      Stream #0:1(und): Video: h264 (High) (avc1 / 0x31637661), yuvj420p(pc, bt709), 1280x640, 2436 kb/s, 29.87 fps, 29.97 tbr, 90k tbn, 180k tbc (default)
  
      Metadata:
  
        creation_time   : 2020-02-21T00:35:25.000000Z
  
        handler_name    : Core Media Video
  
        encoder         : H.264
  
  Input #1, matroska,webm, from 'recab.webm':
  
    Metadata:
  
      encoder         : opus-media-recorder
  
    Duration: N/A, start: 0.000000, bitrate: N/A
  
      Stream #1:0(eng): Audio: opus, 48000 Hz, mono, fltp (default)
  
  Stream mapping:
  
    Stream #0:0 (aac) -> volume (graph 0)
  
    Stream #1:0 (opus) -> amerge:in0 (graph 0)
  
    Stream #0:1 -> #0:0 (h264 (native) -> h264 (libx264))
  
    amerge (graph 0) -> Stream #0:1 (aac)
  
  Press [q] to stop, [?] for help
  
  [libx264 @ 0x7fa0fb004000] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2
  
  [libx264 @ 0x7fa0fb004000] profile High, level 3.1
  
  [libx264 @ 0x7fa0fb004000] 264 - core 155 r2917 0a84d98 - H.264/MPEG-4 AVC codec - Copyleft 2003-2018 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=12 lookahead_threads=2 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
  
  [Parsed_amerge_2 @ 0x7fa0faf01bc0] No channel layout for input 1
  
  [Parsed_amerge_2 @ 0x7fa0faf01bc0] Input channel layouts overlap: output layout will be determined by the number of distinct input channels
  
  Output #0, mp4, to 'mergeab.mp4':
  
    Metadata:
  
      major_brand     : qt  
  
      minor_version   : 0
  
      compatible_brands: qt  
  
      encoder         : Lavf58.29.100
  
      Stream #0:0(und): Video: h264 (libx264) (avc1 / 0x31637661), yuvj420p(pc, progressive), 1280x640, q=-1--1, 29.97 fps, 30k tbn, 29.97 tbc (default)
  
      Metadata:
  
        creation_time   : 2020-02-21T00:35:25.000000Z
  
        handler_name    : Core Media Video
  
        encoder         : Lavc58.54.100 libx264
  
      Side data:
  
        cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: -1
  
      Stream #0:1: Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo, fltp, 128 kb/s (default)
  
      Metadata:
  
        encoder         : Lavc58.54.100 aac
  
  frame=  318 fps=107 q=-1.0 Lsize=    1568kB time=00:00:17.72 bitrate= 724.8kbits/s speed=5.99x    
  
  video:1282kB audio:272kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.902630%
  
  [libx264 @ 0x7fa0fb004000] frame I:2     Avg QP:19.72  size: 51154
  
  [libx264 @ 0x7fa0fb004000] frame P:148   Avg QP:21.28  size:  6644
  
  [libx264 @ 0x7fa0fb004000] frame B:168   Avg QP:25.18  size:  1351
  
  [libx264 @ 0x7fa0fb004000] consecutive B-frames:  1.3% 84.9%  0.0% 13.8%
  
  [libx264 @ 0x7fa0fb004000] mb I  I16..4: 23.2% 49.3% 27.5%
  
  [libx264 @ 0x7fa0fb004000] mb P  I16..4:  2.1%  2.0%  0.2%  P16..4: 29.0%  5.1%  4.8%  0.0%  0.0%    skip:56.8%
  
  [libx264 @ 0x7fa0fb004000] mb B  I16..4:  0.5%  0.3%  0.0%  B16..8: 16.7%  0.8%  0.2%  direct: 6.1%  skip:75.4%  L0:24.8% L1:71.1% BI: 4.1%
  
  [libx264 @ 0x7fa0fb004000] 8x8 transform intra:45.4% inter:74.3%
  
  [libx264 @ 0x7fa0fb004000] coded y,uvDC,uvAC intra: 29.3% 51.4% 11.6% inter: 5.7% 16.6% 0.3%
  
  [libx264 @ 0x7fa0fb004000] i16 v,h,dc,p: 11% 53% 10% 26%
  
  [libx264 @ 0x7fa0fb004000] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 13% 36% 35%  2%  2%  2%  4%  2%  3%
  
  [libx264 @ 0x7fa0fb004000] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 24% 30% 12%  4%  6%  5%  7%  5%  6%
  
  [libx264 @ 0x7fa0fb004000] i8c dc,h,v,p: 47% 33% 17%  4%
  
  [libx264 @ 0x7fa0fb004000] Weighted P-Frames: Y:0.0% UV:0.0%
  
  [libx264 @ 0x7fa0fb004000] ref P L0: 76.6%  7.6% 11.1%  4.7%
  
  [libx264 @ 0x7fa0fb004000] ref B L0: 89.1% 10.3%  0.6%
  
  [libx264 @ 0x7fa0fb004000] ref B L1: 99.7%  0.3%
  
  [libx264 @ 0x7fa0fb004000] kb/s:989.60
  
  [aac @ 0x7fa0fb005800] Qavg: 13376.354

• In Flutter, how to get image pixel

  12 janvier 2024, par Pianone

  my code here

  


  var response = await Dio().get(
   url,
   options: Options(responseType: ResponseType.bytes)
);
Uint8List? srcImage = Uint8List.fromList(response.data);
Uint8List? watermark = await captureWaterMark();
Image i = Image.memory(srcImage!);
//how can I get the pixel (Image i) such like 1920*1080 or just width/hight pixel
...tell me how to do...
srcImage = await addWaterMarkByFfmpegCommand(srcImage, watermark);
   final result = await ImageGallerySaver.saveImage(
      srcImage!, name: name,
   );


  


  i need get the pic pixel so that i can use it in ffmpeg command, it a func that add a watermark into srcImage, but cause their pixel ratio too diff to adapted watermark

  


  i try to get pixel from ffmpeg... but i failed

  


  /// addWaterMark by using ffmpeg Command
Future addWaterMarkByFfmpegCommand(Uint8List srcImg, Uint8List watermark) async {
  try {
    final Directory tempDir = await Directory.systemTemp.createTemp();
    final File image1File = File('${tempDir.path}/srcImg.jpg');
    await image1File.writeAsBytes(srcImg);
    final File image2File = File('${tempDir.path}/watermark.png');
    await image2File.writeAsBytes(watermark);

    final String outputFilePath = '${tempDir.path}/output.jpg';
    //when i get srcImage pixel, the positions in bold(iw*1) in the following commands will be replaced to make watermark adapted
    final String command =
        '-i ${image1File.path} -i ${image2File.path} -filter_complex "[1:v]scale=**iw*1**:-1[v1];[0:v][v1]overlay=10:10" -frames:v 1 $outputFilePath';
    await FFmpegKit.execute(command);

    final File outputFile = File(outputFilePath);
    final Uint8List outputBytes = await outputFile.readAsBytes();
    return outputBytes;
  } catch (e) {
    print('Error executing ffmpeg command: $e');
  }
  return null;
}


  


  ps : i am new to flutter and ffmpeg, plz help me, I'd appreciate, thanks alot