Recherche avancée

Sur d’autres sites (6490)

• Unity : Converting Texture2D to YUV420P and sending with UDP using FFmpeg

  22 juin 2018, par potu1304

  In my Unity game each frame is rendered into a texture and then put together into a video using FFmpeg. Now my questions is if I am doing this right because avcodec_send_frame throws every time an exception.
  I am pretty sure that I am doing something wrong or in the wrong order or simply missing something.

  Here is the code for capturing the texture :

  void Update() {
  
          //StartCoroutine(CaptureFrame());
  
  
  
          if (rt == null)
  
          {
  
              rect = new Rect(0, 0, captureWidth, captureHeight);
  
              rt = new RenderTexture(captureWidth, captureHeight, 24);
  
              frame = new Texture2D(captureWidth, captureHeight, TextureFormat.RGB24, false);
  
          }
  
  
  
          Camera camera = this.GetComponent<camera>(); // NOTE: added because there was no reference to camera in original script; must add this script to Camera
  
          camera.targetTexture = rt;
  
          camera.Render();
  
  
  
          RenderTexture.active = rt;
  
          frame.ReadPixels(rect, 0, 0);
  
          frame.Apply();
  
  
  
          camera.targetTexture = null;
  
          RenderTexture.active = null;
  
  
  
          byte[] fileData = null;
  
          fileData = frame.GetRawTextureData();
  
          encoding(fileData, fileData.Length);
  
  
  
      }
  
  </camera>

  And here is the code for encoding and sending the byte data :

  private unsafe void encoding(byte[] bytes, int size)
  
      {
  
          Debug.Log("Encoding...");
  
          AVCodec* codec;
  
          codec = ffmpeg.avcodec_find_encoder(AVCodecID.AV_CODEC_ID_H264);
  
          int ret, got_output = 0;
  
  
  
          AVCodecContext* codecContext = null;
  
          codecContext = ffmpeg.avcodec_alloc_context3(codec);
  
          codecContext->bit_rate = 400000;
  
          codecContext->width = captureWidth;
  
          codecContext->height = captureHeight;
  
          //codecContext->time_base.den = 25;
  
          //codecContext->time_base.num = 1;
  
  
  
          AVRational timeBase = new AVRational();
  
          timeBase.num = 1;
  
          timeBase.den = 25;
  
          codecContext->time_base = timeBase;
  
          //AVStream* videoAVStream = null;
  
          //videoAVStream->time_base = timeBase;
  
  
  
  
  
  
  
          AVRational frameRate = new AVRational();
  
          frameRate.num = 25;
  
          frameRate.den = 1;
  
          codecContext->framerate = frameRate;
  
  
  
          codecContext->gop_size = 10;
  
          codecContext->max_b_frames = 1;
  
          codecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
  
  
  
          AVFrame* inputFrame;
  
          inputFrame = ffmpeg.av_frame_alloc();
  
          inputFrame->format = (int)codecContext->pix_fmt;
  
          inputFrame->width = captureWidth;
  
          inputFrame->height = captureHeight;
  
          inputFrame->linesize[0] = inputFrame->width;
  
  
  
          AVPixelFormat dst_pix_fmt = AVPixelFormat.AV_PIX_FMT_YUV420P, src_pix_fmt = AVPixelFormat.AV_PIX_FMT_RGBA;
  
          int src_w = 1920, src_h = 1080, dst_w = 1920, dst_h = 1080;
  
          SwsContext* sws_ctx;
  
  
  
          GCHandle pinned = GCHandle.Alloc(bytes, GCHandleType.Pinned);
  
          IntPtr address = pinned.AddrOfPinnedObject();
  
  
  
          sbyte** inputData = (sbyte**)address;
  
          sws_ctx = ffmpeg.sws_getContext(src_w, src_h, src_pix_fmt,
  
                               dst_w, dst_h, dst_pix_fmt,
  
                               0, null, null, null);
  
  
  
          fixed (int* lineSize = new int[1])
  
          {
  
              lineSize[0] = 4 * captureHeight;
  
              // Convert RGBA to YUV420P
  
              ffmpeg.sws_scale(sws_ctx, inputData, lineSize, 0, codecContext->width, inputFrame->extended_data, inputFrame->linesize);
  
          }
  
  
  
          inputFrame->pts = counter++;
  
  
  
          if (ffmpeg.avcodec_send_frame(codecContext, inputFrame) < 0)
  
              throw new ApplicationException("Error sending a frame for encoding!");
  
  
  
          AVPacket pkt;
  
          pkt = new AVPacket();
  
          //pkt.data = inData;
  
          AVPacket* packet = &pkt;
  
          ffmpeg.av_init_packet(packet);
  
  
  
          Debug.Log("pkt.size " + pkt.size);
  
          pinned.Free();
  
          AVDictionary* options = null;
  
          ffmpeg.av_dict_set(&options, "pkt_size", "1300", 0);
  
          ffmpeg.av_dict_set(&options, "buffer_size", "65535", 0);
  
          AVIOContext* server = null;
  
          ffmpeg.avio_open2(&server, "udp://192.168.0.1:1111", ffmpeg.AVIO_FLAG_WRITE, null, &options);
  
          Debug.Log("encoded");
  
          ret = ffmpeg.avcodec_encode_video2(codecContext, &pkt, inputFrame, &got_output);
  
          ffmpeg.avio_write(server, pkt.data, pkt.size);
  
          ffmpeg.av_free_packet(&pkt);
  
          pkt.data = null;
  
          pkt.size = 0;
  
      }

  And every time I start the game

    if (ffmpeg.avcodec_send_frame(codecContext, inputFrame) < 0)
  
              throw new ApplicationException("Error sending a frame for encoding!");

  throws the exception.
  Any help in fixing the issue would be greatly appreciated :)

• Revisiting Nosefart and Discovering GME

  30 mai 2011, par Multimedia Mike — Game Hacking

  I found the following screenshot buried deep in an old directory structure of mine :

  
  
  

  I tried to recall how this screenshot came to exist. Had I actually created a functional KDE frontend to Nosefart yet neglected to release it ? I think it’s more likely that I used some designer tool (possibly KDevelop) to prototype a frontend. This would have been sometime in 2000.

  However, this screenshot prompted me to revisit Nosefart.

  Nosefart Background
  Nosefart is a program that can play Nintendo Sound Format (NSF) files. NSF files are files containing components that were surgically separated from Nintendo Entertainment System (NES) ROM dumps. These components contain the music playback engines for various games. An NSF player is a stripped down emulation system that can simulate the NES6502 CPU along with the custom hardware (2 square waves, 1 triangle wave, 1 noise generator, and 1 limited digital channel).

  Nosefart was written by Matt Conte and eventually imported into a Sourceforge project, though it has not seen any development since then. The distribution contains standalone command line players for Linux and DOS, a GTK frontend for the Linux command line version, and plugins for Winamp, XMMS, and CL-Amp.

  The Sourceforge project page notes that Nosefart is also part of XBMC. Let the record show that Nosefart is also incorporated into xine (I did that in 2002, I think).

  Upgrading the API
  When I tried running the command line version of Nosefart under Linux, I hit hard against the legacy audio API : OSS. Remember that ?

  In fairly short order, I was able to upgrade the CL program to use PulseAudio. The program is not especially sophisticated. It’s a single-threaded affair which checks for a keypress, processes an audio frame, and sends the frame out to the OSS file interface. All that was needed was to rewrite open_hardware() and close_hardware() for PA and then replace the write statement in play(). The only quirk that stood out is that including <pulse/pulseaudio.h> is insufficient for programming PA’s simple API. <pulse/simple.h> must be included separately.

  For extra credit, I adapted the program to ALSA. The program uses the most simplistic audio output API possible — just keep filling a buffer and sending it out to the DAC.

  Discovering GME
  I’m not sure what to do with the the program now since, during my research to attempt to bring Nosefart up to date, I became aware of a software library named Game Music Emu, or GME. It’s a pure C++ library that can essentially play any classic video game format you can possible name. Wow. A lot can happen in 10 years when you’re not paying attention.

  It’s such a well-written library that I didn’t need any tutorial or documentation to come up to speed. Just a quick read of the main gme.h header library enabled me in short order to whip up a quick C program that could play NSF and SPC files. Path of least resistance : Client program asks library to open a hardcoded file, synthesize 10 seconds of audio, and dump it into a file ; ask the FLAC command line program to transcode raw data to .flac file ; use ffplay to verify the results.

  I might develop some other uses for this library.

• How do I sync 4 videos in a grid to play the same frame at the same time ?

  28 décembre 2022, par PirateApp
  &#xA;
  • 4 of us have recorded ourselves playing a game and want to create a 4 x 4 video grid

  &#xA;

  • The game has cutscenes at the beginning followed by each person having their unique part for the rest of the video

  &#xA;

  • I am looking to synchronize the grid such that it starts at the same place in the cutscene for everyone

  &#xA;

  • Kindly take a look at what is happening currently. The cutscene is off by a few seconds for everyone

  &#xA;

  • Imagine a time offset a,b,c,d such that when I add this offet to each video, the entire video grid will be in sync

  &#xA;

  • How to find this a,b,c,d and more importantly how to add it in filter_complex

  &#xA;

  &#xA;

  I used the ffmpeg command below to generate a 4 x 4 video grid and it seems to work

  &#xA;

  ffmpeg
    -i nano_prologue.mkv -i macko_nimble_guardian.mkv -i nano_nimble_guardian.mkv -i ghost_nimble_guardian_subtle_arrow_1.mp4
    -filter_complex "
        nullsrc=size=1920x1080 [base];
        [0:v] setpts=PTS-STARTPTS, scale=960x540 [upperleft];
        [1:v] setpts=PTS-STARTPTS, scale=960x540 [upperright];
        [2:v] setpts=PTS-STARTPTS, scale=960x540 [lowerleft];
        [3:v] setpts=PTS-STARTPTS, scale=960x540 [lowerright];
        [base][upperleft] overlay=shortest=1 [tmp1];
        [tmp1][upperright] overlay=shortest=1:x=960 [tmp2];
        [tmp2][lowerleft] overlay=shortest=1:y=540 [tmp3];
        [tmp3][lowerright] overlay=shortest=1:x=960:y=540
    "
    -c:v libx264 output.mkv


  &#xA;

  My problem though is that since each of us starts recording at slightly different times, the cutscenes are out of sync

  &#xA;

  As per the screenshot below, you can see that each video has the same scene starting at a slightly different time.

  &#xA;

  Is there a way to find where the same frame will start on all videos and then sync each video to start from that frame or 20 seconds before that frame ?

  &#xA;

  

  &#xA;

  UPDATE 1

  &#xA;

  i have figured out the offset for each video in millisecond precision using the following technique

  &#xA;

  take a screenshot of the first video at a particular point in the cutscene and save image as png and run the script below for the remaining 3 videos to find out where this screenshot appears in each video


ffmpeg -i "video2.mp4" -r 1 -loop 1 -i screenshot.png -an -filter_complex "blend=difference:shortest=1,blackframe=90:32" -f null -


  &#xA;

  Use the command above to search for the offset in every video for that cutscene

  &#xA;

  It gave me this

  &#xA;

  VIDEO 3 OFFSET

  &#xA;

  [Parsed_blackframe_1 @ 0x600003af00b0] frame:3144 pblack:92 pts:804861 t:52.399805 type:P last_keyframe:3120

[Parsed_blackframe_1 @ 0x600003af00b0] frame:3145 pblack:96 pts:805117 t:52.416471 type:P last_keyframe:3120


  &#xA;

  VIDEO 2 OFFSET

  &#xA;

  [Parsed_blackframe_1 @ 0x6000014dc0b0] frame:3629 pblack:91 pts:60483 t:60.483000 type:P last_keyframe:3500


  &#xA;

  VIDEO 4 OFFSET

  &#xA;

  [Parsed_blackframe_1 @ 0x600002f84160] frame:2885 pblack:93 pts:48083 t:48.083000 type:P last_keyframe:2880

[Parsed_blackframe_1 @ 0x600002f84160] frame:2886 pblack:96 pts:48100 t:48.100000 type:P last_keyframe:2880


  &#xA;

  Now how do I use filter_complex to say start each video at either the frame above or the timestamp above ?. I would like to include say 10 seconds before the above frame in each video so that it starts from the beginning

  &#xA;

  UPDATE 2

  &#xA;

  This command currently gives me a 100% synced video, how do I make it start 15 seconds before the specified frame numbers and how to make it use the audio track from video 2 instead ?

  &#xA;

  ffmpeg
    -i v_nimble_guardian.mkv -i macko_nimble_guardian.mkv -i ghost_nimble_guardian_subtle_arrow_1.mp4 -i nano_nimble_guardian.mkv
    -filter_complex "
        nullsrc=size=1920x1080 [base];
        [0:v] trim=start_pts=49117,setpts=PTS-STARTPTS, scale=960x540 [upperleft];
        [1:v] trim=start_pts=50483,setpts=PTS-STARTPTS, scale=960x540 [upperright];
        [2:v] trim=start_pts=795117,setpts=PTS-STARTPTS, scale=960x540 [lowerleft];
        [3:v] trim=start_pts=38100,setpts=PTS-STARTPTS, scale=960x540 [lowerright];
        [base][upperleft] overlay=shortest=1 [tmp1];
        [tmp1][upperright] overlay=shortest=1:x=960 [tmp2];
        [tmp2][lowerleft] overlay=shortest=1:y=540 [tmp3];
        [tmp3][lowerright] overlay=shortest=1:x=960:y=540
    "
    -c:v libx264 output.mkv


  &#xA;