Recherche avancée
Autres articles (64)
-
Les autorisations surchargées par les plugins
27 avril 2010, parMediaspip core
autoriser_auteur_modifier() afin que les visiteurs soient capables de modifier leurs informations sur la page d’auteurs -
MediaSPIP Core : La Configuration
9 novembre 2010, parMediaSPIP Core fournit par défaut trois pages différentes de configuration (ces pages utilisent le plugin de configuration CFG pour fonctionner) : une page spécifique à la configuration générale du squelettes ; une page spécifique à la configuration de la page d’accueil du site ; une page spécifique à la configuration des secteurs ;
Il fournit également une page supplémentaire qui n’apparait que lorsque certains plugins sont activés permettant de contrôler l’affichage et les fonctionnalités spécifiques (...) -
Supporting all media types
13 avril 2011, parUnlike most software and media-sharing platforms, MediaSPIP aims to manage as many different media types as possible. The following are just a few examples from an ever-expanding list of supported formats : images : png, gif, jpg, bmp and more audio : MP3, Ogg, Wav and more video : AVI, MP4, OGV, mpg, mov, wmv and more text, code and other data : OpenOffice, Microsoft Office (Word, PowerPoint, Excel), web (html, CSS), LaTeX, Google Earth and (...)
Sur d’autres sites (12795)
-
How can I get decoded frames, which I sent into GPU, can i get them from GPU in rgb
21 mai 2024, par Владислав СапожникI use ffmpeg.av_hwframe_transfer_data to sent decoded frames into GPU, but i can not get them again in another good format. I try to change my shaders, use av_hwframe_transfer_get_formats but it is not working !dfghkdsfiuhsgiherghoeirughoighweroigjoiwejgoiwrjgjeoijgoiewgoheroighoieqfgoihqeoigheiogieiqrhgihergh2eouirghou2rerhg
My code :


{
 private static bool _readingComplete = false;
 private static bool _decodingComplete = false;
 private static readonly object _lock = new object();
 private static Queue<avpacket> packets = new Queue<avpacket>();
 private static readonly object _fileLock = new object();
 public static MyWindow myWindow;
 public static SKBitmap myBitmap;

 public static async unsafe Task Main(string[] args)
 {
 FFmpegBinariesHelper.RegisterFFmpegBinaries();
 DynamicallyLoadedBindings.Initialize();
 Console.WriteLine($"FFmpeg version info: {ffmpeg.av_version_info()}");

 Directory.Delete("frames", true);
 Directory.CreateDirectory("frames");

 var url = "rtsp://admin:123456@192.168.1.12:554/stream0?username=admin&password=E10ADC3949BA59ABBE56E057F20";

 AVDictionary* opts = null;
 ffmpeg.av_dict_set(&opts, "-rtsp_transport", "tcp", 0);

 var vsr = new VideoStreamReader(url, opts);
 var vsd = new VideoStreamDecoder(*vsr.GetCodecParameters(), AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA);

 Task readerTask = Task.Factory.StartNew(() => ReadPackets(vsr), TaskCreationOptions.LongRunning);
 Task decoderTask = Task.Factory.StartNew(() => DecodeFrames(vsd), TaskCreationOptions.LongRunning);

 var nativeWindowSettings = new NativeWindowSettings()
 {
 ClientSize = new Vector2i(800, 600),
 Title = "My first OpenTK program!"
 };

 using (var myWindow = new MyWindow(GameWindowSettings.Default, nativeWindowSettings))
 {
 myWindow.Run();
 }
 }

 private static unsafe void ReadPackets(VideoStreamReader vsr)
 {
 while (!_readingComplete)
 {
 vsr.TryReadNextPacket(out var packet);
 lock (_lock)
 {
 packets.Enqueue(packet);
 }
 }

 _readingComplete = true;
 }

 private static unsafe void DecodeFrames(VideoStreamDecoder vsd)
 {

 Console.WriteLine($"codec name: {vsd.CodecName}");

 //var sourceSize = vsd.FrameSize;
 //var sourcePixelFormat = vsd.PixelFormat;
 //var destinationSize = sourceSize;
 //var destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_RGBA;
 //using var vfc = new VideoFrameConverter(sourceSize, sourcePixelFormat, destinationSize, destinationPixelFormat);

 var frameNumber = 0;

 while (true)
 {
 AVPacket packet;
 lock (_lock)
 {
 if (packets.Count == 0)
 {
 if (_readingComplete)
 {
 break;
 }
 else
 {
 continue;
 }
 }
 packet = packets.Dequeue();
 }

 vsd.TryDecodeNextFrame(out var frame, packet);
 //var convertedFrame = vfc.Convert(frame);

 //var bitmap = new SKBitmap(convertedFrame.width, convertedFrame.height, SKColorType.Bgra8888, SKAlphaType.Opaque);
 //bitmap.InstallPixels(new SKImageInfo(convertedFrame.width, convertedFrame.height, SKColorType.Bgra8888, SKAlphaType.Opaque), (IntPtr)convertedFrame.data[0]);
 //myBitmap = bitmap;
 var bitmap = new SKBitmap(frame.width, frame.height, SKColorType.Bgra8888, SKAlphaType.Opaque);
 bitmap.InstallPixels(new SKImageInfo(frame.width, frame.height, SKColorType.Bgra8888, SKAlphaType.Opaque), (IntPtr)frame.data[0]);
 myBitmap = bitmap;
 
 Console.WriteLine($"frame: {frameNumber}");
 frameNumber++;
 }

 _decodingComplete = true;
 }

 //private static unsafe void WriteFrame(AVFrame convertedFrame, int frameNumber)
 //{
 // var imageInfo = new SKImageInfo(convertedFrame.width, convertedFrame.height, SKColorType.Bgra8888, SKAlphaType.Opaque);
 // using var bitmap = new SKBitmap();
 // bitmap.InstallPixels(imageInfo, (IntPtr)convertedFrame.data[0]);

 // string filePath;
 // lock (_fileLock)
 // {
 // filePath = $"frames/frame.{frameNumber:D8}.jpg";
 // }

 // using var stream = File.Create(filePath);
 // bitmap.Encode(stream, SKEncodedImageFormat.Jpeg, 90);
 //}
}
using OpenTK.Graphics.OpenGL4;
using OpenTK.Mathematics;
using OpenTK.Windowing.Common;
using OpenTK.Windowing.Desktop;
using OpenTK.Windowing.GraphicsLibraryFramework;
using SkiaSharp;

namespace OpenTKTask;

public class MyWindow : GameWindow
{
 private Shader shader;
 private int vertexBufferHandle;
 private int elementBufferHandle;
 private int vertexArrayHandle;
 private int texture;

 //float[] vertices =
 //{
 // 1.0f, 1.0f, 0.0f, 1.0f, 0.0f,
 // 1.0f, -1.0f, 0.0f, 1.0f, 1.0f,
 // -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
 // -1.0f, 1.0f, 0.0f, 0.0f, 1.0f
 //};

 float[] vertices =
{
 //Position | Texture coordinates
 1.0f, 1.0f, 0.0f, 1.0f, 0.0f, // top right
 1.0f, -1.0f, 0.0f, 1.0f, 1.0f, // bottom right
 -1.0f, -1.0f, 0.0f, 0.0f, 1.0f, // bottom left
 -1.0f, 1.0f, 0.0f, 0.0f, 0.0f // top left
};

 uint[] indices =
{
 0, 1, 3,
 1, 2, 3
 };

 float[] texCoords =
 {
 0.0f, 0.0f,
 1.0f, 0.0f,
 0.5f, 1.0f,
 };

 public MyWindow(GameWindowSettings gameWindowSettings, NativeWindowSettings nativeWindowSettings) : base(gameWindowSettings, nativeWindowSettings)
 {
 this.CenterWindow(new Vector2i(1280, 760));
 }

 protected override void OnResize(ResizeEventArgs e)
 {
 GL.Viewport(0, 0, e.Width, e.Height);
 base.OnResize(e);
 }

 protected override void OnLoad()
 {
 base.OnLoad();

 shader = new Shader("C:\\Users\\1\\Desktop\\7h3_C0d3r\\OpenTKTask\\vertexShader.vert", "C:\\Users\\1\\Desktop\\7h3_C0d3r\\OpenTKTask\\fragShader.frag");
 shader.Use();

 vertexArrayHandle = GL.GenVertexArray();
 GL.BindVertexArray(vertexArrayHandle);

 vertexBufferHandle = GL.GenBuffer();
 GL.BindBuffer(BufferTarget.ArrayBuffer, vertexBufferHandle);
 GL.BufferData(BufferTarget.ArrayBuffer, vertices.Length * sizeof(float), vertices, BufferUsageHint.StaticDraw);

 elementBufferHandle = GL.GenBuffer();
 GL.BindBuffer(BufferTarget.ElementArrayBuffer, elementBufferHandle);
 GL.BufferData(BufferTarget.ElementArrayBuffer, indices.Length * sizeof(uint), indices, BufferUsageHint.StaticDraw);

 var positionLocation = shader.GetAttribLocation("aPosition");
 GL.VertexAttribPointer(positionLocation, 3, VertexAttribPointerType.Float, false, 5 * sizeof(float), 0);
 GL.EnableVertexAttribArray(positionLocation);

 var texCoordLocation = shader.GetAttribLocation("aTexCoord");
 GL.VertexAttribPointer(texCoordLocation, 2, VertexAttribPointerType.Float, false, 5 * sizeof(float), 3 * sizeof(float));
 GL.EnableVertexAttribArray(texCoordLocation);

 texture = GL.GenTexture();
 GL.BindTexture(TextureTarget.Texture2D, texture);

 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.Repeat);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.Repeat);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

 GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgba, 800, 600, 0, PixelFormat.Bgra, PixelType.UnsignedByte, IntPtr.Zero);
 GL.BindTexture(TextureTarget.Texture2D, 0);
 }

 protected override void OnUnload()
 {
 base.OnUnload();
 GL.DeleteBuffer(vertexBufferHandle);
 GL.DeleteVertexArray(vertexArrayHandle);
 GL.DeleteProgram(shader.Handle);
 GL.DeleteTexture(texture);
 }

 protected override void OnUpdateFrame(FrameEventArgs args)
 {
 base.OnUpdateFrame(args);
 }

 protected override void OnRenderFrame(FrameEventArgs args)
 {
 base.OnRenderFrame(args);

 UpdateTexture(Program.myBitmap);

 GL.Clear(ClearBufferMask.ColorBufferBit);

 GL.BindTexture(TextureTarget.Texture2D, texture);

 shader.Use();

 GL.BindVertexArray(vertexArrayHandle);

 GL.DrawElements(PrimitiveType.Triangles, indices.Length, DrawElementsType.UnsignedInt, 0);

 SwapBuffers();
 }

 public void UpdateTexture(SKBitmap bitmap)
 {
 GL.BindTexture(TextureTarget.Texture2D, texture);

 if (bitmap != null)
 {
 //byte[] pixels = bitmap.Bytes;
 GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgba, bitmap.Width, bitmap.Height, 0, PixelFormat.Bgra, PixelType.UnsignedByte, 0);
 }
 }

 public SKBitmap LoadBitmap(string path)
 {
 using (var stream = File.OpenRead(path))
 {
 return SKBitmap.Decode(stream);
 }
 }
}
using FFmpeg.AutoGen;
using System.Drawing;

namespace OpenTKTask;

public sealed unsafe class VideoStreamDecoder : IDisposable
{
 private readonly AVCodecContext* _pCodecContext;
 private readonly AVPacket* _pPacket;
 private readonly AVFrame* _pFrame;
 private readonly AVFrame* _receivedFrame;
 private AVFrame* _pSwFrame;
 private AVBufferRef* _pHWDeviceCtx;

 public VideoStreamDecoder(AVCodecParameters parameter, AVHWDeviceType HWDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA)
 {
 _receivedFrame = ffmpeg.av_frame_alloc();

 AVCodec* codec = ffmpeg.avcodec_find_decoder(parameter.codec_id);
 if (codec == null)
 throw new InvalidOperationException("Codec not found.");
 _pCodecContext = ffmpeg.avcodec_alloc_context3(codec);

 ffmpeg.av_hwdevice_ctx_create(&_pCodecContext->hw_device_ctx, HWDeviceType, null, null, 0).ThrowExceptionIfError();

 ffmpeg.avcodec_parameters_to_context(_pCodecContext, &parameter)
 .ThrowExceptionIfError();
 ffmpeg.avcodec_open2(_pCodecContext, codec, null).ThrowExceptionIfError();

 CodecName = ffmpeg.avcodec_get_name(codec->id);
 FrameSize = new Size(_pCodecContext->width, _pCodecContext->height);
 PixelFormat = _pCodecContext->pix_fmt;

 _pFrame = ffmpeg.av_frame_alloc();
 _pPacket = ffmpeg.av_packet_alloc();
 }

 public string CodecName { get; }
 public Size FrameSize { get; }
 public AVPixelFormat PixelFormat { get; }

 public bool TryDecodeNextFrame(out AVFrame frame, AVPacket packet)
 {
 ffmpeg.av_frame_unref(_pFrame);
 ffmpeg.av_frame_unref(_receivedFrame);
 int error;

 do
 {
 ffmpeg.avcodec_send_packet(_pCodecContext, &packet).ThrowExceptionIfError();
 error = ffmpeg.avcodec_receive_frame(_pCodecContext, _pFrame);
 } while (error == ffmpeg.AVERROR(ffmpeg.EAGAIN));

 error.ThrowExceptionIfError();

 if (_pCodecContext->hw_device_ctx != null)
 {
 ffmpeg.av_hwframe_transfer_data(_receivedFrame, _pFrame, 0).ThrowExceptionIfError();
 Console.WriteLine((AVPixelFormat)171);
 frame = *_receivedFrame; // AV_PIX_FMT_NV11
 //Console.WriteLine((AVPixelFormat)frame.format);
 }
 else
 frame = *_pFrame; // AV_PIX_FMT_NV11
 //Console.WriteLine((AVPixelFormat)frame.format);
 return true;
 }

 public void Dispose()
 {
 var pFrame = _pFrame;
 ffmpeg.av_frame_free(&pFrame);

 var pCodecContext = _pCodecContext;
 ffmpeg.avcodec_free_context(&pCodecContext);

 if (_pHWDeviceCtx != null)
 {
 var pHWDeviceCtx = _pHWDeviceCtx;
 ffmpeg.av_buffer_unref(&pHWDeviceCtx);
 }

 if (_pSwFrame != null)
 {
 var pSwFrame = _pSwFrame;
 ffmpeg.av_frame_free(&pSwFrame);
 }
 }
}````
</avpacket></avpacket>

-
Render YUV frame using OpenTK [closed]
20 mai 2024, par dima2012 terminatormy window
Im trying to render YUV AVFrame, that i get from camera using OpenTK, im creating a rectangle and trying to apply a texture to it, but it doesnt work.


Here is my window class


using OpenTK.Graphics.Egl;
using OpenTK.Graphics.OpenGL4;
using OpenTK.Windowing.Common;
using OpenTK.Windowing.Desktop;
using OpenTK.Windowing.GraphicsLibraryFramework;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace myFFmpeg
{
 public class CameraWindow : GameWindow
 {
 private int vertexBufferHandle;
 private int elementBufferHandle;
 private int vertexArrayHandle;
 private int frameNumber = 0;
 private int yTex, uTex, vTex;

 Shader shader;
 Texture texture;

 float[] vertices =
 {
 //Position | Texture coordinates
 0.5f, 0.5f, 0.0f, 1.0f, 0.0f, // top right
 0.5f, -0.5f, 0.0f, 1.0f, 1.0f, // bottom right
 -0.5f, -0.5f, 0.0f, 0.0f, 1.0f, // bottom left
 -0.5f, 0.5f, 0.0f, 0.0f, 0.0f // top left
 };


 private uint[] indices = 
 {
 0, 1, 3, // first triangle
 1, 2, 3 // second triangle
 };

 public CameraWindow(string title) : base(GameWindowSettings.Default, new NativeWindowSettings() { ClientSize = (1280, 720), Title = title }) { UpdateFrequency = 25; }

 protected override void OnUpdateFrame(FrameEventArgs e)
 {
 base.OnUpdateFrame(e);
 }

 protected override void OnLoad()
 {
 GL.ClearColor(0.5f, 0.3f, 0.3f, 1.0f);

 shader = new Shader(@"..\..\shader.vert", @"..\..\shader.frag");
 texture = new Texture();

 elementBufferHandle = GL.GenBuffer();
 GL.BindBuffer(BufferTarget.ElementArrayBuffer, elementBufferHandle);
 GL.BufferData(BufferTarget.ElementArrayBuffer, indices.Length * sizeof(uint), indices, BufferUsageHint.StaticDraw);

 vertexBufferHandle = GL.GenBuffer();
 GL.BindBuffer(BufferTarget.ArrayBuffer, vertexBufferHandle);
 GL.BufferData(BufferTarget.ArrayBuffer, vertices.Length * sizeof(float), vertices, BufferUsageHint.StaticDraw);

 GL.BindBuffer(BufferTarget.ArrayBuffer, 0);

 vertexArrayHandle = GL.GenVertexArray();
 GL.BindVertexArray(vertexArrayHandle);

 GL.BindBuffer(BufferTarget.ArrayBuffer, vertexBufferHandle);
 GL.VertexAttribPointer(0, 3, VertexAttribPointerType.Float, false, 5 * sizeof(float), 0);
 GL.EnableVertexAttribArray(0);

 int vertexShader = GL.CreateShader(ShaderType.VertexShader);
 GL.ShaderSource(vertexShader, @"..\..\shader.vert");
 GL.CompileShader(vertexShader);

 int fragmentShader = GL.CreateShader(ShaderType.FragmentShader);
 GL.ShaderSource(fragmentShader, @"..\..\shader.frag");
 GL.CompileShader(fragmentShader);

 int shaderProgram = GL.CreateProgram();
 GL.AttachShader(shaderProgram, vertexShader);
 GL.AttachShader(shaderProgram, fragmentShader);
 GL.LinkProgram(shaderProgram);


 int vertexPosLocation = GL.GetAttribLocation(shaderProgram, "vertexPos");
 GL.EnableVertexAttribArray(vertexPosLocation);
 GL.VertexAttribPointer(vertexPosLocation, 2, VertexAttribPointerType.Float, false, 4 * sizeof(float), 0);

 int texCoordLocation = GL.GetAttribLocation(shaderProgram, "texCoord");
 GL.EnableVertexAttribArray(texCoordLocation);
 GL.VertexAttribPointer(texCoordLocation, 2, VertexAttribPointerType.Float, false, 4 * sizeof(float), 2 * sizeof(float));

 GL.UseProgram(shaderProgram);

 GL.ActiveTexture(TextureUnit.Texture0);
 GL.BindTexture(TextureTarget.Texture2D, yTex);
 GL.Uniform1(GL.GetUniformLocation(shaderProgram, "yTex"), 0);

 GL.ActiveTexture(TextureUnit.Texture1);
 GL.BindTexture(TextureTarget.Texture2D, uTex);
 GL.Uniform1(GL.GetUniformLocation(shaderProgram, "uTex"), 1);

 GL.ActiveTexture(TextureUnit.Texture2);
 GL.BindTexture(TextureTarget.Texture2D, vTex);
 GL.Uniform1(GL.GetUniformLocation(shaderProgram, "vTex"), 2);

 GL.BindVertexArray(0);
 //code

 base.OnLoad();
 }

 protected override void OnUnload()
 {
 GL.BindBuffer(BufferTarget.ArrayBuffer, 0);
 GL.DeleteBuffer(vertexBufferHandle);
 GL.UseProgram(0);
 shader.Dispose();

 //code

 base.OnUnload();
 }

 protected override void OnRenderFrame(FrameEventArgs e)
 {

 GL.Clear(ClearBufferMask.ColorBufferBit);

 shader.Use();
 texture.Use(frameNumber++);

 GL.BindVertexArray(vertexArrayHandle);

 GL.DrawElements(PrimitiveType.Triangles, indices.Length, DrawElementsType.UnsignedInt, indices);

 Context.SwapBuffers();

 base.OnRenderFrame(e);
 }

 protected override void OnFramebufferResize(FramebufferResizeEventArgs e)
 {
 base.OnFramebufferResize(e);

 GL.Viewport(0, 0, e.Width, e.Height);
 }
 }
}


And my texture class :


using System;
using OpenTK;
using OpenTK.Graphics.OpenGL4;
using SkiaSharp;
using FFmpeg;
using SkiaSharp.Internals;
using StbImageSharp;
using FFmpeg.AutoGen;
using System.Threading;

namespace myFFmpeg
{
 public class Texture
 {
 int Handle, yTex, uTex, vTex;

 Program program = new Program();

 public Texture()
 {
 Handle = GL.GenTexture();
 }


 public unsafe void Use(int frameNumber)
 {
 GL.BindTexture(TextureTarget.Texture2D, Handle);

 // Generate textures only once (outside the loop)
 if (yTex == 0)
 {
 GL.GenTextures(1, out yTex);
 }
 if (uTex == 0)
 {
 GL.GenTextures(1, out uTex);
 }
 if (vTex == 0)
 {
 GL.GenTextures(1, out vTex);
 }

 // Bind textures to specific units before rendering each frame
 GL.ActiveTexture(TextureUnit.Texture0);
 GL.BindTexture(TextureTarget.Texture2D, yTex);
 GL.ActiveTexture(TextureUnit.Texture1);
 GL.BindTexture(TextureTarget.Texture2D, uTex);
 GL.ActiveTexture(TextureUnit.Texture2);

 // Update textures with new frame data from FFmpeg
 AVFrame frame = program.getFrame();
 int width = frame.width;
 int height = frame.height;

 Console.BackgroundColor = ConsoleColor.White;
 Console.ForegroundColor = ConsoleColor.Black;
 Console.WriteLine((AVPixelFormat)frame.format);
 Console.BackgroundColor = ConsoleColor.Black;


 // Assuming YUV data is stored in separate planes (Y, U, V)
 GL.BindTexture(TextureTarget.Texture2D, yTex);
 GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Luminance, width, height, 0, PixelFormat.Luminance, PixelType.UnsignedByte, (IntPtr)frame.data[0]);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

 GL.BindTexture(TextureTarget.Texture2D, uTex);
 GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Luminance, width / 2, height / 2, 0, PixelFormat.Luminance, PixelType.UnsignedByte, (IntPtr)frame.data[1]);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);

 GL.BindTexture(TextureTarget.Texture2D, vTex);
 GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Luminance, width / 2, height / 2, 0, PixelFormat.Luminance, PixelType.UnsignedByte, (IntPtr)frame.data[2]);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
 GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);

 }
 }
}



And my shader class :


using OpenTK.Graphics.OpenGL4;
using System;
using System.IO;

namespace myFFmpeg
{
 public class Shader : IDisposable
 {
 public int Handle { get; private set; }

 public Shader(string vertexPath, string fragmentPath)
 {
 string vertexShaderSource = File.ReadAllText(vertexPath);
 string fragmentShaderSource = File.ReadAllText(fragmentPath);

 int vertexShader = GL.CreateShader(ShaderType.VertexShader);
 GL.ShaderSource(vertexShader, vertexShaderSource);
 GL.CompileShader(vertexShader);
 CheckShaderCompilation(vertexShader);

 int fragmentShader = GL.CreateShader(ShaderType.FragmentShader);
 GL.ShaderSource(fragmentShader, fragmentShaderSource);
 GL.CompileShader(fragmentShader);
 CheckShaderCompilation(fragmentShader);

 Handle = GL.CreateProgram();
 GL.AttachShader(Handle, vertexShader);
 GL.AttachShader(Handle, fragmentShader);
 GL.LinkProgram(Handle);
 CheckProgramLinking(Handle);

 GL.DetachShader(Handle, vertexShader);
 GL.DetachShader(Handle, fragmentShader);
 GL.DeleteShader(vertexShader);
 GL.DeleteShader(fragmentShader);
 }

 public void Use()
 {
 GL.UseProgram(Handle);
 }

 public int GetAttribLocation(string attribName)
 {
 return GL.GetAttribLocation(Handle, attribName);
 }

 public int GetUniformLocation(string uniformName)
 {
 return GL.GetUniformLocation(Handle, uniformName);
 }

 private void CheckShaderCompilation(int shader)
 {
 GL.GetShader(shader, ShaderParameter.CompileStatus, out int success);
 if (success == 0)
 {
 string infoLog = GL.GetShaderInfoLog(shader);
 throw new InvalidOperationException($"Shader compilation failed: {infoLog}");
 }
 }

 private void CheckProgramLinking(int program)
 {
 GL.GetProgram(program, GetProgramParameterName.LinkStatus, out int success);
 if (success == 0)
 {
 string infoLog = GL.GetProgramInfoLog(program);
 throw new InvalidOperationException($"Program linking failed: {infoLog}");
 }
 }

 public void Dispose()
 {
 GL.DeleteProgram(Handle);
 }
 }
}


Vert shader


#version 330 core
layout(location = 0) in vec3 vertexPos;
layout(location = 1) in vec2 texCoord;

out vec2 TexCoord; 

void main()
{
 gl_Position = vec4(vertexPos,1.0);
 TexCoord = texCoord;
}


Frag shader


#version 330 core
in vec2 TexCoord;
out vec4 color;

uniform sampler2D yTex;
uniform sampler2D uTex;
uniform sampler2D vTex;

void main()
{
 float y = texture(yTex, TexCoord).r;
 float u = texture(uTex, TexCoord).r - 0.5;
 float v = texture(vTex, TexCoord).r - 0.5;

 // YUV to RGB conversion (BT.709)
 float r = y + 1.5714 * v;
 float g = y - 0.6486 * u - 0.3918 * v;
 float b = y + 1.8556 * u;

 color = vec4(r, g, b, 1.0);
}


I can provide more code, if needed..


I tried changing shaders, changing textures, getting frame using
ffmpeg.av_hwframe_transfer_data(_receivedFrame, _pFrame, 0);

-
Display real time frames from several RTSP streams
13 février 2024, par MraxI have this class, it uses ffmpeg library for rtsp live streaming :


#include <iostream>
#include <string>
#include <vector>
#include <mutex>

extern "C"
{
#include <libavcodec></libavcodec>avcodec.h>
#include <libavformat></libavformat>avformat.h>
#include <libavformat></libavformat>avio.h>
}

class ryMediaSource
{
public:
 ryMediaSource() {}
 ryMediaSource(const ryMediaSource& other);
 ~ryMediaSource();

 bool ryOpenMediaSource(const std::string&);

private:
 mediaSource pMediaSource;
 AVFormatContext* pFormatCtx;
 mutable std::mutex pMutex;
};
</mutex></vector></string></iostream>

And inside my main file, I have these vector of ryMediaSource and four rstp urls :


std::vector<rymediasource> mediaSources;
std::vector streams =
{
 {"rtsp://1
 {"rtsp://2
 {"rtsp://3
 {"rtsp://4
};
</rymediasource>

Creating a instance for every vector :


for (const auto& stream : streams)
{
 mediaSources.emplace_back(); // Create a new instance for each stream
}


And opening all the streams (I need to have access to all the streams, all the time).


for (size_t s = 0; s < streams.size(); s++)
{
 mediaSources[s].ryOpenMediaSource(streams[s]);
}


After all the streams are loaded, I start to display the videos all of the streams : av_read_frame(pFormatCtx, pPacket).
But I am having a gap from what is been displayed to what is really capturing from the source (IP Cameras).
From ryOpenMediaSource(streams[0]) is about 11 seconds, ryOpenMediaSource(streams[1]) about 7 seconds, ryOpenMediaSource(streams[2]) is about 4 seconds and ryOpenMediaSource(streams[3]) is real time.
I realized that the issue is on my ryOpenMediaSource code :


bool ryMediaSource::ryOpenMediaSource(const std::string& url)
{
 int rc = -1;

 pFormatCtx = avformat_alloc_context();
 if (!pFormatCtx)
 throw std::runtime_error("Failed to allocate AVFormatContext.");
 rc = avformat_open_input(&pFormatCtx, url.c_str(), NULL, NULL);
 if (rc < 0)
 {
 return false;
 }
}


My question is, why this is happening ? Why can't all streams have the same (time stamp ?) , as the last inserted in my vector of ryMediaSource ?


Should I overwrite some variable of pFormatCtx to "force" the all vector to have the (time stamp ?) as the last one ? If so, can you give me some guidance ?


Tried setting some different values on pFormatCtx after loaded with avformat_open_input(&pFormatCtx, url.c_str(), NULL, &pDicts) ; but no luck at all.


I am expecting that all streams started at the same time, even if pre loading them, for later on, transform these frames into a cv::Mat for rendering.


MRE :


Header :

#pragma once

#include <iostream>
#include <string>
#include <vector>
#include <chrono>
#include <thread>
#include <mutex>


extern "C"
{
#include <libavcodec></libavcodec>avcodec.h>
#include <libavformat></libavformat>avformat.h>
#include <libavutil></libavutil>pixdesc.h>
#include <libavutil></libavutil>hwcontext.h>
#include <libavutil></libavutil>opt.h>
#include <libavutil></libavutil>avassert.h>
#include <libavutil></libavutil>imgutils.h>
#include <libswscale></libswscale>swscale.h>
#include <libavdevice></libavdevice>avdevice.h>
#include <libavformat></libavformat>avio.h>
#include <libavutil></libavutil>time.h>
}

class ryMediaSource
{
public:
 ryMediaSource() {}
 ryMediaSource(const ryMediaSource& other);
 ~ryMediaSource();

 struct mediaSourceParams
 {
 int sx;
 int sy;
 int lsize;
 double fps;
 unsigned char* frame;
 };

 bool ryOpenMediaSource(const std::string&);
 mediaSourceParams ryGetMediaSourceFrame();
 void ryCloseMediaSource();

private:
 mediaSource pMediaSource;
 AVFormatContext* pFormatCtx;
 AVCodecContext* pCodecCtx;
 AVFrame* pFrame;
 SwsContext* pSwsCtx;
 AVPacket* pPacket;
 int pVideoStream;
 uint8_t* pBuffer;
 AVFrame* pPict;
 double pFPS;
 mutable std::mutex pMutex;
};

C++ source code :

#include "ryMediaSource.hpp"

ryMediaSource::ryMediaSource(const ryMediaSource& other)
:pFormatCtx(nullptr), 
pCodecCtx(nullptr), 
pFrame(nullptr), 
pSwsCtx(nullptr), 
pPacket(nullptr), 
pBuffer(nullptr), 
pPict(nullptr)
{
 std::lock_guard lock(other.pMutex);
 av_log_set_level(0);
 avformat_network_init();
}

bool ryMediaSource::ryOpenMediaSource(const std::string& url)
{
 int rc = -1;

 try
 {
 AVDictionary* pDicts = nullptr;

 pFormatCtx = avformat_alloc_context();
 if (!pFormatCtx)
 throw std::runtime_error("Failed to allocate AVFormatContext.");
 rc = av_dict_set(&pDicts, "rtsp_transport", "tcp", 0);
 if (rc < 0)
 throw std::runtime_error("av_dict_set failed.");
 rc = avformat_open_input(&pFormatCtx, url.c_str(), NULL, &pDicts);
 if (rc < 0)
 {
 av_dict_free(&pDicts); // Free the dictionary in case of an error
 throw std::runtime_error("Could not open source.");
 }
 }
 catch (const std::exception& e)
 {
 std::cerr << "Exception: " << e.what() << std::endl;
 return false;
 }

 try
 {
 rc = avformat_find_stream_info(pFormatCtx, NULL);
 if (rc < 0)
 {
 throw std::runtime_error("Could not find stream information.");
 }
 pVideoStream = -1;
 for (size_t v = 0; v < pFormatCtx->nb_streams; ++v)
 {
 if (pFormatCtx->streams[v]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
 {
 pVideoStream = static_cast<int>(v);
 AVRational rational = pFormatCtx->streams[pVideoStream]->avg_frame_rate;
 pFPS = 1.0 / ((double)rational.num / (double)(rational.den));
 break;
 }
 }
 if (pVideoStream < 0)
 {
 throw std::runtime_error("Could not find video stream.");
 }

 const AVCodec* pCodec = avcodec_find_decoder(pFormatCtx->streams[pVideoStream]->codecpar->codec_id);
 if (!pCodec)
 {
 throw std::runtime_error("Unsupported codec!");
 }
 pCodecCtx = avcodec_alloc_context3(pCodec);
 if (!pCodecCtx)
 {
 throw std::runtime_error("Failed to allocate AVCodecContext.");
 }
 rc = avcodec_parameters_to_context(pCodecCtx, pFormatCtx->streams[pVideoStream]->codecpar);
 if (rc != 0)
 {
 throw std::runtime_error("Could not copy codec context.");
 }
 rc = avcodec_open2(pCodecCtx, pCodec, NULL);
 if (rc < 0)
 {
 throw std::runtime_error("Could not open codec.");
 }
 pFrame = av_frame_alloc();
 if (!pFrame)
 {
 throw std::runtime_error("Could not allocate frame.");
 }
 pSwsCtx = sws_getContext(pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_BGR24, SWS_BILINEAR, NULL, NULL, NULL);
 if (!pSwsCtx)
 {
 throw std::runtime_error("Failed to allocate SwsContext.");
 }
 pPacket = av_packet_alloc();
 if (!pPacket)
 {
 throw std::runtime_error("Could not allocate AVPacket.");
 }
 pBuffer = (uint8_t*)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_BGR24, pCodecCtx->width, pCodecCtx->height, 1));
 if (!pBuffer)
 {
 throw std::runtime_error("Could not allocate buffer.");
 }
 pPict = av_frame_alloc();
 if (!pPict)
 {
 throw std::runtime_error("Could not allocate frame.");
 }
 av_image_fill_arrays(pPict->data, pPict->linesize, pBuffer, AV_PIX_FMT_BGR24, pCodecCtx->width, pCodecCtx->height, 1);
 }
 catch (const std::exception& e)
 {
 std::cerr << "Exception: " << e.what() << std::endl;
 return false;
 }

 return true;
}

ryMediaSource::mediaSourceParams ryMediaSource::ryGetMediaSourceFrame()
{
 mediaSourceParams msp = { 0, 0, 0, 0.0, nullptr };
 char errbuf[AV_ERROR_MAX_STRING_SIZE];

 std::lock_guard lock(pMutex);
 if (av_read_frame(pFormatCtx, pPacket) >= 0)
 {
 if (pPacket->stream_index == pVideoStream)
 {
 int ret = avcodec_send_packet(pCodecCtx, pPacket);
 if (ret < 0)
 {
 av_strerror(ret, errbuf, sizeof(errbuf));
 std::cerr << "Error sending packet for avcodec_send_packet: " << errbuf << std::endl;

 std::cerr << "avcodec_flush_buffers " << errbuf << std::endl;
 avcodec_flush_buffers(pCodecCtx);
 // Handle specific error cases
 if (ret == AVERROR(EAGAIN))
 {
 std::cerr << "EAGAIN indicates that more input is required" << std::endl;
 }
 else if (ret == AVERROR_EOF)
 {
 std::cerr << "AVERROR_EOF indicates that the encoder has been fully flushed" << std::endl;
 }
 else
 {
 //std::cerr << "avcodec_flush_buffers " << errbuf << std::endl;
 // For other errors, you may choose to flush the codec context and continue decoding.
 //avcodec_flush_buffers(pCodecCtx);
 }
 }
 ret = avcodec_receive_frame(pCodecCtx, pFrame);
 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
 {
 av_strerror(ret, errbuf, sizeof(errbuf));

 std::cerr << "Error receiving packet for avcodec_receive_frame: " << errbuf << std::endl;


 // EAGAIN indicates that more frames are needed or EOF is reached.
 // You may choose to break out of the loop or handle it based on your application's logic.

 return msp;
 }
 else if (ret < 0)
 {
 av_strerror(ret, errbuf, sizeof(errbuf));
 std::cerr << "Error receiving frame for avcodec_receive_frame: " << errbuf << std::endl;
 // Optionally, handle specific error cases
 if (ret == AVERROR(EINVAL))
 {
 std::cerr << "EINVAL indicates that more input is required" << std::endl;

 //break;
 }
 else
 {
 std::cerr << "For other errors" << std::endl;

 //break;
 }
 }
 // Move memory allocation outside the loop if frame size is constant
 size_t bufferSize = static_cast(pPict->linesize[0]) * pCodecCtx->height;
 msp.frame = new unsigned char[bufferSize];
 msp.lsize = pPict->linesize[0];
 msp.sx = pCodecCtx->width;
 msp.sy = pCodecCtx->height;
 msp.fps = pFPS;
 sws_scale(pSwsCtx, (uint8_t const* const*)pFrame->data, pFrame->linesize, 0, pCodecCtx->height, pPict->data, pPict->linesize);
 std::memcpy(msp.frame, pBuffer, bufferSize);
 //delete[] msp.frame;
 }

 // Unref packet for non-video streams
 av_packet_unref(pPacket);
 }

 return msp;
}

main.cpp

std::vector streams =
{
 {"rtsp://1},
 {"rtsp://2},
 {"rtsp://3},
 {"rtsp://4},
};

std::vector<rymediasource> mediaSources;

void main()
{
 int key = 0;
 int channel = 0;
 std::vector streamFrame(streams.size());
 ryMediaSource::mediaSourceParams msp = { 0, 0, 0, 0.0, nullptr };

 for (const auto& stream : streams)
 {
 mediaSources.emplace_back(); // Create a new instance for each stream
 }
 for (size_t s = 0; s < streams.size(); s++)
 {
 try
 {
 mediaSources[s].ryOpenMediaSource(streams[s]);
 }
 catch (const std::exception& e)
 {
 std::cerr << "Error initializing stream " << s << ": " << e.what() << std::endl;
 }
 }

 cv::namedWindow("ryInferenceServer", cv::WINDOW_FREERATIO);
 cv::resizeWindow("ryInferenceServer", 640, 480);
 cv::moveWindow("ryInferenceServer", 0, 0);
 for (;;)
 {
 for (size_t st = 0; st < mediaSources.size(); ++st)
 {
 msp = mediaSources[st].ryGetMediaSourceFrame();
 if (msp.frame != nullptr)
 {
 cv::Mat preview;
 cv::Mat frame(msp.sy, msp.sx, CV_8UC3, msp.frame, msp.lsize);
 cv::resize(frame, preview, cv::Size(640, 480));
 if (!frame.empty())
 {
 try
 {
 streamFrame[st] = frame.clone();
 if (channel == st)
 {
 cv::imshow("ryInferenceServer", preview);
 key = cv::waitKeyEx(1);
 if (key == LEFT_KEY)
 {
 channel--;
 if (channel < 0)
 channel = 0;
 }
 if (key == RIGHT_KEY)
 {
 channel++;
 if (channel >= mediaSources.size())
 channel = mediaSources.size() - 1;
 }
 if (key == 27)
 break;
 }
 streamFrame[st].release();
 delete[] msp.frame;
 }
 catch (const std::exception& e)
 {
 std::cerr << "Exception in processing frame for stream " << st << ": " << e.what() << std::endl;
 }
 }
 frame.release();
 }
 }
 }
}
</rymediasource></int></mutex></thread></chrono></vector></string></iostream>

Syndiquer tout le site
