Recherche avancée
Médias (1)
-
Rennes Emotion Map 2010-11
19 octobre 2011, par
Mis à jour : Juillet 2013
Langue : français
Type : Texte
Autres articles (62)
-
Les autorisations surchargées par les plugins
27 avril 2010, parMediaspip core
autoriser_auteur_modifier() afin que les visiteurs soient capables de modifier leurs informations sur la page d’auteurs -
Support de tous types de médias
10 avril 2011Contrairement à beaucoup de logiciels et autres plate-formes modernes de partage de documents, MediaSPIP a l’ambition de gérer un maximum de formats de documents différents qu’ils soient de type : images (png, gif, jpg, bmp et autres...) ; audio (MP3, Ogg, Wav et autres...) ; vidéo (Avi, MP4, Ogv, mpg, mov, wmv et autres...) ; contenu textuel, code ou autres (open office, microsoft office (tableur, présentation), web (html, css), LaTeX, Google Earth) (...)
-
Support audio et vidéo HTML5
10 avril 2011MediaSPIP utilise les balises HTML5 video et audio pour la lecture de documents multimedia en profitant des dernières innovations du W3C supportées par les navigateurs modernes.
Pour les navigateurs plus anciens, le lecteur flash Flowplayer est utilisé.
Le lecteur HTML5 utilisé a été spécifiquement créé pour MediaSPIP : il est complètement modifiable graphiquement pour correspondre à un thème choisi.
Ces technologies permettent de distribuer vidéo et son à la fois sur des ordinateurs conventionnels (...)
Sur d’autres sites (8116)
-
FFMPEG Presentation Time Stamps (PTS) calculation in RTSP stream
8 décembre 2020, par BadaBudaBuduBelow please find en raw example of my code for your better understanding of what it does. Please note that this is an updated (removed deprecated methods, etc.) example code by myself from the official FFMPEG documentation complemented by my encoder.


/// STD
#include <iostream>
#include <string>

/// FFMPEG
extern "C"
{
 #include <libavformat></libavformat>avformat.h>
 #include <libswscale></libswscale>swscale.h>
 #include <libavutil></libavutil>imgutils.h>
}

/// VideoLib
#include <tools></tools>multimediaprocessing.h>
#include 
#include 
#include <enums></enums>codec.h>
#include <enums></enums>pixelformat.h>

/// OpenCV
#include <opencv2></opencv2>opencv.hpp>

inline static const char *inputRtspAddress = "rtsp://192.168.0.186:8080/video/h264";

int main()
{
 AVFormatContext* formatContext = nullptr;

 AVStream* audioStream = nullptr;
 AVStream* videoStream = nullptr;
 AVCodec* audioCodec = nullptr;
 AVCodec* videoCodec = nullptr;
 AVCodecContext* audioCodecContext = nullptr;
 AVCodecContext* videoCodecContext = nullptr;
 vl::AudioSettings audioSettings;
 vl::VideoSettings videoSettings;

 int audioIndex = -1;
 int videoIndex = -1;

 SwsContext* swsContext = nullptr;
 std::vector frameBuffer;
 AVFrame* frame = av_frame_alloc();
 AVFrame* decoderFrame = av_frame_alloc();

 AVPacket packet;
 cv::Mat mat;

 vl::tools::MultimediaProcessing multimediaProcessing("rtsp://127.0.0.1:8080/stream", vl::configs::rtspStream, 0, vl::enums::EPixelFormat::ABGR);

 // *** OPEN STREAM *** //
 if(avformat_open_input(&formatContext, inputRtspAddress, nullptr, nullptr) < 0)
 {
 std::cout << "Failed to open input." << std::endl;
 return EXIT_FAILURE;
 }

 if(avformat_find_stream_info(formatContext, nullptr) < 0)
 {
 std::cout << "Failed to find stream info." << std::endl;
 return EXIT_FAILURE;
 }

 // *** FIND DECODER FOR BOTH AUDIO AND VIDEO STREAM *** //
 audioCodec = avcodec_find_decoder(AVCodecID::AV_CODEC_ID_AAC);
 videoCodec = avcodec_find_decoder(AVCodecID::AV_CODEC_ID_H264);

 if(audioCodec == nullptr || videoCodec == nullptr)
 {
 std::cout << "No AUDIO or VIDEO in stream." << std::endl;
 return EXIT_FAILURE;
 }

 // *** FIND STREAM FOR BOTH AUDIO AND VIDEO STREAM *** //

 audioIndex = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &audioCodec, 0);
 videoIndex = av_find_best_stream(formatContext, AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0);

 if(audioIndex < 0 || videoIndex < 0)
 {
 std::cout << "Failed to find AUDIO or VIDEO stream." << std::endl;
 return EXIT_FAILURE;
 }

 audioStream = formatContext->streams[audioIndex];
 videoStream = formatContext->streams[videoIndex];

 // *** ALLOC CODEC CONTEXT FOR BOTH AUDIO AND VIDEO STREAM *** //
 audioCodecContext = avcodec_alloc_context3(audioCodec);
 videoCodecContext = avcodec_alloc_context3(videoCodec);

 if(audioCodecContext == nullptr || videoCodecContext == nullptr)
 {
 std::cout << "Can not allocate AUDIO or VIDEO context." << std::endl;
 return EXIT_FAILURE;
 }

 if(avcodec_parameters_to_context(audioCodecContext, formatContext->streams[audioIndex]->codecpar) < 0 || avcodec_parameters_to_context(videoCodecContext, formatContext->streams[videoIndex]->codecpar) < 0)
 {
 std::cout << "Can not fill AUDIO or VIDEO codec context." << std::endl;
 return EXIT_FAILURE;
 }

 if(avcodec_open2(audioCodecContext, audioCodec, nullptr) < 0 || avcodec_open2(videoCodecContext, videoCodec, nullptr) < 0)
 {
 std::cout << "Failed to open AUDIO codec" << std::endl;
 return EXIT_FAILURE;
 }

 // *** INITIALIZE MULTIMEDIA PROCESSING *** //
 std::vector<unsigned char="char"> extraData(audioStream->codecpar->extradata_size);
 std::copy_n(audioStream->codecpar->extradata, extraData.size(), extraData.begin());

 audioSettings.sampleRate = audioStream->codecpar->sample_rate,
 audioSettings.bitrate = audioStream->codecpar->bit_rate,
 audioSettings.codec = vl::enums::EAudioCodec::AAC,
 audioSettings.channels = audioStream->codecpar->channels,
 audioSettings.bitsPerCodedSample = audioStream->codecpar->bits_per_coded_sample,
 audioSettings.bitsPerRawSample = audioStream->codecpar->bits_per_raw_sample,
 audioSettings.blockAlign = audioStream->codecpar->block_align,
 audioSettings.channelLayout = audioStream->codecpar->channel_layout,
 audioSettings.format = audioStream->codecpar->format,
 audioSettings.frameSize = audioStream->codecpar->frame_size,
 audioSettings.codecExtraData = std::move(extraData);

 videoSettings.width = 1920;
 videoSettings.height = 1080;
 videoSettings.framerate = 25;
 videoSettings.pixelFormat = vl::enums::EPixelFormat::ARGB;
 videoSettings.bitrate = 8000 * 1000;
 videoSettings.codec = vl::enums::EVideoCodec::H264;

 multimediaProcessing.initEncoder(videoSettings, audioSettings);

 // *** INITIALIZE SWS CONTEXT *** //
 swsContext = sws_getCachedContext(nullptr, videoCodecContext->width, videoCodecContext->height, videoCodecContext->pix_fmt, videoCodecContext->width, videoCodecContext->height, AV_PIX_FMT_RGBA, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr);

 if (const auto inReturn = av_image_get_buffer_size(AV_PIX_FMT_RGBA, videoCodecContext->width, videoCodecContext->height, 1); inReturn > 0)
 {
 frameBuffer.reserve(inReturn);
 }
 else
 {
 std::cout << "Can not get buffer size." << std::endl;
 return EXIT_FAILURE;
 }

 if (const auto inReturn = av_image_fill_arrays(frame->data, frame->linesize, frameBuffer.data(), AV_PIX_FMT_RGBA, videoCodecContext->width, videoCodecContext->height, 1); inReturn < 0)
 {
 std::cout << "Can not fill buffer arrays." << std::endl;
 return EXIT_FAILURE;
 }

 // *** MAIN LOOP *** //
 while(true)
 {
 // Return the next frame of a stream.
 if(av_read_frame(formatContext, &packet) == 0)
 {
 if(packet.stream_index == videoIndex) // Check if it is video packet.
 {
 // Send packet to decoder.
 if(avcodec_send_packet(videoCodecContext, &packet) == 0)
 {
 int returnCode = avcodec_receive_frame(videoCodecContext, decoderFrame); // Get Frame from decoder.

 if (returnCode == 0) // Transform frame and send it to encoder. And re-stream that.
 {
 sws_scale(swsContext, decoderFrame->data, decoderFrame->linesize, 0, decoderFrame->height, frame->data, frame->linesize);

 mat = cv::Mat(videoCodecContext->height, videoCodecContext->width, CV_8UC4, frameBuffer.data(), frame->linesize[0]);

 cv::resize(mat, mat, cv::Size(1920, 1080), cv::INTER_NEAREST);

 multimediaProcessing.encode(mat.data, packet.dts, packet.dts, packet.flags == AV_PKT_FLAG_KEY); // Thise line sends cv::Mat to encoder and re-streams it.

 av_packet_unref(&packet);
 }
 else if(returnCode == AVERROR(EAGAIN))
 {
 av_frame_unref(decoderFrame);
 av_freep(decoderFrame);
 }
 else
 {
 av_frame_unref(decoderFrame);
 av_freep(decoderFrame);

 std::cout << "Error during decoding." << std::endl;
 return EXIT_FAILURE;
 }
 }
 }
 else if(packet.stream_index == audioIndex) // Check if it is audio packet.
 {
 std::vector vectorPacket(packet.data, packet.data + packet.size);

 multimediaProcessing.addAudioPacket(vectorPacket, packet.dts, packet.dts);
 }
 else
 {
 av_packet_unref(&packet);
 }
 }
 else
 {
 std::cout << "Can not send video packet to decoder." << std::endl;
 std::this_thread::sleep_for(std::chrono::seconds(1));
 }
 }

 return EXIT_SUCCESS;
 }
</unsigned></string></iostream>

What does It do ?


It takes a single RTSP stream to decode its data so I can, for example, draw something to its frames or whatever, and then stream it under a different address.


Basically, I am opening the RTSP stream, check if it does contain both audio and video streams, and find a decoder for them. Then I create an encoder to which I will tell how the output stream should look like and that's it.


And this point I will create an endless loop Where I will read all packets coming from the input stream, then decode it does something to it and again encode it and re=stream it.


What is the issue ?


If you take a closer look I am sending both video and audio frame together with lastly received PTS and DTS contained in AVPacket, to the encoder.


The PTS and DTS from the point when I receive the first AVPacket looks for example like this.


IN AUDIO STREAM :




-22783, -21759, -20735, -19711, -18687, -17663, -16639, -15615, -14591, -13567, -12543, -11519, -10495, -9471, -8447, -7423, -6399, -5375, -4351, -3327, -2303, -1279, -255, 769, 1793, 2817, 3841, 4865, 5889, 6913, 7937, 8961, 9985, 11009, 12033, 13057, 14081, 15105, 16129, 17153




As you can see it is every time incremented by 1024 and that is a sample rate of the audio stream. Quite clear here.


IN VIDEO STREAM :




86400, 90000, 93600, 97200, 100800, 104400, 108000, 111600, 115200, 118800, 122400, 126000, 129600, 133200, 136800, 140400, 144000, 147600, 151200, 154800, 158400, 162000, 165600




As you can see it is every time incremented by 3600 but WHY ?. What this number actually mean ?


From what I can understand, those received PTS and DTS are for the following :


DTS should tell the encoder when it should start encoding the frame so the frame in time are in the correct order and not mishmashed.


PTS should say the correct time when the frame should be played/displayed in the output stream so the frame in time are in the correct order and not mishmashed.


What I am trying to achieve ?


As I said I need to restream a RTSP stream. I can not use PTS and DTS which comes from received AVPackets, because at some point it can happen that the input stream can randomly close and I need to open it again. The problem is that when I actually do it, then the PTS and DTS start to generate again from the minus values same as you could see in the samples. I CAN NOT send those "new" PTS and DTS to the encoder because they are now lower than the encoder/muxer expects.


I need to continually stream something (both audio and video), even it is a blank black screen or silent audio. And each frame the PTS and DTS should rise by a specific number. I need to figure out how the increment is calculated.


----------------------------------


The final result should look like a mosaic of multiple input streams in a single output stream. A single input stream (main) has both audio and video and the rest (side) has just video. Some of those streams can randomly close in time and I need to ensure that it will be back again once it is possible.


-
ffmpeg scale 4k video and add subtitles
29 novembre 2020, par goodkid38I am having trouble scaling video from 4k to 1080 while still retaining the video sub titles. These subtitles are PGS based and therefore need to be baked into the output video. In order to scale the video I am using ffmpeg with a filter_complex like so :


'-filter_complex', `[0:v][0:s:randomSubtitleHere]scale=1920:1080[scaled];[scaled][0:s:0]overlay[v]`, 
'-map', '[v]', '-map', 
`0:a:${movieTitle['audioSelect']}`,


This will give me the video at 1080, but it i still am not seeing the subtitles I have selected.


-
Frame sliced into 2 under some degree. How fix it ?
26 novembre 2020, par Алекс АникейI trying use h264 codec in videochat application. And in some reason frame sliced into 2 triangle (picture below). I try send my desktop image to another person and get this image on another client.


What settings i set wrong ?
My code :


Init :


VCSession *vc_new_x264(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data,
 VCSession *vc)
{

 if (x264_param_default_preset(&param, "slow", "zerolatency") < 0) {
 // goto fail;
 }

 param.i_csp = X264_CSP_I420;
 param.i_width = 1920;
 param.i_height = 1080;
 vc->h264_enc_width = param.i_width;
 vc->h264_enc_height = param.i_height;

 param.i_keyint_max = 30;

 param.b_vfr_input = 1; /* VFR input. If 1, use timebase and timestamps for ratecontrol purposes.
 * If 0, use fps only. */
 param.i_timebase_num = 1; // 1 ms = timebase units = (1/1000)s
 param.i_timebase_den = 1000; // 1 ms = timebase units = (1/1000)s
 param.b_repeat_headers = 1;
 param.b_annexb = 1;

 param.rc.f_rate_tolerance = VIDEO_F_RATE_TOLERANCE_H264;
 param.rc.i_vbv_buffer_size = 1500;
 param.rc.i_vbv_max_bitrate = VIDEO_BITRATE_INITIAL_VALUE_H264 * 1;

 vc->h264_enc_bitrate = VIDEO_BITRATE_INITIAL_VALUE_H264;

 param.rc.i_qp_min = 13;
 param.rc.i_qp_max = 35; // max quantizer for x264

 vc->h264_enc_bitrate = VIDEO_BITRATE_INITIAL_VALUE_H264;

 param.rc.b_stat_read = 0;
 param.rc.b_stat_write = 0;


 if (x264_param_apply_profile(&param,
 "high") < 0) { // "baseline", "main", "high", "high10", "high422", "high444"
 // goto fail;
 }


 if (x264_picture_alloc(&(vc->h264_in_pic), param.i_csp, param.i_width, param.i_height) < 0) {
 // goto fail;
 }

 vc->h264_encoder = x264_encoder_open(&param);

 AVCodec *codec = NULL;
 vc->h264_decoder = NULL;
 avcodec_register_all();
 codec = NULL;

 codec = avcodec_find_decoder(AV_CODEC_ID_H264);

 if (!codec) {
 LOGGER_WARNING(log, "codec not found H264 on decoder");
 }

 vc->h264_decoder = avcodec_alloc_context3(codec);

 if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
 vc->h264_decoder->flags |= AV_CODEC_FLAG_TRUNCATED; /* we do not send complete frames */
 }


 vc->h264_decoder->delay = 5;

 if (avcodec_open2(vc->h264_decoder, codec, NULL) < 0) {
 LOGGER_WARNING(log, "could not open codec H264 on decoder");
 }


 return vc;
}


Get frame and decoding him :


void vc_iterate_x264(VCSession *vc)
{

 if (!vc) {
 return;
 }

 pthread_mutex_lock(vc->queue_mutex);

 struct RTPMessage *p;

 if (!rb_read(vc->vbuf_raw, (void **)&p)) {
 LOGGER_TRACE(vc->log, "no Video frame data available");
 pthread_mutex_unlock(vc->queue_mutex);
 return;
 }

 pthread_mutex_unlock(vc->queue_mutex);
 const struct RTPHeader *const header = &p->header;

 uint32_t full_data_len;

 if (header->flags & RTP_LARGE_FRAME) {
 full_data_len = header->data_length_full;
 LOGGER_WARNING(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len);
 } else {
 full_data_len = p->len;
 if (header->data_length_lower != full_data_len)
 {
 LOGGER_ERROR("Data header and packet don't equal: %d - header %d - packet", header->data_length_lower, full_data_len);
 }
 LOGGER_DEBUG(vc->log, "vc_iterate:002");
 }

 decode_frame_h264(vc, p, full_data_len);
}

void decode_frame_h264(VCSession *vc,
 struct RTPMessage *p,
 uint32_t full_data_len)
{

 AVPacket *compr_data;
 compr_data = av_packet_alloc();


 uint8_t *tmp_buf = calloc(1, full_data_len + FF_INPUT_BUFFER_PADDING_SIZE);
 memcpy(tmp_buf, p->data, full_data_len);

 compr_data->data = tmp_buf; // p->data;
 compr_data->size = (int)full_data_len; // hmm, "int" again

 avcodec_send_packet(vc->h264_decoder, compr_data);

 int ret_ = 0;
 while (ret_ >= 0) {
 AVFrame *frame = av_frame_alloc();
 ret_ = avcodec_receive_frame(vc->h264_decoder, frame);
 if (ret_ == AVERROR(EAGAIN) || ret_ == AVERROR_EOF) {
 // error
 break;
 } else if (ret_ < 0) {
 // Error during decoding
 break;
 } else if (ret_ == 0) {
 vc->vcb(vc->av, vc->friend_number, frame->width, frame->height,
 (const uint8_t *)frame->data[0],
 (const uint8_t *)frame->data[1],
 (const uint8_t *)frame->data[2],
 frame->linesize[0], frame->linesize[1],
 frame->linesize[2], vc->vcb_user_data);
 } else {
 // some other error
 }
 av_frame_free(&frame);
 }
 av_packet_free(&compr_data);
 free(tmp_buf);
 free(p);
}


Send frame and encoding :


bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y,
 const uint8_t *u, const uint8_t *v, Toxav_Err_Send_Frame *error, int16_t kf_max_dist, vpx_codec_er_flags_t error_resilient,
 unsigned int my_lag_in_frames, uint16_t kf_mode, uint16_t quality_mode)
{
 Toxav_Err_Send_Frame rc = TOXAV_ERR_SEND_FRAME_OK;
 ToxAVCall *call;
 uint64_t video_frame_record_timestamp = current_time_monotonic(av->m->mono_time);

 int vpx_encode_flags = 0;

 pthread_mutex_lock(call->mutex_video);
 pthread_mutex_unlock(av->mutex);

 if (y == nullptr || u == nullptr || v == nullptr) {
 pthread_mutex_unlock(call->mutex_video);
 rc = TOXAV_ERR_SEND_FRAME_NULL;
 goto RETURN;
 }


 if (call->video_rtp->ssrc < VIDEO_SEND_X_KEYFRAMES_FIRST) {
 // Key frame flag for first frames
 vpx_encode_flags = VPX_EFLAG_FORCE_KF;
 LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d only-i-frame mode", call->video_rtp->ssrc);

 ++call->video_rtp->ssrc;
 } else if (call->video_rtp->ssrc == VIDEO_SEND_X_KEYFRAMES_FIRST) {
 // normal keyframe placement
 vpx_encode_flags = 0;
 LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d normal mode", call->video_rtp->ssrc);

 ++call->video_rtp->ssrc;
 }


 x264_nal_t *nal = NULL;
 int i_frame_size = 0;

 uint32_t result = encode_frame_h264(av, friend_number, width, height,
 y, u, v,
 &video_frame_record_timestamp,
 vpx_encode_flags,
 &nal,
 &i_frame_size);
 if (result != 0) {
 pthread_mutex_unlock(call->mutex_video);
 rc = TOXAV_ERR_SEND_FRAME_INVALID;
 goto RETURN;
 }

 ++call->video->frame_counter;

 rc = send_frames_h264(av, friend_number, width, height,
 y, u, v, call,
 &video_frame_record_timestamp,
 vpx_encode_flags,
 &nal,
 &i_frame_size,
 &rc);

 pthread_mutex_unlock(call->mutex_video);

RETURN:

 if (error) {
 *error = rc;
 }

 return rc == TOXAV_ERR_SEND_FRAME_OK;
}

uint32_t send_frames_h264(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height,
 const uint8_t *y,
 const uint8_t *u, const uint8_t *v, ToxAVCall *call,
 uint64_t *video_frame_record_timestamp,
 int vpx_encode_flags,
 x264_nal_t **nal,
 int *i_frame_size,
 TOXAV_ERR_SEND_FRAME *rc)
{

 if (*i_frame_size > 0) {

 // use the record timestamp that was actually used for this frame
 *video_frame_record_timestamp = (uint64_t)call->video->h264_in_pic.i_pts;
 const uint32_t frame_length_in_bytes = *i_frame_size;
 const int keyframe = (int)call->video->h264_out_pic.b_keyframe;

 LOGGER_DEBUG(av->m->log, "video packet record time: %lu", (*video_frame_record_timestamp));

 int res = rtp_send_data
 (
 call->video_rtp,
 (const uint8_t *)((*nal)->p_payload),
 frame_length_in_bytes,
 keyframe,
 *video_frame_record_timestamp,
 av->m->log
 );

 (*video_frame_record_timestamp)++;

 if (res < 0) {
 LOGGER_WARNING(av->m->log, "Could not send video frame: %s", strerror(errno));
 *rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
 return 1;
 }

 return 0;
 } else {
 *rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
 return 1;
 }

}


I get image like this :




Syndiquer tout le site
