Recherche avancée

Sur d’autres sites (6015)

• H264 codec encode, decode and write to file

  30 novembre 2020, par Алекс Аникей

  I try to use ffmpeg and h264 codec to translate the video in realtime. But at the state of decoding encoded frame, I get some "bad" image.
Init encoder and decoder :

  


      VCSession *vc_new_x264(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data,
                       VCSession *vc)
{

// ------ ffmpeg encoder ------
    AVCodec *codec2 = NULL;
    vc->h264_encoder_ctx = NULL;//AVCodecContext type

    codec2 = NULL;
    avcodec_register_all();
    codec2 = avcodec_find_encoder(AV_CODEC_ID_H264);
    if (codec2 == NULL)
    {
        LOGGER_WARNING(log, "h264: not find encoder");
    }

    vc->h264_encoder_ctx = avcodec_alloc_context3(codec2);

    vc->h264_out_pic2 = av_packet_alloc();

    vc->h264_encoder_ctx->bit_rate = 10 *1000 * 1000;
    vc->h264_encoder_ctx->width = 800;
    vc->h264_encoder_ctx->height = 600;

    vc->h264_enc_width = vc->h264_encoder_ctx->width;
    vc->h264_enc_height = vc->h264_encoder_ctx->height;
    vc->h264_encoder_ctx->time_base = (AVRational) {
            1, 30
    };
    vc->h264_encoder_ctx->gop_size = 30;
    vc->h264_encoder_ctx->max_b_frames = 1;
    vc->h264_encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;


    av_opt_set(vc->h264_encoder_ctx->priv_data, "preset", "veryfast", 0);


    av_opt_set(vc->h264_encoder_ctx->priv_data, "annex_b", "1", 0);
    av_opt_set(vc->h264_encoder_ctx->priv_data, "repeat_headers", "1", 0);
    av_opt_set(vc->h264_encoder_ctx->priv_data, "tune", "zerolatency", 0);
    av_opt_set_int(vc->h264_encoder_ctx->priv_data, "zerolatency", 1, 0);

    vc->h264_encoder_ctx->time_base.num = 1;
    vc->h264_encoder_ctx->time_base.den = 1000;

    vc->h264_encoder_ctx->framerate = (AVRational) {
        1000, 40
    };

    AVDictionary *opts = NULL;

    if (avcodec_open2(vc->h264_encoder_ctx, codec2, &opts) < 0) {
        LOGGER_ERROR(log, "could not open codec H264 on encoder");
    }

    av_dict_free(&opts);



    AVCodec *codec = NULL;
    vc->h264_decoder_ctx = NULL;// AVCodecContext - type
    codec = NULL;

    codec = avcodec_find_decoder(AV_CODEC_ID_H264);

    if (!codec) {
        LOGGER_WARNING(log, "codec not found H264 on decoder");
    }

    vc->h264_decoder_ctx = avcodec_alloc_context3(codec);

    if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
        vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG_TRUNCATED; /* we do not send complete frames */
    }

    if (codec->capabilities & AV_CODEC_FLAG_LOW_DELAY) {
        vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
    }

    vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG2_SHOW_ALL;

     vc->h264_decoder_ctx->refcounted_frames = 0;

    vc->h264_decoder_ctx->delay = 0;
    vc->h264_decoder_ctx->sw_pix_fmt = AV_PIX_FMT_YUV420P;
    av_opt_set_int(vc->h264_decoder_ctx->priv_data, "delay", 0, AV_OPT_SEARCH_CHILDREN);
    vc->h264_decoder_ctx->time_base = (AVRational) {
            40, 1000
};
    vc->h264_decoder_ctx->framerate = (AVRational) {
        1000, 40
    };

    if (avcodec_open2(vc->h264_decoder_ctx, codec, NULL) < 0) {
        LOGGER_WARNING(log, "could not open codec H264 on decoder");
    }
    vc->h264_decoder_ctx->refcounted_frames = 0;

    return vc;
}


  


  Encoding (in this function i encode frame and for debugging decode and save him in file) :

  


  uint32_t encode_frame_h264_p(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height,
                           const uint8_t *y,
                           const uint8_t *u, const uint8_t *v, ToxAVCall *call,
                           uint64_t *video_frame_record_timestamp,
                           int vpx_encode_flags,
                           x264_nal_t **nal,
                           int *i_frame_size)
{
    AVFrame *frame;
    int ret;
    uint32_t result = 1;

    frame = av_frame_alloc();

    frame->format = call->video->h264_encoder_ctx->pix_fmt;
    frame->width  = width;
    frame->height = height;

    ret = av_frame_get_buffer(frame, 32);

    if (ret < 0) {
        LOGGER_ERROR(av->m->log, "av_frame_get_buffer:Could not allocate the video frame data");
    }

    /* make sure the frame data is writable */
    ret = av_frame_make_writable(frame);

    if (ret < 0) {
        LOGGER_ERROR(av->m->log, "av_frame_make_writable:ERROR");
    }

    frame->pts = (int64_t)(*video_frame_record_timestamp);


    // copy YUV frame data into buffers
    memcpy(frame->data[0], y, width * height);
    memcpy(frame->data[1], u, (width / 2) * (height / 2));
    memcpy(frame->data[2], v, (width / 2) * (height / 2));

    // encode the frame
    ret = avcodec_send_frame(call->video->h264_encoder_ctx, frame);

    if (ret < 0) {
        LOGGER_ERROR(av->m->log, "Error sending a frame for encoding:ERROR");
    }


    ret = avcodec_receive_packet(call->video->h264_encoder_ctx, call->video->h264_out_pic2);



    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
        *i_frame_size = 0;
    } else if (ret < 0) {
        *i_frame_size = 0;
        // fprintf(stderr, "Error during encoding\n");
    } else {

        // Decoded encoded frame and save him to file

        saveInFile(call->video->h264_decoder_ctx, frame, call->video->h264_out_pic2, "/home/user/testSave");

        // printf("Write packet %3"PRId64" (size=%5d)\n", call->video->h264_out_pic2->pts, call->video->h264_out_pic2->size);
        // fwrite(call->video->h264_out_pic2->data, 1, call->video->h264_out_pic2->size, outfile);

        global_encoder_delay_counter++;

        if (global_encoder_delay_counter > 60) {
            global_encoder_delay_counter = 0;
            LOGGER_DEBUG(av->m->log, "enc:delay=%ld",
                         (long int)(frame->pts - (int64_t)call->video->h264_out_pic2->pts)
                        );
        }


        *i_frame_size = call->video->h264_out_pic2->size;
        *video_frame_record_timestamp = (uint64_t)call->video->h264_out_pic2->pts;

        result = 0;
    }

    av_frame_free(&frame);

    return result;

}


  


  Decode and save frame code :

  


  void saveInFile(AVCodecContext *dec_ctx, AVFrame *frame, AVPacket *pkt, const char *filename)
{
    if (!pkt)
        return;
    char buf[1024];
    int ret;
    static int curNumber = 0;
    ret = avcodec_send_packet(dec_ctx, pkt);
    if (ret < 0 && ret != AVERROR_EOF)
    {
        fprintf(stderr, "Error sending a packet for decoding'\n");
        if ( ret == AVERROR(EAGAIN))
            return;
        if (ret == AVERROR(EINVAL))
            return;
        if (ret == AVERROR(ENOMEM))
            return;

    }

        ret = avcodec_receive_frame(dec_ctx, frame);
        if (ret == AVERROR(EAGAIN) )
            return;
        if (ret == AVERROR_EOF)
        {
            return;
        }
        else if (ret < 0)
        {
            fprintf(stderr, "Error during decoding\n");
        }
        printf("saving frame %3d\n", dec_ctx->frame_number);
        sprintf(buf, "%s%d", filename, curNumber);
        curNumber++;
        pgm_save(frame->data[0], frame->linesize[0], frame->width, frame->height, buf);

}

void pgm_save(unsigned char* buf, int wrap, int xsize, int ysize, char *filename)
{
    FILE *f;
    int i;
    f = fopen(filename, "w");
    fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
    for (i =0; i < ysize; i++)
        fwrite(buf + i* wrap, 1, xsize, f);
    fclose(f);
}


  


  After this manipulation I have smth like that :


  


• Revision 30147 : corrections orthographiques

  24 juillet 2009, par denisb@… — Log

  corrections orthographiques

• ffmpeg.wasm - How to do literally anything with a blob url

  24 novembre 2024, par SeriousLee

  I'm using the ffmpeg.wasm for the first time and I can't get anything working, beyond loading it. I have this function that does nothing in particular (I got it from the vite + react example in the docs, slightly modified) and all I want to do is pass it a blob URL like this blob:http://localhost:5173/c7a9ea7c-aa26-4f4f-9c80-11b8aef3e81f and run through the function and have it give me anything back. But instead, it hangs on the ffmpeg.exec command and never completes. And yes, I've determined that the input blob does work - it's an 8MB, 12-second long mp4 clip.

  


  Here's the function :

  


      const processOutputVideo = async (videoURL) => {
      const ffmpeg = ffmpegRef.current;

      await ffmpeg.writeFile("input.mp4", await fetchFile(videoURL));
      await ffmpeg.exec(["-i", "input.mp4", "output.mp4"]);

      const fileData = await ffmpeg.readFile("output.mp4");
      const blob = new Blob([fileData.buffer], { type: "video/mp4" });
      const blobUrl = URL.createObjectURL(blob);

      return blobUrl;
    };


  


  And here's the ffmpeg logs from my terminal.

  


  [FFMPEG stderr] ffmpeg version 5.1.4 Copyright (c) 2000-2023 the FFmpeg developers
Post.jsx:35 [FFMPEG stderr]   built with emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) 3.1.40 (5c27e79dd0a9c4e27ef2326841698cdd4f6b5784)
Post.jsx:35 [FFMPEG stderr]   configuration: --target-os=none --arch=x86_32 --enable-cross-compile --disable-asm --disable-stripping --disable-programs --disable-doc --disable-debug --disable-runtime-cpudetect --disable-autodetect --nm=emnm --ar=emar --ranlib=emranlib --cc=emcc --cxx=em++ --objcc=emcc --dep-cc=emcc --extra-cflags='-I/opt/include -O3 -msimd128 -sUSE_PTHREADS -pthread' --extra-cxxflags='-I/opt/include -O3 -msimd128 -sUSE_PTHREADS -pthread' --enable-gpl --enable-libx264 --enable-libx265 --enable-libvpx --enable-libmp3lame --enable-libtheora --enable-libvorbis --enable-libopus --enable-zlib --enable-libwebp --enable-libfreetype --enable-libfribidi --enable-libass --enable-libzimg
Post.jsx:35 [FFMPEG stderr]   libavutil      57. 28.100 / 57. 28.100
Post.jsx:35 [FFMPEG stderr]   libavcodec     59. 37.100 / 59. 37.100
Post.jsx:35 [FFMPEG stderr]   libavformat    59. 27.100 / 59. 27.100
Post.jsx:35 [FFMPEG stderr]   libavdevice    59.  7.100 / 59.  7.100
Post.jsx:35 [FFMPEG stderr]   libavfilter     8. 44.100 /  8. 44.100
Post.jsx:35 [FFMPEG stderr]   libswscale      6.  7.100 /  6.  7.100
Post.jsx:35 [FFMPEG stderr]   libswresample   4.  7.100 /  4.  7.100
Post.jsx:35 [FFMPEG stderr]   libpostproc    56.  6.100 / 56.  6.100
Post.jsx:35 [FFMPEG stderr] Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'input.mp4':
Post.jsx:35 [FFMPEG stderr]   Metadata:
Post.jsx:35 [FFMPEG stderr]     major_brand     : mp42
Post.jsx:35 [FFMPEG stderr]     minor_version   : 0
Post.jsx:35 [FFMPEG stderr]     compatible_brands: mp42mp41isomavc1
Post.jsx:35 [FFMPEG stderr]     creation_time   : 2019-03-15T17:39:05.000000Z
Post.jsx:35 [FFMPEG stderr]   Duration: 00:00:12.82, start: 0.000000, bitrate: 5124 kb/s
Post.jsx:35 [FFMPEG stderr]   Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p(tv, bt709, progressive), 1920x1080, 4985 kb/s, 29.97 fps, 29.97 tbr, 30k tbn (default)
Post.jsx:35 [FFMPEG stderr]     Metadata:
Post.jsx:35 [FFMPEG stderr]       creation_time   : 2019-03-15T17:39:05.000000Z
Post.jsx:35 [FFMPEG stderr]       handler_name    : L-SMASH Video Handler
Post.jsx:35 [FFMPEG stderr]       vendor_id       : [0][0][0][0]
Post.jsx:35 [FFMPEG stderr]       encoder         : AVC Coding
Post.jsx:35 [FFMPEG stderr]   Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo, fltp, 137 kb/s (default)
Post.jsx:35 [FFMPEG stderr]     Metadata:
Post.jsx:35 [FFMPEG stderr]       creation_time   : 2019-03-15T17:39:05.000000Z
Post.jsx:35 [FFMPEG stderr]       handler_name    : L-SMASH Audio Handler
Post.jsx:35 [FFMPEG stderr]       vendor_id       : [0][0][0][0]
Post.jsx:35 [FFMPEG stderr] Stream mapping:
Post.jsx:35 [FFMPEG stderr]   Stream #0:0 -> #0:0 (h264 (native) -> h264 (libx264))
Post.jsx:35 [FFMPEG stderr]   Stream #0:1 -> #0:1 (aac (native) -> aac (native))
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0x154e4f0] using cpu capabilities: none!
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0x154e4f0] profile High, level 4.0, 4:2:0, 8-bit
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0x154e4f0] 264 - core 164 - H.264/MPEG-4 AVC codec - Copyleft 2003-2022 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=6 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=25 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00


  


  And it just hangs there. When I use the example video URL from the official example (https://raw.githubusercontent.com/ffmpegwasm/testdata/master/video-15s.avi), it doesn't hang and it completes the function and returns me a blob URL in the same format as that first blob URL I showed you guys and this is what the ffmpeg output looks like in my console in that case :

  


  Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] frame P:160   Avg QP:23.62  size:  1512
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] frame B:385   Avg QP:26.75  size:   589
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] consecutive B-frames:  5.5%  3.6%  0.0% 90.9%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] mb I  I16..4: 12.6% 87.4%  0.0%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] mb P  I16..4:  3.8% 47.5%  1.6%  P16..4: 12.9%  7.4%  5.0%  0.0%  0.0%    skip:21.7%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] mb B  I16..4:  1.2% 10.3%  0.4%  B16..8: 22.3%  6.9%  1.4%  direct: 2.7%  skip:54.8%  L0:46.9% L1:40.2% BI:12.9%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] 8x8 transform intra:88.7% inter:74.7%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] coded y,uvDC,uvAC intra: 68.3% 0.0% 0.0% inter: 11.8% 0.0% 0.0%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] i16 v,h,dc,p: 33% 40% 24%  3%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 15% 26% 52%  2%  1%  1%  1%  1%  3%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 27% 21% 20%  5%  5%  5%  4%  6%  5%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] i8c dc,h,v,p: 100%  0%  0%  0%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] Weighted P-Frames: Y:12.5% UV:0.0%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] ref P L0: 48.9% 12.5% 22.3% 14.7%  1.6%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] ref B L0: 77.5% 15.7%  6.8%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] ref B L1: 90.9%  9.1%
Post.jsx:35 [FFMPEG stderr] [libx264 @ 0xdf3000] kb/s:242.65
Post.jsx:35 [FFMPEG stderr] Aborted()


  


  Where am I going wrong, what should I convert my input blob into ? And just FYI, ChatGPT has been completely garbage at helping me solve this lmao.