Recherche avancée

Sur d’autres sites (10347)

• ffmpeg/c++ Encode additional information of video frame with ffmpeg

  29 janvier 2018, par 8793

  I am new with ffmpeg & video encoding, after looking for some related questions on this page, I found this post which is very useful to understand the overview process of ffmpeg.

  However, my work not only needs to manipulate with Mat frame, after extract important information from video (extract edge, position of edge block, type of each edge block, block number, motion vector), I have to encode and send them to client. I tried to find an example code for this part but it seems nobody have done it before.

  My problems is how to encode these additional information along with video frame, and send both to client. I read about Huffman Coding which can help lossless compression, But is it possible encode edge & motion data using huffman coding while encoding video frame using ffmpeg ? I’m doing experiment using udp protocol.

  I can not find any information about this.
  I read into metadata & side information in ffmpeg but it’s not what I want to do.

  I hope if you can give me an advice or a directions to research into this area, so I can understand and try to implement it. If there is any example code for this case, I would be very grateful for your sharing.

  Thank you so much.

  Below is encoder part on server side :

  int encode(Mat& input_frame, EncodedCallback callback, void* userdata = nullptr) {
  
      AVPacket pkt;
  
      /* encode 1 second of video */
  
      av_init_packet(&pkt);
  
      pkt.data = NULL; // packet data will be allocated by the encoder
  
      pkt.size = 0;
  
      int size = 0;
  
      fflush(stdout);
  
      cvtFrame2AVFrameYUV420(input_frame, &frame);
  
  
  
      static int time; 
  
      frame->pts = time++;
  
      /* encode the image */
  
      ret = avcodec_send_frame(c, frame);
  
      if (ret < 0) {
  
          fprintf(stderr, "Error avcodec_send_frame\n");
  
          exit(1);
  
      }
  
      nbFramesEncoded++;
  
      ret = avcodec_receive_packet(c, &pkt);
  
      if (!isFirstFrameEmmited) {
  
          nbNeededFramesInBuffer++;
  
  
  
          printf("nbNeededFramesInBuffer: %d\n", nbNeededFramesInBuffer);
  
      }
  
      if (ret < 0) {
  
          if (ret == -EAGAIN) {
  
              //output is not available, we must send more input
  
          } else {
  
              fprintf(stderr, "Error avcodec_receive_packet %d\n", ret);
  
              exit(1);
  
          }
  
      } else {
  
  
  
          if (callback) {
  
              callback(pkt, userdata);
  
          }
  
          size = pkt.size + 4;
  
          av_packet_unref(&pkt);
  
      }
  
  
  
      return size;
  
  }

  Below is code to handle frame processing (presently we check & send motioned block to client)

  void updateFrame(Mat& frame) {
  
      //Get all Streams ready
  
      bool isReady = true;
  
      if (!frameStreamer->encoder->isFirstFrameEmmited) {
  
          frameStreamer->sendFrame(frame);
  
          isReady = false;
  
      }
  
      for (int yidx = 0; yidx < gridSize.height; yidx++) {
  
          for (int xidx = 0; xidx < gridSize.width; xidx++) {
  
              StreamPtr& stream = streamGrid[yidx][xidx];
  
              if (!stream->encoder->isFirstFrameEmmited) {
  
                  Mat block = frame(stream->irect);
  
                  stream->sendFrame(block);
  
                  isReady = false;
  
              }
  
          }
  
      }
  
      if (isReady == false) {
  
          return;
  
      }
  
  
  
      if (pGray.empty()) {
  
  
  
          frameStreamer->sendFrame(frame);
  
          frameStreamer->sendFrame(frame);
  
  
  
          cvtColor(frame, pGray, CV_BGR2GRAY);
  
          return;
  
      }
  
  
  
      //Motion Detection
  
      Mat gray;
  
      cvtColor(frame, gray, CV_BGR2GRAY);
  
      Mat diff;
  
      absdiff(gray, pGray, diff);
  
      threshold(diff, diff, NOISE_THRESHOLD, 255, CV_THRESH_BINARY);
  
      if (HEAT_IMAGE) {
  
          gray.copyTo(diff, diff);
  
          imshow("Gray", gray);
  
          threshold(diff, diff, HEAT_THRESH, 255, CV_THRESH_TOZERO);
  
      }
  
      if (USE_MORPH_NOISE) {
  
          Morph_Noise(diff);
  
      }
  
  
  
      Mat motionImg = Mat::zeros(frameSize, CV_8UC3);
  
      //Block Classification
  
      int nbModifiedBlocks = 0;
  
      for (int yidx = 0; yidx < gridSize.height; yidx++) {
  
          for (int xidx = 0; xidx < gridSize.width; xidx++) {
  
              Rect irect(xidx * blockSize.width, yidx * blockSize.height,
  
                      blockSize.width, blockSize.height);
  
              int blockDiff = sum(diff(irect))[0];
  
              if (blockDiff > BLOCK_THRESHOLD * 255) {
  
                  this->blockCls.at<uchar>(yidx, xidx) = MODI_BLOCK;
  
                  nbModifiedBlocks++;
  
              } else {
  
                  this->blockCls.at<uchar>(yidx, xidx) = SKIP_BLOCK;
  
              }
  
          }
  
      }
  
  
  
      //Send
  
      if (nbModifiedBlocks > this->nbBlocksThresh) {
  
          nbSentBytes += this->frameStreamer->sendFrame(frame);
  
      } else {
  
          for (int yidx = 0; yidx &lt; gridSize.height; yidx++) {
  
              for (int xidx = 0; xidx &lt; gridSize.width; xidx++) {
  
                  uchar cls = this->blockCls.at<uchar>(yidx, xidx);
  
                  StreamPtr&amp; stream = streamGrid[yidx][xidx];
  
                  bool send = false;
  
                  if (cls == MODI_BLOCK) {
  
                      if (DEBUG_NETWORK) {
  
                          printf("Normal (%d, %d): ", xidx, yidx);
  
                      }
  
                      send = true;
  
                      stream->encoder->nbFramesBuffered = stream->encoder->nbNeededFramesInBuffer;
  
  
  
                      rectangle(motionImg, stream->irect, Scalar(0, 0, 255), CV_FILLED);
  
  
  
                  } else if (stream->encoder->nbFramesBuffered > 0) {
  
                      if (DEBUG_NETWORK) {
  
                          printf("Extra (%d, %d): ", xidx, yidx);
  
                      }
  
                      send = true;
  
                      stream->encoder->nbFramesBuffered--;
  
                      stream->encoder->nbFlushFrames++;
  
                      rectangle(motionImg, stream->irect, Scalar(0, 255, 0), CV_FILLED);
  
                  }
  
  
  
                  if (send) {
  
                      Mat block = frame(stream->irect);
  
                      nbSentBytes += stream->sendFrame(block);
  
                      gray(stream->irect).copyTo(pGray(stream->irect));
  
                  }
  
              }
  
          }
  
      }
  
  </uchar></uchar></uchar>

  }

• avcodec/s3tc : fix alpha decoding when dimensions are not a multiple of 4

  7 mai 2015, par Tom Butterworth

  avcodec/s3tc : fix alpha decoding when dimensions are not a multiple of 4
  Fix alpha position error for edge blocks of odd-dimensioned frames
  Signed-off-by : Michael Niedermayer <michaelni@gmx.at>
  

  • [DH] libavcodec/s3tc.c

• using pocketsphinx_continuous with a .wav file

  3 avril 2013, par user2242131

  I am attempting to write an application that will allow a user to speak a small set of commands from a remote system and have them executed on my server. Using pocketsphinx to parse the spoken text. When run locally with the microphone, pocketsphinx_continuous works perfectly no matter how I slur the words. But when importing the audio file and using ffmpeg to downsample the audio to a single channel, 16 bit PCM file, it will parse the first word without difficulty. Then it will skip everything else and treat it as . I am confident that the problem is in the file format and not in the pocketsphinx configuration.

  Using command line
  ffmpeg -y -i Sound\AddSheet.wav -ac 1 -f s16le -acodec pcm_s16le -ar 16k AddTmp.wav
  in a batch file.

  The bottom of the output I get is :

  INFO: fsg_search.c(1407): Start node ADD.0:5:47
  
  INFO: fsg_search.c(1407): Start node <sil>.0:2:49
  
  INFO: fsg_search.c(1446): End node <sil>.126:128:305 (-486)
  
  INFO: fsg_search.c(1662): lattice start node <s>.0 end node <sil>.126
  
  INFO: ps_lattice.c(1352): Normalizer P(O) = alpha(<sil>:126:305) = -175371
  
  INFO: ps_lattice.c(1390): Joint P(O,S) = -176076 P(S|O) = -705
  
  000000000: ADD USER
  
  </sil></sil></s></sil></sil>

  Which is not the audio in the file. The words spoken in the file are "ADD SPREADSHEET", which works perfectly from the same microphone without the intervening .wav file.

  I have tried increasing the audio volume and decreasing the background noise using sox :

  sox -v 3.0 Sound\%1 Sound\%1-loud.wav ffmpeg -i Sound\%1-loud.wav -vn -ss 00:00:00 -t 00:00:01 -y Sound\%1-noiseaud.wav 
  
  sox Sound\%1-noiseaud.wav -n noiseprof Sound\%1-noise.prof 
  
  sox Sound\%1 Sound\%1-clean.wav noisered sound\noise.prof 0.21 
  
  ffmpeg -y -i Sound\%1-clean.wav -ac 1 -f s16le -acodec pcm_s16le -ar 16k AddTmp.wav

  with no noticeable effect on the final results.

  If you look at the output you will notice that fsg_search.c has found ADD as the start node, then silence for the remainder. Please help on this.