Recherche avancée

Sur d’autres sites (8056)

• Live audio using ffmpeg, javascript and nodejs

  8 novembre 2017, par klaus

  I am new to this thing. Please don’t hang me for the poor grammar. I am trying to create a proof of concept application which I will later extend. It does the following : We have a html page which asks for permission to use the microphone. We capture the microphone input and send it via websocket to a node js app.

  JS (Client) :

  var bufferSize = 4096;
  
  var socket = new WebSocket(URL);
  
  var myPCMProcessingNode = context.createScriptProcessor(bufferSize, 1, 1);
  
  myPCMProcessingNode.onaudioprocess = function(e) {
  
    var input = e.inputBuffer.getChannelData(0);
  
    socket.send(convertFloat32ToInt16(input));
  
  }
  
  
  
  function convertFloat32ToInt16(buffer) {
  
    l = buffer.length;
  
    buf = new Int16Array(l);
  
    while (l--) {
  
      buf[l] = Math.min(1, buffer[l])*0x7FFF;
  
    }
  
    return buf.buffer;
  
  }
  
  
  
  navigator.mediaDevices.getUserMedia({audio:true, video:false})
  
                                  .then(function(stream){
  
                                    var microphone = context.createMediaStreamSource(stream);
  
                                    microphone.connect(myPCMProcessingNode);
  
                                    myPCMProcessingNode.connect(context.destination);
  
                                  })
  
                                  .catch(function(e){});

  In the server we take each incoming buffer, run it through ffmpeg, and send what comes out of the std out to another device using the node js ’http’ POST. The device has a speaker. We are basically trying to create a 1 way audio link from the browser to the device.

  Node JS (Server) :

  var WebSocketServer = require('websocket').server;
  
  var http = require('http');
  
  var children = require('child_process');
  
  
  
  wsServer.on('request', function(request) {
  
    var connection = request.accept(null, request.origin);
  
    connection.on('message', function(message) {
  
      if (message.type === 'utf8') { /*NOP*/ }
  
      else if (message.type === 'binary') {
  
        ffm.stdin.write(message.binaryData);
  
      }
  
    });
  
    connection.on('close', function(reasonCode, description) {});
  
    connection.on('error', function(error) {});
  
  });
  
  
  
  var ffm = children.spawn(
  
      './ffmpeg.exe'
  
     ,'-stdin -f s16le -ar 48k -ac 2 -i pipe:0 -acodec pcm_u8 -ar 48000 -f aiff pipe:1'.split(' ')
  
  );
  
  
  
  ffm.on('exit',function(code,signal){});
  
  
  
  ffm.stdout.on('data', (data) => {
  
    req.write(data);
  
  });
  
  
  
  var options = {
  
    host: 'xxx.xxx.xxx.xxx',
  
    port: xxxx,
  
    path: '/path/to/service/on/device',
  
    method: 'POST',
  
    headers: {
  
     'Content-Type': 'application/octet-stream',
  
     'Content-Length': 0,
  
     'Authorization' : 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
  
     'Transfer-Encoding' : 'chunked',
  
     'Connection': 'keep-alive'
  
    }
  
  };
  
  
  
  var req = http.request(options, function(res) {});

  The device supports only continuous POST and only a couple of formats (ulaw, aiff, wav)

  This solution doesn’t seem to work. In the device speaker we only hear something like white noise.

  Also, I think I may have a problem with the buffer I am sending to the ffmpeg std in -> Tried to dump whatever comes out of the websocket to a .wav file then play it with VLC -> it plays everything in the record very fast -> 10 seconds of recording played in about 1 second.

  I am new to audio processing and have searched for about 3 days now for solutions on how to improve this and found nothing.

  I would ask from the community for 2 things :

  1. Is something wrong with my approach ? What more can I do to make this work ? I will post more details if required.

  2. If what I am doing is reinventing the wheel then I would like to know what other software / 3rd party service (like amazon or whatever) can accomplish the same thing.

  Thank you.

• dnn_backend_native_layer_mathunary : add abs support

  25 mai 2020, par Ting Fu

  dnn_backend_native_layer_mathunary : add abs support
  more math unary operations will be added here
  It can be tested with the model file generated with below python scripy :
  import tensorflow as tf
  
  import numpy as np
  
  import imageio
  in_img = imageio.imread('input.jpeg')
  
  in_img = in_img.astype(np.float32)/255.0
  
  in_data = in_img[np.newaxis, :]
  x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in')
  
  x1 = tf.subtract(x, 0.5)
  
  x2 = tf.abs(x1)
  
  y = tf.identity(x2, name='dnn_out')
  sess=tf.Session()
  
  sess.run(tf.global_variables_initializer())
  graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out'])
  
  tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False)
  print("image_process.pb generated, please use \
  
  path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n")
  output = sess.run(y, feed_dict=x : in_data)
  
  imageio.imsave("out.jpg", np.squeeze(output))
  Signed-off-by : Ting Fu <ting.fu@intel.com>
  
  Signed-off-by : Guo, Yejun <yejun.guo@intel.com>
  

  • [DH] libavfilter/dnn/Makefile
  • [DH] libavfilter/dnn/dnn_backend_native.h
  • [DH] libavfilter/dnn/dnn_backend_native_layer_mathunary.c
  • [DH] libavfilter/dnn/dnn_backend_native_layer_mathunary.h
  • [DH] libavfilter/dnn/dnn_backend_native_layers.c
  • [DH] tools/python/convert_from_tensorflow.py
  • [DH] tools/python/convert_header.py

• vf_dnn_processing : add support for more formats gray8 and grayf32

  27 décembre 2019, par Guo, Yejun

  vf_dnn_processing : add support for more formats gray8 and grayf32
  The following is a python script to halve the value of the gray
  
  image. It demos how to setup and execute dnn model with python+tensorflow.
  
  It also generates .pb file which will be used by ffmpeg.
  import tensorflow as tf
  
  import numpy as np
  
  from skimage import color
  
  from skimage import io
  
  in_img = io.imread('input.jpg')
  
  in_img = color.rgb2gray(in_img)
  
  io.imsave('ori_gray.jpg', np.squeeze(in_img))
  
  in_data = np.expand_dims(in_img, axis=0)
  
  in_data = np.expand_dims(in_data, axis=3)
  
  filter_data = np.array([0.5]).reshape(1,1,1,1).astype(np.float32)
  
  filter = tf.Variable(filter_data)
  
  x = tf.placeholder(tf.float32, shape=[1, None, None, 1], name='dnn_in')
  
  y = tf.nn.conv2d(x, filter, strides=[1, 1, 1, 1], padding='VALID', name='dnn_out')
  
  sess=tf.Session()
  
  sess.run(tf.global_variables_initializer())
  
  graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out'])
  
  tf.train.write_graph(graph_def, '.', 'halve_gray_float.pb', as_text=False)
  
  print("halve_gray_float.pb generated, please use \
  
  path_to_ffmpeg/tools/python/convert.py to generate halve_gray_float.model\n")
  
  output = sess.run(y, feed_dict=x : in_data)
  
  output = output * 255.0
  
  output = output.astype(np.uint8)
  
  io.imsave("out.jpg", np.squeeze(output))
  To do the same thing with ffmpeg :
  
   generate halve_gray_float.pb with the above script
  
   generate halve_gray_float.model with tools/python/convert.py
  
   try with following commands
  
    ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png
  
    ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow out.tf.png
  Signed-off-by : Guo, Yejun <yejun.guo@intel.com>
  
  Signed-off-by : Pedro Arthur <bygrandao@gmail.com>
  

  • [DH] doc/filters.texi
  • [DH] libavfilter/vf_dnn_processing.c