Recherche avancée

Sur d’autres sites (12969)

• memory leak reading video frames to numpy array using ffmpeg as a python subprocess

  19 novembre 2023, par paddyg

  I can stream videos frame by frame to an OpenGL Texture2D OK in python (pi3d module, example in pi3d_demos/VideoWalk.py) but I've noticed that it gradually leaks memory. Below is a stripped down version of the code that shows the problem.

  


  Can anyone see where I'm leaking ? The memory seems to be recovered when python stops. I've tried explicitly setting things to None or calling the garbage collector manually.

  


  #!/usr/bin/python
import os
import numpy as np
import subprocess
import threading
import time
import json

def get_dimensions(video_path):
    probe_cmd = f'ffprobe -v error -show_entries stream=width,height,avg_frame_rate -of json "{video_path}"'
    probe_result = subprocess.check_output(probe_cmd, shell=True, text=True)
    video_info_list = [vinfo for vinfo in json.loads(probe_result)['streams'] if 'width' in vinfo]
    if len(video_info_list) > 0:
        video_info = video_info_list[0] # use first if more than one!
        return(video_info['width'], video_info['height'])
    else:
        return None

class VideoStreamer:
    def __init__(self, video_path):
        self.flag = False # use to signal new texture
        self.kill_thread = False
        self.command = [ 'ffmpeg', '-i', video_path, '-f', 'image2pipe',
                        '-pix_fmt', 'rgb24', '-vcodec', 'rawvideo', '-']
        dimensions = get_dimensions(video_path)
        if dimensions is not None:
            (self.W, self.H) = dimensions
            self.P = 3
            self.image = np.zeros((self.H, self.W, self.P), dtype='uint8')
            self.t = threading.Thread(target=self.pipe_thread)
            self.t.start()
        else: # couldn't get dimensions for some reason - assume not able to read video
            self.W = 240
            self.H = 180
            self.P = 3
            self.image = np.zeros((self.H, self.W, self.P), dtype='uint8')
            self.t = None

    def pipe_thread(self):
        pipe = None
        while not self.kill_thread:
            st_tm = time.time()
            if pipe is None:
                pipe = subprocess.Popen(self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1)
            self.image = np.frombuffer(pipe.stdout.read(self.H * self.W * self.P), dtype='uint8') # overwrite array
            pipe.stdout.flush() # presumably nothing else has arrived since read()
            pipe.stderr.flush() # ffmpeg sends commentary to stderr
            if len(self.image) < self.H * self.W * self.P: # end of video, reload
                pipe.terminate()
                pipe = None
            else:
                self.image.shape = (self.H, self.W, self.P)
                self.flag = True
            step = time.time() - st_tm
            time.sleep(max(0.04 - step, 0.0)) # adding fps info to ffmpeg doesn't seem to have any effect
        if pipe is not None:
            pipe.terminate()
            pipe = None

    def kill(self):
        self.kill_thread = True
        if self.t is not None:
            self.t.join()

vs = None
try:
    while True:
        for (path, _, videos) in os.walk("/home/patrick/Pictures/videos"):
            for video in videos:
                print(video)
                os.system("free") # shows gradually declining memory available
                vs = VideoStreamer(os.path.join(path, video))
                for i in range(500):
                    tries = 0
                    while not vs.flag and tries < 5:
                        time.sleep(0.001)
                        tries += 1
                    # at this point vs.image is a numpy array HxWxP bytes
                    vs.flag = False
                vs.kill()
except KeyboardInterrupt:
    if vs is not None:
        vs.kill()


os.system("free")


  


• Python running ffmpeg with Popen causes memory leak

  13 janvier 2020, par loretoparisi

  I’m using ffmpeg in Python3 to run a command :

      process = subprocess.Popen(
  
          command,
  
          stdout=open(os.devnull, 'wb'),
  
          stdin=subprocess.PIPE,
  
          stderr=subprocess.PIPE)
  
  
  
      # Write data to STDIN.
  
      try:
  
          process.stdin.write(data.astype('code>

  where data representing a wave file, hence it is appended to stdin.
  This code has a memory leak. The memory is not being released, in fact if we do memory tracing :

      is_tracing = tracemalloc.is_tracing()
  
      if not is_tracing:
  
          nframe = 6
  
          tracemalloc.start(nframe)
  
      current_mem, peak_mem = tracemalloc.get_traced_memory()
  
      overhead = tracemalloc.get_tracemalloc_memory()
  
      summary = "traced memory: %d KiB  peak: %d KiB  overhead: %d KiB" % (
  
          int(current_mem // 1024), int(peak_mem // 1024), int(overhead // 1024)
  
      )
  
      print( "before save", summary )
  
  
  
      process = subprocess.Popen(
  
          command,
  
          stdout=open(os.devnull, 'wb'),
  
          stdin=subprocess.PIPE,
  
          stderr=subprocess.PIPE)
  
  
  
      try:
  
          process.stdin.write(data.astype('/ 1024), int(peak_mem // 1024), int(overhead // 1024)
  
      )
  
      print( "after save", summary )

  and running it more times it will give

  after save traced memory: 18 KiB  peak: 1419 KiB  overhead: 10 KiB
  
  before save traced memory: 27459 KiB  peak: 28152 KiB  overhead: 28293 KiB
  
  after save traced memory: 27384 KiB  peak: 28872 KiB  overhead: 28267 KiB
  
  before save traced memory: 52707 KiB  peak: 53400 KiB  overhead: 53132 KiB
  
  after save traced memory: 52653 KiB  peak: 54120 KiB  overhead: 53109 KiB

  Specifically the ffmpeg command executed was

       command = (
  
          self._get_command_builder()
  
          .flag('-y')
  
          .opt('-loglevel', 'error')
  
          .opt('-f', 'f32le')
  
          .opt('-ar', sample_rate)
  
          .opt('-ac', data.shape[1])
  
          .opt('-i', '-')
  
          .flag('-vn')
  
          .opt('-acodec', codec)
  
          .opt('-ar', sample_rate)
  
          .opt('-strict', '-2')
  
          .opt('-ab', bitrate)
  
          .flag(path)
  
          .command())

  Which is the variabile that leaks memory ? I have tried to clean data and flush the stdin without any success.
  I have also tried to run within a threading.Thread like :

  class MyClass(threading.Thread):
  
      def __init__(self, *args):
  
          self.stdout = None
  
          self.stderr = None
  
          self.args = args
  
          threading.Thread.__init__(self)
  
  
  
      def run(self):
  
          command = self.args[ 0 ]
  
          data = self.args[ 1 ]
  
          process = subprocess.Popen(
  
              command,
  
              stdout=open(os.devnull, 'wb'),
  
              stdin=subprocess.PIPE,
  
              stderr=subprocess.PIPE)
  
          try:
  
              process.stdin.write(data.astype('code>

  and then

  myclass = MyClass(command, data)
  
  myclass.start()
  
  myclass.join()

• Revision 36454 : uniformiser les inputs du formulaire de login

  19 mars 2010, par brunobergot@… — Log

  uniformiser les inputs du formulaire de login