Recherche avancée

Sur d’autres sites (8987)

• Révision 21930 : Correction de quelques warnings php sur l’installation de SPIP avec Mysqli (et Ma...

  9 mars 2015, par marcimat ☺☮☯♫

  1) éviter un warning si l’on se trompe de mot de passe de connexion à l’installation pour notre utilisateur.

  2) il semble que mariadb sous linux autorise tout utilisateur à se connecter à localhost sans mot de passe.
  Il a alors des droits USAGE sur la base de données ’test’ si elle est présente (c’est le cas par défaut),
  mais il ne peut pas créer de nouvelles base de données. On retourne maintenant une erreur indiquant la situation
  si la connexion mysqli a réussie (par exemple si on ne renseigne pas de mot de passe de connexion),
  et qu’on tente de créer une nouvelle base de données à l’installation, le tout en évitant une foultitude de notices php que cela générait.

  Il reste le fait qu’il semble que par défaut, en tout cas sous ubuntu/mariadb/mysqli choisi, on peut utiliser,
  avec une connexion locale uniquement (localhost), n’importe quel nom d’utilisateur,
  sans mot de passe, et installer le spip dans la base ’test’ !

• Evolution #4102 (En cours) : Ordre des inclures dans cache/charger_plugins_options.php

  24 décembre 2018, par cam.lafit -

  Salut

  Je me permet de réouvrir le ticket car en fait on ne regle pas le problème de fond.

  Je suis tombé sur le cas où j’ai installé bêtement les plugins au fil de l’eau. Du fait de leur nommage, le second plugin ne peut surcharger le define et on ne peut demander de bidouiller un config/mes_options.php pour palier à ce défaut.

  Je pense qu’on devrait pouvoir soit charger par ordre inverse (de l’arbre de dépendance) les fichiers options des plugins ou proposer une alternative à la méthode define / is_defined qui permettrait de prendre en compte les dépendances des plugins et rendre opérant les meta plugins.

• Ffmpeg encoding too slow

  8 mars 2024, par Marc Cuadras

  I'm using a Python script to encode mp4 videos in various qualities, in my case 360 +720 +1080.
Ffmpeg encoding videos very slow, 1 video taking 2 hours or more, I'm using good dedicated server from hetzner (ryzen 5 3600 + 64gm ram), any suggestion to improve speed will be really appreciated

  


  import os
import glob
from datetime import datetime
import subprocess
from rich import print
import time
import sys
import pymysql as sql
import move

MYSQL_HOST = "127.0.0.1"
MYSQL_USER = ""
MYSQL_PASSWORD = ""
MYSQL_DB = ""

BASE_PATH = ''
UPLOAD_PATH = os.path.join(BASE_PATH, 'upload')
UPLOAD_PATH2 = os.path.join(BASE_PATH, 'upload2')
VIDEO_PATH = os.path.join(BASE_PATH, 'video')
LOGO_PATH = os.path.join(BASE_PATH, 'logo.png')
ERROR_PATH = os.path.join(BASE_PATH, 'error')
RCLONE_PATH = os.path.join(BASE_PATH, 'rclone')
WAIT = 60
VIDEO_LENGTH = 10
MOVE_CMD = "screen -dmS move python3 move.py --folder {}"

global current_video_id

db = sql.connect(
        host=MYSQL_HOST,
        user=MYSQL_USER,
        password=MYSQL_PASSWORD,
        database=MYSQL_DB
    )

def executedb(query):
    cursor = db.cursor()
    cursor.execute(query)
    db.commit()

def time_now():
    now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    return now

def write_log(msg):
    log_msg = f"{time_now()}: {msg}"
    with open('log.txt','a') as f:
        f.write(log_msg)
        f.write('\n')

def change_extension(files):
    for file in files:
        os.rename(file, file.split('.')[0] + '.mkv')

def change_all_videos_extension():
    # changing all the vidoe's extension to mkv format
    MKV_files = glob.glob('*.MKV')
    AVI_files = glob.glob('*.AVI')
    avi_files = glob.glob('*.avi')
    MP4_files = glob.glob('*.MK4')
    mp4_files = glob.glob('*.mp4')
    webm_files = glob.glob('*.webm')
    ts_files = glob.glob('*.ts')

    if len(avi_files) > 0:
        print(f'[{time_now()}] Converting avi videos to mkv format')
        change_extension(avi_files)

    if len(MKV_files) > 0:
        print(f'[{time_now()}] Converting MKV videos to mkv format')
        change_extension(MKV_files)

    if len(AVI_files) > 0:
        print(f'[{time_now()}] Converting AVI videos to mkv format')
        change_extension(AVI_files)

    if len(MP4_files) > 0:
        print(f'[{time_now()}] Converting MP4 videos to mkv format')
        change_extension(MP4_files)

    if len(mp4_files) > 0:
        print(f'[{time_now()}] Converting mp4 videos to mkv format')
        change_extension(mp4_files)

    if len(webm_files) > 0:
        print(f'[{time_now()}] Converting webm videos to mkv format')
        change_extension(webm_files)

    if len(ts_files) > 0:
        print(f'[{time_now()}] Converting ts videos to mkv format')
        change_extension(ts_files)
        
def encode_480(filename):
    FILENAME_PATH = filename
    newname = filename.split('.')[0]
    newname_path = os.path.join(VIDEO_PATH, newname)

    poster_cmd = f'ffmpeg -y -ss  00:00:10 -i {FILENAME_PATH} -vframes  1 -q:v  2 poster.jpg'
    os.system(poster_cmd)
    
    if not os.path.exists(newname_path):
        os.mkdir(newname_path)

    os.replace('poster.jpg', os.path.join(newname_path, 'poster.jpg'))
    

    ffmpeg480_cmd = f'ffmpeg -hide_banner -y -i {FILENAME_PATH} -sn -c:a aac -ac 2 -c:v libx264 -crf 23 -preset fast -sc_threshold 0 -g 48 -keyint_min 48 -hls_time 4 -hls_playlist_type vod -maxrate 1024k -bufsize 1536k -b:a 128k -pix_fmt yuv420p -hls_segment_filename 4835JRK9%03d.ts 480p.m3u8'
    os.system(ffmpeg480_cmd)
    
    # os.remove(FILENAME_PATH)
    
    ts_files = glob.glob('*.ts')
    for ts_file in ts_files:
        os.replace(ts_file, os.path.join(newname_path, ts_file))
    
    master_text = '''#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=854x480
480p.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=840000,RESOLUTION=640x360
360p.m3u8
'''
    with open('master.m3u8', 'w') as f:
        f.write(master_text)
    
    m3u8_files = glob.glob('*.m3u8')
    for m3u8_file in m3u8_files:
        os.replace(m3u8_file, os.path.join(newname_path, m3u8_file))


def encode_360(filename):
    FILENAME_PATH = filename
    newname = filename.split('.')[0]
    newname_path = os.path.join(VIDEO_PATH,newname)
    poster_cmd = f'ffmpeg -y -ss 00:00:10 -i {FILENAME_PATH} -vframes 1 -q:v 2 poster.jpg'
    os.system(poster_cmd)
    if not os.path.exists(newname_path):
        os.mkdir(newname_path)
    os.replace('poster.jpg',os.path.join(newname_path, 'poster.jpg'))

    
    ffmpeg360_cmd = f'ffmpeg -hide_banner -y -i {FILENAME_PATH} -sn -c:a aac -ac 2 -c:v libx264 -crf 23 -preset fast -sc_threshold 0 -g 48 -keyint_min 48 -hls_time 4 -hls_playlist_type vod -maxrate 512k -bufsize 768k -b:a 128k -pix_fmt yuv420p -hls_segment_filename 365RL6TJ%03d.ts 360p.m3u8'
    os.system(ffmpeg360_cmd)
    # return
    
    
    ts_files = glob.glob('*.ts')
    for ts_file in ts_files:
        os.replace(ts_file, os.path.join(newname_path,ts_file))
    master_text = '''#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=840000,RESOLUTION=640x360
360p.m3u8
    '''
    with open('master.m3u8', 'w') as f:
        f.write(master_text)
    m3u8_files = glob.glob('*.m3u8')
    for m3u8_file in m3u8_files:
        os.replace(m3u8_file, os.path.join(newname_path, m3u8_file))


def encode_720(filename):
    FILENAME_PATH = filename
    newname = filename.split('.')[0]
    newname_path = os.path.join(VIDEO_PATH,newname)
    poster_cmd = f'ffmpeg -y -ss 00:00:10 -i {FILENAME_PATH} -vframes 1 -q:v 2 poster.jpg'
    os.system(poster_cmd)

    if not os.path.exists(newname_path):
        os.mkdir(newname_path)

    os.replace('poster.jpg',os.path.join(newname_path, 'poster.jpg'))

    ffmpeg720_cmd = f'ffmpeg -hide_banner -y -i {FILENAME_PATH} -sn -c:a aac -ac 2 -c:v libx264 -crf 23 -preset fast -sc_threshold 0 -g 48 -keyint_min 48 -hls_time 4 -hls_playlist_type vod -maxrate 2048k -bufsize 3072k -b:a 160k -pix_fmt yuv420p -hls_segment_filename 7269TKL0%03d.ts 720p.m3u8'
    os.system(ffmpeg720_cmd)

    # os.remove(FILENAME_PATH)

    ts_files = glob.glob('*.ts')
    for ts_file in ts_files:
        os.replace(ts_file, os.path.join(newname_path,ts_file))
        
    m3u8_text = '''#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
720p.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=840000,RESOLUTION=640x360
360p.m3u8        
        '''

    with open('master.m3u8','w') as f:
        f.write(m3u8_text)
            
    m3u8_files = glob.glob('*.m3u8')
    for m3u8_file in m3u8_files:
        os.replace(m3u8_file, os.path.join(newname_path,m3u8_file))  


def encode_1080(filename):
    FILENAME_PATH = filename
    newname = filename.split('.')[0]
    newname_path = os.path.join(VIDEO_PATH, newname)
    poster_cmd = f'ffmpeg -y -ss 00:00:10 -i {FILENAME_PATH} -vframes 1 -q:v 2 poster.jpg'
    os.system(poster_cmd)

    if not os.path.exists(newname_path):
        os.mkdir(newname_path)

    os.replace('poster.jpg', os.path.join(newname_path, 'poster.jpg'))

    ffmpeg1080_cmd = f'ffmpeg -hide_banner -y -i {FILENAME_PATH} -sn -c:a aac -ac 2 -c:v libx264 -crf 23 -preset fast -sc_threshold 0 -g 48 -keyint_min 48 -hls_time 4 -hls_playlist_type vod -maxrate 4000k -bufsize 6000k -b:a 192k -pix_fmt yuv420p -hls_segment_filename 108YUT8T%03d.ts 1080p.m3u8'
    os.system(ffmpeg1080_cmd)

    # os.remove(FILENAME_PATH)

    ts_files = glob.glob('*.ts')
    for ts_file in ts_files:
        os.replace(ts_file, os.path.join(newname_path, ts_file))

    m3u8_text = '''#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1920x1080
1080p.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=2800000,RESOLUTION=1280x720
720p.m3u8
#EXT-X-STREAM-INF:BANDWIDTH=840000,RESOLUTION=640x360
360p.m3u8
'''

    with open('master.m3u8', 'w') as f:
        f.write(m3u8_text)

    m3u8_files = glob.glob('*.m3u8')
    for m3u8_file in m3u8_files:
        os.replace(m3u8_file, os.path.join(newname_path, m3u8_file))



def vod(filename, resolution):
    os.replace(filename,filename.replace(' ','_'))
    filename = filename.replace(' ','_')
    FILENAME_PATH = filename
    width_cmd = f'ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=nw=1:nk=1 {FILENAME_PATH}'
    height_cmd= f'ffprobe -v error -select_streams v:0 -show_entries stream=height -of default=nw=1:nk=1 {FILENAME_PATH}'

    width_result = subprocess.run(width_cmd.split(), stdout=subprocess.PIPE)
    width = width_result.stdout.strip().decode('utf-8')

    height_result = subprocess.run(height_cmd.split(), stdout=subprocess.PIPE)
    height = height_result.stdout.strip().decode('utf-8')

    if not os.path.exists(VIDEO_PATH):
        os.mkdir(VIDEO_PATH)

    if resolution == 360: 
        write_log(f'Encoding {filename} in 360p') 
        encode_360(filename)

    if resolution == 720 :
        if int(height) >=400 :
            if int(height) <= 700:
                query = f"""UPDATE videos SET encoding_status = 'encoding480' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                executedb(query)
                write_log(f'Encoding {filename} in 480p')
                encode_480(filename) 
                query = f"""UPDATE videos SET encoding_status = 'done480' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                executedb(query)
                query = f"""UPDATE videos SET quality = 2 WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                executedb(query)
            else:
                if int(height) >= 800:
                    query = f"""UPDATE videos SET encoding_status = 'encoding720' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    write_log(f'Encoding {filename} in 720p')
                    encode_720(filename)   
                    query = f"""UPDATE videos SET encoding_status = 'done720' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    query = f"""UPDATE videos SET quality = 3 WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    query = f"""UPDATE videos SET encoding_status = 'encoding1080' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    write_log(f'Encoding {filename} in 720p')
                    encode_1080(filename)   
                    query = f"""UPDATE videos SET encoding_status = 'done1080' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    query = f"""UPDATE videos SET quality = 4 WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                else:
                    query = f"""UPDATE videos SET encoding_status = 'encoding720' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    write_log(f'Encoding {filename} in 720p')
                    encode_720(filename)   
                    query = f"""UPDATE videos SET encoding_status = 'done720' WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                    query = f"""UPDATE videos SET quality = 3 WHERE file_uid LIKE '%{filename.replace(".mkv","")}'"""
                    executedb(query)
                
        
            
        os.remove(filename)


def move_to_rclone():
    if not os.path.exists(RCLONE_PATH):
        os.mkdir(RCLONE_PATH)
    folders = os.listdir(VIDEO_PATH)
    if len(folders) > 0:
        for folder in folders:
            folder_in_rclone = os.path.join(RCLONE_PATH, folder)
            if not os.path.exists(folder_in_rclone):
                os.mkdir(folder_in_rclone)
            files = os.listdir(os.path.join(VIDEO_PATH, folder))
            for file in files:
                os.replace(os.path.join(VIDEO_PATH, folder, file), os.path.join(folder_in_rclone, file))
            os.system(MOVE_CMD.format(folder_in_rclone))
        if len(os.listdir(os.path.join(VIDEO_PATH, folder))) == 0:
            os.rmdir(os.path.join(VIDEO_PATH, folder))
    # rclone_folders = os.listdir(RCLONE_PATH)
    # if len(rclone_folders)> 0:
    #     for rclone_folder in rclone_folders:
    #         os.system(MOVE_CMD.format(rclone_folder))

def get_length(input_video):
    result = subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', input_video], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    try:
        output = float(result.stdout)
    except:
        output = 0
    return output

def move_to_error(file):
    if not os.path.exists(ERROR_PATH):
        os.mkdir(ERROR_PATH)
    print(f'[red][-][/red] Moving {file} to the error folder')
    os.replace(file, os.path.join(ERROR_PATH, file.split('/')[-1]))

def main():
    db = sql.connect(
        host=MYSQL_HOST,
        user=MYSQL_USER,
        password=MYSQL_PASSWORD,
        database=MYSQL_DB
    )

    def get_video_data(file_uid):
        cursor = db.cursor()
        query = f"""SELECT id, post_title, file_uid, group_id FROM videos WHERE file_uid LIKE %s"""
        cursor.execute(query, (f'%{file_uid}%',))
        data = cursor.fetchone()
        return data if data else None
    
    if os.path.exists(UPLOAD_PATH):
        upload_files = os.listdir(UPLOAD_PATH)
        upload_files = [os.path.join(UPLOAD_PATH, x) for x in upload_files]
        upload_files.sort(key=lambda x: os.path.getmtime(x))
    else:
        upload_files = []

    if len(upload_files) > 0:
        for upload_file in upload_files:
            if get_length(upload_file) < VIDEO_LENGTH:
                move_to_error(upload_file)
                write_log(f'Length of {upload_file} is less than 60 sec.  Moving to error folder')
                continue
            try:
                os.replace(upload_file, upload_file.split('/')[-1])
                query = f"""UPDATE videos SET encoding_status = 'uploaded' WHERE file_uid LIKE '%{upload_file.replace(".mkv","").replace(".mp4","")}'"""
                executedb(query)
            except:
                print(e)
                continue
            change_all_videos_extension()

            mkv_files = glob.glob('*.mkv')
            if len(mkv_files) > 0:
                for mkv_file in mkv_files:
                    try:
                        query = f"""UPDATE videos SET encoding_status = 'encoding360' WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)
                        vod(mkv_file, 360)
                        query = f"""UPDATE videos SET encoding_status = 'done360' WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)
                        query = f"""UPDATE videos SET quality = 1 WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)

                        
                        vod(mkv_file, 720)
                        
                    except Exception as e:
                        query = f"""UPDATE videos SET encoding_status = 'error' WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)
                        video_data = get_video_data(
                            mkv_file.replace(".mkv", ""))
                        if video_data:
                            cursor = db.cursor()
                            video_id, video_title, video_uid, group_id = video_data
                            error_log_query = f"""INSERT INTO error_logs (video_id, video_title, video_uid, group_id, log, created_at, updated_at) 
                                                VALUES (%s, %s, %s, %s, %s, NOW(), NOW())"""
                            cursor.execute(
                                error_log_query, (video_id, video_title, video_uid, group_id, str(e)))
                            db.commit()
                        write_log(f'Error: {e}')
                        move_to_error(mkv_file)
                    move_to_rclone()
    else:
        print(f'[{time_now()}] No new video found on upload folder')

    if os.path.exists(UPLOAD_PATH2):
        upload_files2 = os.listdir(UPLOAD_PATH2)
        upload_files2 = [os.path.join(UPLOAD_PATH2, x) for x in upload_files2]
        upload_files2.sort(key=lambda x: os.path.getmtime(x))
    else:
        upload_files2 = []

    if len(upload_files2) > 0:
        for upload_file2 in upload_files2:
            if get_length(upload_file2) < VIDEO_LENGTH:
                move_to_error(upload_file2)
                continue
            if len(os.listdir(UPLOAD_PATH)) != 0:
                main()
            try:
                os.replace(upload_file2, upload_file2.split('/')[-1])
                query = f"""UPDATE videos SET encoding_status = 'uploaded' WHERE file_uid LIKE '%{upload_file2.replace(".mkv","").replace(".mp4","")}'"""
                executedb(query)
            except:
                continue

            change_all_videos_extension()

            mkv_files = glob.glob('*.mkv')
            if len(mkv_files) > 0:
                for mkv_file in mkv_files:
                    try:
                        query = f"""UPDATE videos SET encoding_status = 'encoding360' WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)
                        vod(mkv_file, 360)
                        query = f"""UPDATE videos SET encoding_status = 'done360' WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)
                        query = f"""UPDATE videos SET quality = 1 WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)


                        vod(mkv_file, 720)

                    except Exception as e:
                        query = f"""UPDATE videos SET encoding_status = 'error' WHERE file_uid LIKE '%{mkv_file.replace(".mkv","")}'"""
                        executedb(query)
                        video_data = get_video_data(
                            mkv_file.replace(".mkv", ""))
                        if video_data:
                            cursor = db.cursor()
                            video_id, video_title, video_uid, group_id = video_data
                            error_log_query = f"""INSERT INTO error_logs (video_id, video_title, video_uid, group_id, log, created_at, updated_at) 
                                                VALUES (%s, %s, %s, %s, %s, NOW(), NOW())"""
                            cursor.execute(
                                error_log_query, (video_id, video_title, video_uid, group_id, str(e)))
                            db.commit()
                        write_log(f'Error: {e}')
                        move_to_error(mkv_file)
            move_to_rclone()
    else:
        print(f'[{time_now()}] No new video found on upload2 folder.')
    move_to_rclone()
    db.close()

if __name__=="__main__":
    while True:
        try:
            main()
        except Exception as e:
            print(f'Error: {e}')
        for i in range(WAIT):
            print(f'[green][+][/green] Waiting for {WAIT-i} seconds  ', end="\r")
            time.sleep(1)




  


  looking for suggestion to improve encoding speed