Recherche avancée

Sur d’autres sites (10735)

• I have a series of images recorded by a webcam with unstable frame rate at about 22 fps. Did ffmpeg interpolate these image into other fps ? [closed]

  13 septembre 2020, par yangze68

  I have a series of images and its recorded timestamps. Their real fps is about 22 and it's unstable. If I use the FFmpeg to encode it into a 30fps video, do these really change the fps and interpolate image at an interval of 1/30 per second ? or Just arrange those images in the interval of 1/30 and decrease the total duration time ?

  


  my command :
ffmpeg -framerate 30 -i img%03d.png output.mp4

  


• Node js async/await promise problem with ytdl and ffmpeg

  31 août 2020, par WorkoutBuddy

  I built a simple youtube downloader cli. It looks like this (without any arg parsing for easier reproduction) :

  


  const ytdl = require('ytdl-core');
const config = require('config');
const progressBar = require('./progressBar');
const logger = require('./logger');
const ffmpeg = require('fluent-ffmpeg');

const url = 'https://www.youtube.com/watch?v=Np8ibIagn3M';

const getInfo = async (url) => {
    logger.info(`Getting metadata for ${url}`);
    const response = await ytdl.getBasicInfo(url);
    const info = response.videoDetails.title;
    logger.info(`Title: ${info}`);
    return info;
};

const getStream = async (url) => {
    logger.info(`Downloading from ${url} ...`);

    let allReceived = false;
    return new Promise((resolve, reject) => {
        const stream = ytdl(url, {
            quality: 'highest',
            filter: (format) => format.container === 'mp4',
        })
            .on('progress', (_, totalDownloaded, total) => {
                if (!allReceived) {
                    progressBar.start(total, 0, {
                        mbTotal: (total / 1024 / 1024).toFixed(2),
                        mbValue: 0,
                    });
                    allReceived = true;
                }
                progressBar.increment();
                progressBar.update(totalDownloaded, {
                    mbValue: (totalDownloaded / 1024 / 1024).toFixed(2),
                });
            })
            .on('end', () => {
                progressBar.stop();
                logger.info('Successfully downloaded the stream!');
            });
        return resolve(stream);
    });
};

const convertToMp3 = async (stream, title) => {
    return new Promise((resolve, reject) => {
        ffmpeg({ source: stream })
            .output(`${config.get('audioDir')}/${title}.mp3`)
            .audioBitrate('192k')
            .run();
        return resolve();
    });
};

const main = async (url) => {
    const info = await getInfo(url);
    console.log('here 1');
    const stream = await getStream(url);
    console.log('here 2');
    await convertToMp3(stream, info);
    console.log('done');
};

main(url);


  


  The output looks like :

  


  ➜ node youtube.js
info:    Getting metadata for https://www.youtube.com/watch?v=Np8ibIagn3M
info:    Title: Tu boca -  (Bachata Remix Dj Khalid)
here 1
info:    Downloading from https://www.youtube.com/watch?v=Np8ibIagn3M ...
here 2
done
[Progress] [████████████████████████████████████████] 100% | Downloaded: 7.15/7.15 MB | Elapsed Time: 5s
info:    Successfully downloaded the stream!


  


  However, I would expect this output :

  


  ➜ node youtube.js
info:    Getting metadata for https://www.youtube.com/watch?v=Np8ibIagn3M
info:    Title: Tu boca -  (Bachata Remix Dj Khalid)
here 1
info:    Downloading from https://www.youtube.com/watch?v=Np8ibIagn3M ...
[Progress] [████████████████████████████████████████] 100% | Downloaded: 7.15/7.15 MB | Elapsed Time: 5s
here 2
info:    Successfully downloaded the stream!
done


  


  I think I have troubles to understand async/await. As far as I understood, promisifying a function allows me wait for the result. However, it seems that it does not work. I do not know why and how to properly debug it.

  


  EDITED :

  


  const getStream = async (url) => {
    logger.info(`Downloading from ${url} ...`);

    let allReceived = false;
    return new Promise((resolve, reject) => {
        const stream = ytdl(url, {
            quality: 'highest',
            filter: (format) => format.container === 'mp4',
        })
            .on('progress', (_, totalDownloaded, total) => {
                console.log('totalDownloaded: ' + totalDownloaded);
                if (!allReceived) {
                    console.log('total: ' + total);
                    progressBar.start(total, 0, {
                        mbTotal: (total / 1024 / 1024).toFixed(2),
                        mbValue: 0,
                    });
                    allReceived = true;
                }
                progressBar.increment();
                progressBar.update(totalDownloaded, {
                    mbValue: (totalDownloaded / 1024 / 1024).toFixed(2),
                });
            })
            .on('end', () => {
                progressBar.stop();
                logger.info('Successfully downloaded the stream!');
                resolve(stream);
            });
    });
};


  


  But now it is like this :

  


  ➜ node youtube.js
info:    Getting metadata for https://www.youtube.com/watch?v=Np8ibIagn3M
info:    Title: Tu boca -  (Bachata Remix Dj Khalid)
here 1
info:    Downloading from https://www.youtube.com/watch?v=Np8ibIagn3M ...
[Progress] [██████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 14% | Downloaded: 1.02/7.15 MB | Elapsed Time: 52s


  


  Added console.log :

  


  totalDownloaded: 16384
total: 7493903
[Progress] [░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% | Downloaded: 0/7.15 MB | Elapsed Time: 0stotalDownloaded: 32768
totalDownloaded: 49152
totalDownloaded: 65536
totalDownloaded: 81920
totalDownloaded: 98304
totalDownloaded: 114688
totalDownloaded: 131072
totalDownloaded: 147456
totalDownloaded: 163840
totalDownloaded: 180224
totalDownloaded: 196608
totalDownloaded: 212992
totalDownloaded: 229376
totalDownloaded: 245760
totalDownloaded: 262144
totalDownloaded: 278528
totalDownloaded: 294912
totalDownloaded: 311296
totalDownloaded: 327680
totalDownloaded: 344064
totalDownloaded: 360448
totalDownloaded: 376832
totalDownloaded: 393216
totalDownloaded: 409600
totalDownloaded: 425984
totalDownloaded: 442368
totalDownloaded: 458752
totalDownloaded: 475136
totalDownloaded: 491520
totalDownloaded: 507904
totalDownloaded: 524288
totalDownloaded: 540672
totalDownloaded: 557056
totalDownloaded: 573440
totalDownloaded: 589824
totalDownloaded: 606208
totalDownloaded: 622592
totalDownloaded: 638976
totalDownloaded: 655360
totalDownloaded: 671744
totalDownloaded: 688128
totalDownloaded: 704512
totalDownloaded: 720896
totalDownloaded: 737280
totalDownloaded: 753664
totalDownloaded: 770048
totalDownloaded: 786432
totalDownloaded: 802816
totalDownloaded: 819200
totalDownloaded: 835584
totalDownloaded: 851968
totalDownloaded: 868352
totalDownloaded: 884736
totalDownloaded: 901120
totalDownloaded: 917504
totalDownloaded: 933888
totalDownloaded: 950272
totalDownloaded: 966656
totalDownloaded: 983040
totalDownloaded: 999424
totalDownloaded: 1015808
totalDownloaded: 1032192
totalDownloaded: 1048576
totalDownloaded: 1064960
[Progress] [██████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 14% | Downloaded: 1.02/7.15 MB | Elapsed Time: 25s


  


• Very slow writes on MySQL 8 - waiting for handler commit

  23 mai 2023, par Akshat Goel

  I have MySQL 8 docker installation installed on an edge device which has the following two tables to write to

  


  video_paths | CREATE TABLE `video_paths` (
  `entry` int(11) NOT NULL AUTO_INCREMENT,
  `timestamp` bigint(20) NOT NULL,
  `duration` int(11) NOT NULL,
  `path` varchar(255) NOT NULL,
  `motion` int(11) NOT NULL DEFAULT '0',
  `cam_id` varchar(255) NOT NULL DEFAULT '',
  `hd` tinyint(1) NOT NULL DEFAULT '0',
  PRIMARY KEY (`entry`),
  KEY `cam_id` (`cam_id`),
  KEY `timestamp` (`timestamp`)
) ENGINE=InnoDB AUTO_INCREMENT=7342309 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci


  


  AND

  


  CREATE TABLE `tracker` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `table_name` varchar(255) NOT NULL,
  `primary_key_name` varchar(255) NOT NULL,
  `pointer` int(11) NOT NULL DEFAULT '0',
  PRIMARY KEY (`id`),
  UNIQUE KEY `table_name` (`table_name`)
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci


  


  The following queries are run every few secs for up to 32 cameras and are taking a lot of time as indicated by the slow query log.

  


  UPDATE tracker SET pointer = 7342046 WHERE table_name = 'video_paths'

INSERT INTO video_paths (timestamp,duration,path,cam_id,hd) VALUES (1597548365000,5000,'/s/ss/x-0/v/2020-08-16/3/1.ts','x-1',1)



  


  Most of the time is spent in the waiting for handler commit state

  


  The total size of my data (tables + index) is 1GB and I have the following settings enabled to optimise for write

  


  skip-log-bin - Disabled the bin log because I don't have a replica and therefore no use for it
innodb_flush_log_at_trx_commit =2 - I am Optimising for performance rather than consistency here.
range_optimizer_max_mem_size =0 As mention in this question, I have allowed max memory to range optimiser.
inndo_buffer_pool_size= 512Mb - This should be enough for my data ?.
  
innodb_log_file_size= 96Mb *2 files

  


  I am seeing queries that are taking up to 90-100 secs sometimes.

  


  SET timestamp=1597549337;
INSERT INTO video_paths (timestamp,duration,path,cam_id,hd) VALUES (1597548365000,5000,'/s/ss/x-0/v/2020-08-16/3/1.ts','x-1',1);
# Time: 2020-08-16T03:42:24.533408Z
# Query_time: 96.712976  Lock_time: 0.000033 Rows_sent: 0  Rows_examined: 0


  


  ---UPDATE---
Here's the complete my.cnf file

  


  my.cnf

[mysqld]
pid-file        = /var/run/mysqld/mysqld.pid
socket          = /var/run/mysqld/mysqld.sock
datadir         = /var/lib/mysql
secure-file-priv= NULL
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0

skip-log-bin
innodb_buffer_pool_size=536870912
innodb_log_file_size=100663296

# Custom config should go here
!includedir /etc/mysql/conf.d/

conf.d/docker.cnf 
[mysqld]
skip-host-cache
skip-name-resolve 


  


  The docker container is using the host mode so complete 15GB memory is available to the container.

  


  --- UPDATE 2 ---
After increasing the innodb_buffer_pool_size to 2GB as suggested by @fyrye, the statements have now started getting stuck on STATE = UPDATE instead of waiting for handler commit.

  


  ---- UPDATE 3 ---
Looks like the CPU is causing the bottleneck


  


  ** ---- UPDATE 4 ---- **
Additional info

  


  

  1. Ram Size

  


  


                total        used        free      shared  buff/cache   available
Mem:          15909        1711        9385        2491        4813       11600
Swap:             0           0           0


  


  

  2. No SSD/NVMe devices attached

  


  3. SHOW GLOBAL STATUS - https://pastebin.com/vtWi0PUq

  


  4. SHOW GLOBAL VARIABLES - https://pastebin.com/MUZeG959

  


  5. SHOW FULL PROCESSLIST - https://pastebin.com/eebEcYk7

  


  6. htop - htop here is for the edge system which has 4 other containers running which include the main app, ffmpeg, mqtt, etc.


  


  7. ulimit -a :

  


  


  core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 62576
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 1024
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 62576
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited


  


  

  8. opstat -xm 5 4

  


  


  Linux 4.15.0-106-generic (xxxx)    08/18/2020  _x86_64_    (4 CPU)

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
         26.97    0.00   22.36   22.53    0.00   28.14

Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
loop0             0.00     0.00    0.00    0.00     0.00     0.00     3.20     0.00    2.40    2.40    0.00   0.00   0.00
sda              13.78     9.89   32.24   11.44     0.37     4.10   209.51    47.52 1079.07   44.07 3994.87  22.39  97.81

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
         19.71    0.00   27.85   40.87    0.00   11.57

Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
loop0             0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
sda               0.00     0.00    1.40    4.60     0.03     2.71   934.93   142.66 24221.33  666.29 31390.26 166.67 100.00

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
         20.16    0.00   26.77   28.30    0.00   24.77

Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
loop0             0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
sda               0.00     0.00    8.80    5.60     0.03     3.45   496.11   141.28 12507.78  194.00 31858.00  69.44 100.00


  


  

  9. mpstat -P ALL 5 3

  


  


  Linux 4.15.0-106-generic (sn-1f0ce8)    08/18/2020  _x86_64_    (4 CPU)

02:15:47 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
02:15:52 PM  all   21.48    0.00   20.40   29.01    0.00    7.94    0.00    0.00    0.00   21.17
02:15:52 PM    0   24.95    0.00   20.86    5.32    0.00    0.61    0.00    0.00    0.00   48.26
02:15:52 PM    1   17.59    0.00   18.81   57.67    0.00    5.93    0.00    0.00    0.00    0.00
02:15:52 PM    2   21.28    0.00   17.36    0.21    0.00   24.79    0.00    0.00    0.00   36.36
02:15:52 PM    3   22.34    0.00   24.59   52.46    0.00    0.61    0.00    0.00    0.00    0.00

02:15:52 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
02:15:57 PM  all   20.56    0.00   20.00   28.26    0.00    7.08    0.00    0.00    0.00   24.10
02:15:57 PM    0   24.44    0.00   18.89   12.32    0.00    0.21    0.00    0.00    0.00   44.15
02:15:57 PM    1   17.73    0.00   15.46   33.20    0.00    4.95    0.00    0.00    0.00   28.66
02:15:57 PM    2   18.93    0.00   22.22   12.35    0.00   22.84    0.00    0.00    0.00   23.66
02:15:57 PM    3   21.06    0.00   23.31   55.21    0.00    0.41    0.00    0.00    0.00    0.00

02:15:57 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
02:16:02 PM  all   21.81    0.00   18.32   26.42    0.00    7.03    0.00    0.00    0.00   26.42
02:16:02 PM    0   26.43    0.00   19.67    0.20    0.00    0.41    0.00    0.00    0.00   53.28
02:16:02 PM    1   20.57    0.00   17.11   45.21    0.00    5.30    0.00    0.00    0.00   11.81
02:16:02 PM    2   19.67    0.00   16.74    0.21    0.00   21.97    0.00    0.00    0.00   41.42
02:16:02 PM    3   20.45    0.00   19.84   58.91    0.00    0.81    0.00    0.00    0.00    0.00

Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:     all   21.28    0.00   19.57   27.90    0.00    7.35    0.00    0.00    0.00   23.90
Average:       0   25.27    0.00   19.81    5.94    0.00    0.41    0.00    0.00    0.00   48.57
Average:       1   18.63    0.00   17.13   45.39    0.00    5.39    0.00    0.00    0.00   13.45
Average:       2   19.96    0.00   18.78    4.28    0.00   23.20    0.00    0.00    0.00   33.77
Average:       3   21.28    0.00   22.57   55.54    0.00    0.61    0.00    0.00    0.00    0.00