From ef126e11ffded87b7de6fb56ba7048a20d87af7e Mon Sep 17 00:00:00 2001 From: Umar Hansa Date: Wed, 22 Jan 2020 23:11:15 +0000 Subject: [PATCH] Hitting the FS concurrently too often causes os-level issues, for now, stick with sequential media segment generation --- package-lock.json | 122 +++++++++++++---------- package.json | 12 +-- src/server/lib/generate-video-segment.js | 9 +- src/server/lib/is-valid-media-type.js | 3 +- src/server/lib/prepare-media.js | 11 +- 5 files changed, 91 insertions(+), 66 deletions(-) diff --git a/package-lock.json b/package-lock.json index 392b548..2a26cca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -712,9 +712,9 @@ }, "dependencies": { "readable-stream": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", - "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.5.0.tgz", + "integrity": "sha512-gSz026xs2LfxBPudDuI41V1lka8cxg64E66SGe78zJlsUofOg/yqwezdIcdfwik6B4h8LFmWPA9ef9X3FiNFLA==", "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1246,9 +1246,9 @@ } }, "commander": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.0.1.tgz", - "integrity": "sha512-IPF4ouhCP+qdlcmCedhxX4xiGBPyigb8v5NeUp+0LyhwLgxMqyp3S0vl7TAPfS/hiP7FC3caI/PB9lTmP8r1NA==" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.0.tgz", + "integrity": "sha512-NIQrwvv9V39FHgGFm36+U9SMQzbiHvU79k+iADraJTpmrFFfx7Ds0IvDoAdZsDrknlkRk14OYoWXb57uTh7/sw==" }, "commondir": { "version": "1.0.1", @@ -1333,9 +1333,9 @@ } }, "config": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/config/-/config-3.2.4.tgz", - "integrity": "sha512-H1XIGfnU1EAkfjSLn9ZvYDRx9lOezDViuzLDgiJ/lMeqjYe3q6iQfpcLt2NInckJgpAeekbNhQkmnnbdEDs9rw==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/config/-/config-3.2.5.tgz", + "integrity": "sha512-8itpjyR01lAJanhAlPncBngYRZez/LoRLW8wnGi+6SEcsUyA1wvHvbpIrAJYDJT+W9BScnj4mYoUgbtp9I+0+Q==", "requires": { "json5": "^1.0.1" } @@ -2588,6 +2588,11 @@ "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==", "dev": true }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, "espree": { "version": "6.1.2", "resolved": "https://registry.npmjs.org/espree/-/espree-6.1.2.tgz", @@ -2670,21 +2675,6 @@ "es5-ext": "~0.10.14" } }, - "execa": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", - "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", - "dev": true, - "requires": { - "cross-spawn": "^5.0.1", - "get-stream": "^3.0.0", - "is-stream": "^1.1.0", - "npm-run-path": "^2.0.0", - "p-finally": "^1.0.0", - "signal-exit": "^3.0.0", - "strip-eof": "^1.0.0" - } - }, "exif-reader": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/exif-reader/-/exif-reader-1.0.3.tgz", @@ -3322,9 +3312,9 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-minipass": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.0.0.tgz", - "integrity": "sha512-40Qz+LFXmd9tzYVnnBmZvFfvAADfUA14TXPK1s7IfElJTIZ97rA8w4Kin7Wt5JBrC3ShnnFJO/5vPjPEeJIq9A==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", "requires": { "minipass": "^3.0.0" } @@ -5285,14 +5275,15 @@ "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" }, "knex": { - "version": "0.20.4", - "resolved": "https://registry.npmjs.org/knex/-/knex-0.20.4.tgz", - "integrity": "sha512-gNpYj9BtacWnQwkyPaHOlzHRLJ7N6Abz8AKYb4OqoDA/iCY50VAUSpigjkS7Z4sr7uW64sxW2cVoXudaLN1ZQw==", + "version": "0.20.8", + "resolved": "https://registry.npmjs.org/knex/-/knex-0.20.8.tgz", + "integrity": "sha512-fLiSg5PIBisORs0M+UGjg2s1P/E1BrYvb/NkSVk6Y90HJujkqLufSC6ag+hDgXqW73mFAF283M6+q3/NW0TrHw==", "requires": { - "bluebird": "^3.7.1", + "bluebird": "^3.7.2", "colorette": "1.1.0", - "commander": "^4.0.1", + "commander": "^4.1.0", "debug": "4.1.1", + "esm": "^3.2.25", "getopts": "2.2.5", "inherits": "~2.0.4", "interpret": "^2.0.0", @@ -7431,9 +7422,9 @@ } }, "rollup": { - "version": "1.27.14", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.27.14.tgz", - "integrity": "sha512-DuDjEyn8Y79ALYXMt+nH/EI58L5pEw5HU9K38xXdRnxQhvzUTI/nxAawhkAHUQeudANQ//8iyrhVRHJBuR6DSQ==", + "version": "1.29.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.29.1.tgz", + "integrity": "sha512-dGQ+b9d1FOX/gluiggTAVnTvzQZUEkCi/TwZcax7ujugVRHs0nkYJlV9U4hsifGEMojnO+jvEML2CJQ6qXgbHA==", "dev": true, "requires": { "@types/estree": "*", @@ -7476,16 +7467,16 @@ } }, "rollup-plugin-terser": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/rollup-plugin-terser/-/rollup-plugin-terser-5.1.3.tgz", - "integrity": "sha512-FuFuXE5QUJ7snyxHLPp/0LFXJhdomKlIx/aK7Tg88Yubsx/UU/lmInoJafXJ4jwVVNcORJ1wRUC5T9cy5yk0wA==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-terser/-/rollup-plugin-terser-5.2.0.tgz", + "integrity": "sha512-jQI+nYhtDBc9HFRBz8iGttQg7li9klmzR62RG2W2nN6hJ/FI2K2ItYQ7kJ7/zn+vs+BP1AEccmVRjRN989I+Nw==", "dev": true, "requires": { - "@babel/code-frame": "^7.0.0", - "jest-worker": "^24.6.0", - "rollup-pluginutils": "^2.8.1", + "@babel/code-frame": "^7.5.5", + "jest-worker": "^24.9.0", + "rollup-pluginutils": "^2.8.2", "serialize-javascript": "^2.1.2", - "terser": "^4.1.0" + "terser": "^4.6.2" } }, "rollup-pluginutils": { @@ -7621,7 +7612,8 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true }, "semver-diff": { "version": "2.1.0", @@ -7740,19 +7732,26 @@ "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, "sharp": { - "version": "0.23.4", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.23.4.tgz", - "integrity": "sha512-fJMagt6cT0UDy9XCsgyLi0eiwWWhQRxbwGmqQT6sY8Av4s0SVsT/deg8fobBQCTDU5iXRgz0rAeXoE2LBZ8g+Q==", + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.24.0.tgz", + "integrity": "sha512-kUtQE6+HJnNqO0H6ueOBtRXahktuqydIBaFMvhDelf/KaK9j/adEdjf4Y3+bbjYOa5i6hi2EAa2Y2G9umP4s2g==", "requires": { "color": "^3.1.2", "detect-libc": "^1.0.3", "nan": "^2.14.0", "npmlog": "^4.1.2", "prebuild-install": "^5.3.3", - "semver": "^6.3.0", + "semver": "^7.1.1", "simple-get": "^3.1.0", "tar": "^5.0.5", "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "semver": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.1.1.tgz", + "integrity": "sha512-WfuG+fl6eh3eZ2qAf6goB7nhiCd7NPXhmyFxigB/TOkQyeLP8w8GsVehvtGNtnNmyboz4TgeK40B1Kbql/8c5A==" + } } }, "shebang-command": { @@ -8366,9 +8365,9 @@ }, "dependencies": { "readable-stream": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", - "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.5.0.tgz", + "integrity": "sha512-gSz026xs2LfxBPudDuI41V1lka8cxg64E66SGe78zJlsUofOg/yqwezdIcdfwik6B4h8LFmWPA9ef9X3FiNFLA==", "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -8389,12 +8388,29 @@ "dev": true, "requires": { "execa": "^0.7.0" + }, + "dependencies": { + "execa": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", + "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", + "dev": true, + "requires": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + } } }, "terser": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/terser/-/terser-4.4.3.tgz", - "integrity": "sha512-0ikKraVtRDKGzHrzkCv5rUNDzqlhmhowOBqC0XqUHFpW+vJ45+20/IFBcebwKfiS2Z9fJin6Eo+F1zLZsxi8RA==", + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.6.3.tgz", + "integrity": "sha512-Lw+ieAXmY69d09IIc/yqeBqXpEQIpDGZqT34ui1QWXIUpR2RjbqEkT8X7Lgex19hslSqcWM5iMN2kM11eMsESQ==", "dev": true, "requires": { "commander": "^2.20.0", diff --git a/package.json b/package.json index 541f7de..dc14c65 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,12 @@ "update-deps": "ncu -u", "test": "xo", "gulp": "gulp -f gulpfile.cjs", - "reset": "rm db-development-video-everyday.sqlite && npm run migrate-db-dev && export NODE_ENV=development && nodemon src/server/server.js" + "reset": "rm -rf ~/Downloads/video-everyday/segments/* && rm -rf ~/Downloads/video-everyday/thumbnails/* && rm db-development-video-everyday.sqlite && npm run migrate-db-dev && export NODE_ENV=development && node src/server/server.js" }, "dependencies": { "body-parser": "^1.19.0", "compression": "^1.7.4", - "config": "^3.2.4", + "config": "^3.2.5", "connect-flash": "^0.1.1", "connect-session-knex": "^1.5.0", "cookie-parser": "~1.4.4", @@ -30,12 +30,12 @@ "express-session": "^1.17.0", "fluent-ffmpeg": "^2.1.2", "forcedomain": "^2.0.0", - "knex": "^0.20.4", + "knex": "^0.20.8", "mkdirp": "^0.5.1", "nunjucks": "^3.2.0", "passport": "^0.4.1", "rimraf": "^3.0.0", - "sharp": "^0.23.4", + "sharp": "^0.24.0", "signale": "^1.4.0", "sqlite3": "^4.1.1" }, @@ -50,10 +50,10 @@ "gulp-rev-delete-original": "^0.2.3", "gulp-sass": "^4.0.2", "gulp-sourcemaps": "^2.6.5", - "rollup": "^1.27.14", + "rollup": "^1.29.1", "rollup-plugin-commonjs": "^10.1.0", "rollup-plugin-node-resolve": "^5.2.0", - "rollup-plugin-terser": "^5.1.3", + "rollup-plugin-terser": "^5.2.0", "tiny-lr": "^1.1.1", "vinyl-paths": "^3.0.1", "xo": "^0.25.3" diff --git a/src/server/lib/generate-video-segment.js b/src/server/lib/generate-video-segment.js index 21e3433..e0eacc4 100644 --- a/src/server/lib/generate-video-segment.js +++ b/src/server/lib/generate-video-segment.js @@ -5,6 +5,7 @@ import {exec as execOld} from 'child_process'; import path from 'path'; import config from 'config'; import mkdirp from 'mkdirp'; +import execa from 'execa'; const exec = promisify(execOld); const videoSegmentFolder = config.get('video-segment-folder'); @@ -25,7 +26,12 @@ async function init({mediaFile, totalVideoDuration}) { const halfWayMark = Math.floor(totalVideoDuration / 2); const command = `(cd '${videoSegmentFolderForMedia}' && ${MP4BoxBinary} -splitx ${halfWayMark}:${halfWayMark + defaultVideoSegmentDuration} '${absoluteFilePathForMedia}')`; + const hrtime = process.hrtime()[1]; + + console.time(`MP4Box Video Segment Creation ${hrtime}`) const {stderr} = await exec(command); + // const {stderr} = await execa(command); + console.timeEnd(`MP4Box Video Segment Creation ${hrtime}`) const indexOfFileName = stderr.indexOf(parsedMediaFileName.name); const indexOfFileExtension = stderr.indexOf(`${parsedMediaFileName.ext} - duration`); @@ -53,7 +59,8 @@ async function init({mediaFile, totalVideoDuration}) { const newCommand = `ffmpeg -y -i '${videoSegmentAbsolutePath}' -vf "scale=640:-2" '${videoSegmentAbsolutePath}.mini${ext}'`; - console.log('Mini video command:', newCommand); + await exec(newCommand); + console.log('This mini video segment has finished creation\n'); return { relativeVideoSegmentPath: newFileRelativePath, diff --git a/src/server/lib/is-valid-media-type.js b/src/server/lib/is-valid-media-type.js index 266d381..fb08a11 100644 --- a/src/server/lib/is-valid-media-type.js +++ b/src/server/lib/is-valid-media-type.js @@ -1,7 +1,8 @@ const allowedMediaTypes = { video: ['.mp4', '.mov'], - image: ['.jpg', '.jpeg'] + // Uncomment line below, currently testing issues with video only + // image: ['.jpg', '.jpeg'] }; function isValidMediaType(mediaPath) { diff --git a/src/server/lib/prepare-media.js b/src/server/lib/prepare-media.js index c26feef..0690237 100644 --- a/src/server/lib/prepare-media.js +++ b/src/server/lib/prepare-media.js @@ -70,7 +70,8 @@ async function init() { let unprocessableMediaCount = 0; - const mediaFilesWhichNeedProcessingPromises = mediaFilesWhichNeedProcessing.map(async mediaFile => { + // const mediaFilesWhichNeedProcessingPromises = mediaFilesWhichNeedProcessing.map(async mediaFile => { + for (const mediaFile of mediaFilesWhichNeedProcessing) { const isVideo = getMediaType(mediaFile) === 'video'; let metadata; @@ -82,8 +83,8 @@ async function init() { if (!metadata) { unprocessableMediaCount++; - console.log(`${mediaFile} couldn't be processed`); - return; + console.log(`${mediaFile} couldn't be processed. Skipping this item.`); + continue; } const DBRecord = { @@ -107,9 +108,9 @@ async function init() { } await mediaMetadataQueries.insert(DBRecord); - }); + } - await Promise.all(mediaFilesWhichNeedProcessingPromises); + // await Promise.all(mediaFilesWhichNeedProcessingPromises); console.log(`${unprocessableMediaCount} media items couldn't be processed`); await generateThumbnails();