Hitting the FS concurrently too often causes os-level issues, for now…

…, stick with sequential media segment generation

package.json

@@ -13,12 +13,12 @@
     "update-deps": "ncu -u",
     "test": "xo",
     "gulp": "gulp -f gulpfile.cjs",
-    "reset": "rm db-development-video-everyday.sqlite  && npm run migrate-db-dev && export NODE_ENV=development && nodemon src/server/server.js"
+    "reset": "rm -rf ~/Downloads/video-everyday/segments/* && rm -rf ~/Downloads/video-everyday/thumbnails/* && rm db-development-video-everyday.sqlite  && npm run migrate-db-dev && export NODE_ENV=development && node src/server/server.js"
   },
   "dependencies": {
     "body-parser": "^1.19.0",
     "compression": "^1.7.4",
-    "config": "^3.2.4",
+    "config": "^3.2.5",
     "connect-flash": "^0.1.1",
     "connect-session-knex": "^1.5.0",
     "cookie-parser": "~1.4.4",
@@ -30,12 +30,12 @@
     "express-session": "^1.17.0",
     "fluent-ffmpeg": "^2.1.2",
     "forcedomain": "^2.0.0",
-    "knex": "^0.20.4",
+    "knex": "^0.20.8",
     "mkdirp": "^0.5.1",
     "nunjucks": "^3.2.0",
     "passport": "^0.4.1",
     "rimraf": "^3.0.0",
-    "sharp": "^0.23.4",
+    "sharp": "^0.24.0",
     "signale": "^1.4.0",
     "sqlite3": "^4.1.1"
   },
@@ -50,10 +50,10 @@
     "gulp-rev-delete-original": "^0.2.3",
     "gulp-sass": "^4.0.2",
     "gulp-sourcemaps": "^2.6.5",
-    "rollup": "^1.27.14",
+    "rollup": "^1.29.1",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-node-resolve": "^5.2.0",
-    "rollup-plugin-terser": "^5.1.3",
+    "rollup-plugin-terser": "^5.2.0",
     "tiny-lr": "^1.1.1",
     "vinyl-paths": "^3.0.1",
     "xo": "^0.25.3"

src/server/lib/generate-video-segment.js

@@ -5,6 +5,7 @@ import {exec as execOld} from 'child_process';
 import path from 'path';
 import config from 'config';
 import mkdirp from 'mkdirp';
+import execa from 'execa';
 
 const exec = promisify(execOld);
 const videoSegmentFolder = config.get('video-segment-folder');
@@ -25,7 +26,12 @@ async function init({mediaFile, totalVideoDuration}) {
 		const halfWayMark = Math.floor(totalVideoDuration / 2);
 		const command = `(cd '${videoSegmentFolderForMedia}' && ${MP4BoxBinary} -splitx ${halfWayMark}:${halfWayMark + defaultVideoSegmentDuration} '${absoluteFilePathForMedia}')`;
 
+		const hrtime = process.hrtime()[1];
+
+		console.time(`MP4Box Video Segment Creation ${hrtime}`)
 		const {stderr} = await exec(command);
+		// const {stderr} = await execa(command);
+		console.timeEnd(`MP4Box Video Segment Creation ${hrtime}`)
 
 		const indexOfFileName = stderr.indexOf(parsedMediaFileName.name);
 		const indexOfFileExtension = stderr.indexOf(`${parsedMediaFileName.ext} - duration`);
@@ -53,7 +59,8 @@ async function init({mediaFile, totalVideoDuration}) {
 
 	const newCommand = `ffmpeg -y -i '${videoSegmentAbsolutePath}' -vf "scale=640:-2" '${videoSegmentAbsolutePath}.mini${ext}'`;
 
-	console.log('Mini video command:', newCommand);
+	await exec(newCommand);
+	console.log('This mini video segment has finished creation\n');
 
 	return {
 		relativeVideoSegmentPath: newFileRelativePath,

src/server/lib/is-valid-media-type.js

@@ -1,7 +1,8 @@
 
 const allowedMediaTypes = {
 	video: ['.mp4', '.mov'],
-	image: ['.jpg', '.jpeg']
+	// Uncomment line below, currently testing issues with video only
+	// image: ['.jpg', '.jpeg']
 };
 
 function isValidMediaType(mediaPath) {

src/server/lib/prepare-media.js

@@ -70,7 +70,8 @@ async function init() {
 
 	let unprocessableMediaCount = 0;
 
-	const mediaFilesWhichNeedProcessingPromises = mediaFilesWhichNeedProcessing.map(async mediaFile => {
+	// const mediaFilesWhichNeedProcessingPromises = mediaFilesWhichNeedProcessing.map(async mediaFile => {
+	for (const mediaFile of mediaFilesWhichNeedProcessing) {
 		const isVideo = getMediaType(mediaFile) === 'video';
 		let metadata;
 
@@ -82,8 +83,8 @@ async function init() {
 
 		if (!metadata) {
 			unprocessableMediaCount++;
-			console.log(`${mediaFile} couldn't be processed`);
-			return;
+			console.log(`${mediaFile} couldn't be processed. Skipping this item.`);
+			continue;
 		}
 
 		const DBRecord = {
@@ -107,9 +108,9 @@ async function init() {
 		}
 
 		await mediaMetadataQueries.insert(DBRecord);
-	});
+	}
 
-	await Promise.all(mediaFilesWhichNeedProcessingPromises);
+	// await Promise.all(mediaFilesWhichNeedProcessingPromises);
 	console.log(`${unprocessableMediaCount} media items couldn't be processed`);
 
 	await generateThumbnails();