From 1ba41207d3613be4c85f51095cf1242e8b86ca28 Mon Sep 17 00:00:00 2001 From: HaveAGitGat <43864057+HaveAGitGat@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:21:07 +0100 Subject: [PATCH 1/4] Use time status --- FlowPlugins/FlowHelpers/1.0.0/cliParsers.js | 83 ++++++++++------- FlowPluginsTs/FlowHelpers/1.0.0/cliParsers.ts | 93 +++++++++++-------- 2 files changed, 106 insertions(+), 70 deletions(-) diff --git a/FlowPlugins/FlowHelpers/1.0.0/cliParsers.js b/FlowPlugins/FlowHelpers/1.0.0/cliParsers.js index ee72a82ad..27d860114 100644 --- a/FlowPlugins/FlowHelpers/1.0.0/cliParsers.js +++ b/FlowPlugins/FlowHelpers/1.0.0/cliParsers.js @@ -51,7 +51,7 @@ var getFFmpegVar = function (_a) { }; exports.getFFmpegVar = getFFmpegVar; var getFFmpegPercentage = function (_a) { - var f = _a.f, fc = _a.fc, vf = _a.vf, d = _a.d; + var time = _a.time, f = _a.f, fc = _a.fc, vf = _a.vf, d = _a.d; var frameCount01 = fc; var VideoFrameRate = vf; var Duration = d; @@ -60,14 +60,19 @@ var getFFmpegPercentage = function (_a) { frameCount01 = Math.ceil(frameCount01); VideoFrameRate = Math.ceil(VideoFrameRate); Duration = Math.ceil(Duration); - if (frameCount01 > 0) { - perc = ((frame / frameCount01) * 100); - } - else if (VideoFrameRate > 0 && Duration > 0) { - perc = ((frame / (VideoFrameRate * Duration)) * 100); + if (frame > 0) { + if (frameCount01 > 0) { + perc = ((frame / frameCount01) * 100); + } + else if (VideoFrameRate > 0 && Duration > 0) { + perc = ((frame / (VideoFrameRate * Duration)) * 100); + } + else { + perc = (frame); + } } - else { - perc = (frame); + else if (time > 0 && Duration > 0) { + perc = ((time / Duration) * 100); } var percString = perc.toFixed(2); // eslint-disable-next-line no-restricted-globals @@ -84,33 +89,45 @@ var ffmpegParser = function (_a) { } var percentage = 0; if (str.length >= 6) { - var n = str.indexOf('fps'); - if (n >= 6) { - // get frame - var frame = getFFmpegVar({ - str: str, - variable: 'frame', - }); - var frameRate = videoFrameRate || 0; - var duration = 0; - if (ffprobeDuration - && parseFloat(ffprobeDuration) > 0) { - duration = parseFloat(ffprobeDuration); - } - else if (metaDuration) { - duration = metaDuration; - } - var per = getFFmpegPercentage({ - f: frame, - fc: frameCount, - vf: frameRate, - d: duration, - }); - var outputNum = Number(per); - if (outputNum > 0) { - percentage = outputNum; + var frame = getFFmpegVar({ + str: str, + variable: 'frame', + }); + var time = 0; + // get time + var timeStr = getFFmpegVar({ + str: str, + variable: 'time', + }); + if (timeStr) { + var timeArr = timeStr.split(':'); + if (timeArr.length === 3) { + var hours = parseInt(timeArr[0], 10); + var minutes = parseInt(timeArr[1], 10); + var seconds = parseInt(timeArr[2], 10); + time = (hours * 3600) + (minutes * 60) + seconds; } } + var frameRate = videoFrameRate || 0; + var duration = 0; + if (ffprobeDuration + && parseFloat(ffprobeDuration) > 0) { + duration = parseFloat(ffprobeDuration); + } + else if (metaDuration) { + duration = metaDuration; + } + var per = getFFmpegPercentage({ + time: time, + f: frame, + fc: frameCount, + vf: frameRate, + d: duration, + }); + var outputNum = Number(per); + if (outputNum > 0) { + percentage = outputNum; + } } return percentage; }; diff --git a/FlowPluginsTs/FlowHelpers/1.0.0/cliParsers.ts b/FlowPluginsTs/FlowHelpers/1.0.0/cliParsers.ts index a9604d9ca..8e79cc66a 100644 --- a/FlowPluginsTs/FlowHelpers/1.0.0/cliParsers.ts +++ b/FlowPluginsTs/FlowHelpers/1.0.0/cliParsers.ts @@ -66,12 +66,13 @@ const getFFmpegVar = ({ }; const getFFmpegPercentage = ({ + time, f, fc, vf, d, }: { - + time: number, f: string, fc: number, vf: number, d: number }): number => { let frameCount01: number = fc; @@ -85,12 +86,16 @@ const getFFmpegPercentage = ({ VideoFrameRate = Math.ceil(VideoFrameRate); Duration = Math.ceil(Duration); - if (frameCount01 > 0) { - perc = ((frame / frameCount01) * 100); - } else if (VideoFrameRate > 0 && Duration > 0) { - perc = ((frame / (VideoFrameRate * Duration)) * 100); - } else { - perc = (frame); + if (frame > 0) { + if (frameCount01 > 0) { + perc = ((frame / frameCount01) * 100); + } else if (VideoFrameRate > 0 && Duration > 0) { + perc = ((frame / (VideoFrameRate * Duration)) * 100); + } else { + perc = (frame); + } + } else if (time > 0 && Duration > 0) { + perc = ((time / Duration) * 100); } const percString = perc.toFixed(2); @@ -124,40 +129,54 @@ const ffmpegParser = ({ let percentage = 0; if (str.length >= 6) { - const n = str.indexOf('fps'); - - if (n >= 6) { - // get frame - const frame = getFFmpegVar({ - str, - variable: 'frame', - }); + const frame = getFFmpegVar({ + str, + variable: 'frame', + }); + + let time = 0; + + // get time + const timeStr = getFFmpegVar({ + str, + variable: 'time', + }); + + if (timeStr) { + const timeArr = timeStr.split(':'); + if (timeArr.length === 3) { + const hours = parseInt(timeArr[0], 10); + const minutes = parseInt(timeArr[1], 10); + const seconds = parseInt(timeArr[2], 10); + time = (hours * 3600) + (minutes * 60) + seconds; + } + } - const frameRate = videoFrameRate || 0; - let duration = 0; + const frameRate = videoFrameRate || 0; + let duration = 0; - if ( - ffprobeDuration + if ( + ffprobeDuration && parseFloat(ffprobeDuration) > 0 - ) { - duration = parseFloat(ffprobeDuration); - } else if (metaDuration) { - duration = metaDuration; - } + ) { + duration = parseFloat(ffprobeDuration); + } else if (metaDuration) { + duration = metaDuration; + } - const per = getFFmpegPercentage( - { - f: frame, - fc: frameCount, - vf: frameRate, - d: duration, - }, - ); - - const outputNum = Number(per); - if (outputNum > 0) { - percentage = outputNum; - } + const per = getFFmpegPercentage( + { + time, + f: frame, + fc: frameCount, + vf: frameRate, + d: duration, + }, + ); + + const outputNum = Number(per); + if (outputNum > 0) { + percentage = outputNum; } } From 759a873c30906b1b503692192d243c19d6c8fe69 Mon Sep 17 00:00:00 2001 From: HaveAGitGat <43864057+HaveAGitGat@users.noreply.github.com> Date: Tue, 17 Oct 2023 05:53:33 +0100 Subject: [PATCH 2/4] Only check perc > 0 --- FlowPlugins/FlowHelpers/1.0.0/cliUtils.js | 2 +- FlowPluginsTs/FlowHelpers/1.0.0/cliUtils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/FlowPlugins/FlowHelpers/1.0.0/cliUtils.js b/FlowPlugins/FlowHelpers/1.0.0/cliUtils.js index bebffc8bf..5451c39da 100644 --- a/FlowPlugins/FlowHelpers/1.0.0/cliUtils.js +++ b/FlowPlugins/FlowHelpers/1.0.0/cliUtils.js @@ -197,7 +197,7 @@ var CLI = /** @class */ (function () { fps: fps, }); } - if (shouldUpdate === true && percentage > 0) { + if (percentage > 0) { _this.updateETA(percentage); _this.config.updateWorker({ percentage: percentage, diff --git a/FlowPluginsTs/FlowHelpers/1.0.0/cliUtils.ts b/FlowPluginsTs/FlowHelpers/1.0.0/cliUtils.ts index 966f75ec9..ef5f30271 100644 --- a/FlowPluginsTs/FlowHelpers/1.0.0/cliUtils.ts +++ b/FlowPluginsTs/FlowHelpers/1.0.0/cliUtils.ts @@ -212,7 +212,7 @@ class CLI { }); } - if (shouldUpdate === true && percentage > 0) { + if (percentage > 0) { this.updateETA(percentage); this.config.updateWorker({ percentage, From 6b1157763f95e8b830248e76133ebee84260e369 Mon Sep 17 00:00:00 2001 From: HaveAGitGat <43864057+HaveAGitGat@users.noreply.github.com> Date: Tue, 17 Oct 2023 07:07:01 +0100 Subject: [PATCH 3/4] Add file access checks option with pause node --- .../input/inputFile/1.0.0/index.js | 175 ++++++++++++++++-- .../input/inputFile/1.0.0/index.ts | 95 +++++++++- 2 files changed, 256 insertions(+), 14 deletions(-) diff --git a/FlowPlugins/CommunityFlowPlugins/input/inputFile/1.0.0/index.js b/FlowPlugins/CommunityFlowPlugins/input/inputFile/1.0.0/index.js index ba0378d17..7e1903e1e 100644 --- a/FlowPlugins/CommunityFlowPlugins/input/inputFile/1.0.0/index.js +++ b/FlowPlugins/CommunityFlowPlugins/input/inputFile/1.0.0/index.js @@ -1,6 +1,44 @@ "use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; Object.defineProperty(exports, "__esModule", { value: true }); exports.plugin = exports.details = void 0; +var fs_1 = require("fs"); +var fileUtils_1 = require("../../../../FlowHelpers/1.0.0/fileUtils"); /* eslint no-plusplus: ["error", { "allowForLoopAfterthoughts": true }] */ var details = function () { return ({ name: 'Input File', @@ -14,7 +52,34 @@ var details = function () { return ({ requiresVersion: '2.11.01', sidebarPosition: -1, icon: '', - inputs: [], + inputs: [ + { + name: 'fileAccessChecks', + type: 'boolean', + defaultValue: 'false', + inputUI: { + type: 'dropdown', + options: [ + 'false', + 'true', + ], + }, + tooltip: 'Will check if input file and cache are readable and writable', + }, + { + name: 'pauseNodeIfAccessChecksFail', + type: 'boolean', + defaultValue: 'false', + inputUI: { + type: 'dropdown', + options: [ + 'false', + 'true', + ], + }, + tooltip: 'This will pause the node if the file access checks fail', + }, + ], outputs: [ { number: 1, @@ -24,14 +89,102 @@ var details = function () { return ({ }); }; exports.details = details; // eslint-disable-next-line @typescript-eslint/no-unused-vars -var plugin = function (args) { - var lib = require('../../../../../methods/lib')(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars,no-param-reassign - args.inputs = lib.loadDefaultValues(args.inputs, details); - return { - outputFileObj: args.inputFileObj, - outputNumber: 1, - variables: args.variables, - }; -}; +var plugin = function (args) { return __awaiter(void 0, void 0, void 0, function () { + var lib, orignalFolder, _a, fileAccessChecks, pauseNodeIfAccessChecksFail, nodeID, _b, serverIP, serverPort, url, pauseNode, checkReadWrite; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + lib = require('../../../../../methods/lib')(); + // eslint-disable-next-line @typescript-eslint/no-unused-vars,no-param-reassign + args.inputs = lib.loadDefaultValues(args.inputs, details); + orignalFolder = (0, fileUtils_1.getFileAbosluteDir)(args.originalLibraryFile._id); + _a = args.inputs, fileAccessChecks = _a.fileAccessChecks, pauseNodeIfAccessChecksFail = _a.pauseNodeIfAccessChecksFail; + nodeID = process.argv[8]; + _b = args.deps.configVars.config, serverIP = _b.serverIP, serverPort = _b.serverPort; + url = "http://".concat(serverIP, ":").concat(serverPort, "/api/v2/update-node"); + pauseNode = function () { return __awaiter(void 0, void 0, void 0, function () { + var requestConfig; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + args.jobLog('Pausing node'); + requestConfig = { + method: 'post', + url: url, + headers: {}, + data: { + data: { + nodeID: nodeID, + nodeUpdates: { + nodePaused: true, + }, + }, + }, + }; + return [4 /*yield*/, args.deps.axios(requestConfig)]; + case 1: + _a.sent(); + args.jobLog('Node paused'); + return [2 /*return*/]; + } + }); + }); }; + checkReadWrite = function (location) { return __awaiter(void 0, void 0, void 0, function () { + var err_1, err_2; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + _a.trys.push([0, 2, , 5]); + return [4 /*yield*/, fs_1.promises.access(location, fs_1.promises.constants.R_OK)]; + case 1: + _a.sent(); + return [3 /*break*/, 5]; + case 2: + err_1 = _a.sent(); + args.jobLog(JSON.stringify(err_1)); + if (!pauseNodeIfAccessChecksFail) return [3 /*break*/, 4]; + return [4 /*yield*/, pauseNode()]; + case 3: + _a.sent(); + _a.label = 4; + case 4: throw new Error("Location not readable:".concat(location)); + case 5: + _a.trys.push([5, 7, , 10]); + return [4 /*yield*/, fs_1.promises.access(location, fs_1.promises.constants.W_OK)]; + case 6: + _a.sent(); + return [3 /*break*/, 10]; + case 7: + err_2 = _a.sent(); + args.jobLog(JSON.stringify(err_2)); + if (!pauseNodeIfAccessChecksFail) return [3 /*break*/, 9]; + return [4 /*yield*/, pauseNode()]; + case 8: + _a.sent(); + _a.label = 9; + case 9: throw new Error("Location not writeable:".concat(location)); + case 10: return [2 /*return*/]; + } + }); + }); }; + if (!fileAccessChecks) return [3 /*break*/, 3]; + args.jobLog('Checking file access'); + return [4 /*yield*/, checkReadWrite(orignalFolder)]; + case 1: + _c.sent(); + return [4 /*yield*/, checkReadWrite(args.librarySettings.cache)]; + case 2: + _c.sent(); + return [3 /*break*/, 4]; + case 3: + args.jobLog('Skipping file access checks'); + _c.label = 4; + case 4: return [2 /*return*/, { + outputFileObj: args.inputFileObj, + outputNumber: 1, + variables: args.variables, + }]; + } + }); +}); }; exports.plugin = plugin; diff --git a/FlowPluginsTs/CommunityFlowPlugins/input/inputFile/1.0.0/index.ts b/FlowPluginsTs/CommunityFlowPlugins/input/inputFile/1.0.0/index.ts index e638af895..187bbe002 100644 --- a/FlowPluginsTs/CommunityFlowPlugins/input/inputFile/1.0.0/index.ts +++ b/FlowPluginsTs/CommunityFlowPlugins/input/inputFile/1.0.0/index.ts @@ -1,11 +1,14 @@ +import { promises as fs } from 'fs'; + import { IpluginDetails, IpluginInputArgs, IpluginOutputArgs, } from '../../../../FlowHelpers/1.0.0/interfaces/interfaces'; +import { getFileAbosluteDir } from '../../../../FlowHelpers/1.0.0/fileUtils'; /* eslint no-plusplus: ["error", { "allowForLoopAfterthoughts": true }] */ -const details = ():IpluginDetails => ({ +const details = (): IpluginDetails => ({ name: 'Input File', description: 'Start the flow with an input file', style: { @@ -17,7 +20,34 @@ const details = ():IpluginDetails => ({ requiresVersion: '2.11.01', sidebarPosition: -1, icon: '', - inputs: [], + inputs: [ + { + name: 'fileAccessChecks', + type: 'boolean', + defaultValue: 'false', + inputUI: { + type: 'dropdown', + options: [ + 'false', + 'true', + ], + }, + tooltip: 'Will check if input file and cache are readable and writable', + }, + { + name: 'pauseNodeIfAccessChecksFail', + type: 'boolean', + defaultValue: 'false', + inputUI: { + type: 'dropdown', + options: [ + 'false', + 'true', + ], + }, + tooltip: 'This will pause the node if the file access checks fail', + }, + ], outputs: [ { number: 1, @@ -27,11 +57,70 @@ const details = ():IpluginDetails => ({ }); // eslint-disable-next-line @typescript-eslint/no-unused-vars -const plugin = (args:IpluginInputArgs):IpluginOutputArgs => { +const plugin = async (args: IpluginInputArgs): Promise => { const lib = require('../../../../../methods/lib')(); // eslint-disable-next-line @typescript-eslint/no-unused-vars,no-param-reassign args.inputs = lib.loadDefaultValues(args.inputs, details); + const orignalFolder = getFileAbosluteDir(args.originalLibraryFile._id); + const { fileAccessChecks, pauseNodeIfAccessChecksFail } = args.inputs; + + const nodeID = process.argv[8]; + const { serverIP, serverPort } = args.deps.configVars.config; + + const url = `http://${serverIP}:${serverPort}/api/v2/update-node`; + + const pauseNode = async () => { + args.jobLog('Pausing node'); + const requestConfig = { + method: 'post', + url, + headers: {}, + data: { + data: { + nodeID, + nodeUpdates: { + nodePaused: true, + }, + }, + }, + }; + + await args.deps.axios(requestConfig); + args.jobLog('Node paused'); + }; + + const checkReadWrite = async (location: string) => { + try { + await fs.access(location, fs.constants.R_OK); + } catch (err) { + args.jobLog(JSON.stringify(err)); + if (pauseNodeIfAccessChecksFail) { + await pauseNode(); + } + + throw new Error(`Location not readable:${location}`); + } + + try { + await fs.access(location, fs.constants.W_OK); + } catch (err) { + args.jobLog(JSON.stringify(err)); + if (pauseNodeIfAccessChecksFail) { + await pauseNode(); + } + throw new Error(`Location not writeable:${location}`); + } + }; + + if (fileAccessChecks) { + args.jobLog('Checking file access'); + await checkReadWrite(orignalFolder); + await checkReadWrite(args.librarySettings.cache); + } else { + args.jobLog('Skipping file access checks'); + } + return { outputFileObj: args.inputFileObj, outputNumber: 1, From 45e833d2852b92bb0b134d1f9f92ab0f719dc7d6 Mon Sep 17 00:00:00 2001 From: HaveAGitGat <43864057+HaveAGitGat@users.noreply.github.com> Date: Tue, 17 Oct 2023 07:07:20 +0100 Subject: [PATCH 4/4] Update interfaces.ts --- FlowPluginsTs/FlowHelpers/1.0.0/interfaces/interfaces.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/FlowPluginsTs/FlowHelpers/1.0.0/interfaces/interfaces.ts b/FlowPluginsTs/FlowHelpers/1.0.0/interfaces/interfaces.ts index 794be8b51..917ee09e3 100644 --- a/FlowPluginsTs/FlowHelpers/1.0.0/interfaces/interfaces.ts +++ b/FlowPluginsTs/FlowHelpers/1.0.0/interfaces/interfaces.ts @@ -136,6 +136,12 @@ export interface IpluginInputArgs { axios: any, // eslint-disable-next-line @typescript-eslint/no-explicit-any crudTransDBN: (collection: string, mode: string, docID: string, obj: any)=> any, + configVars:{ + config:{ + serverIP: string, + serverPort: string, + } + } }, // eslint-disable-next-line @typescript-eslint/no-explicit-any installClassicPluginDeps: (deps: string[]) => Promise,