diff --git a/db/schema.sql b/db/schema.sql index 3e155fd..1bf05ff 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -53,7 +53,7 @@ CREATE TABLE takes( filepath TEXT, downloaded BOOLEAN NOT NULL CHECK (downloaded IN (0,1)), rating INTEGER CHECK (rating BETWEEN 0 AND 5), - metadata_json JSON + metadata_json JSON NOT NULL ); CREATE TRIGGER AutoGenerateUid diff --git a/package-lock.json b/package-lock.json index a70effc..94b4d82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3356,24 +3356,6 @@ "pend": "~1.2.0" } }, - "ffmpeg-stream": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/ffmpeg-stream/-/ffmpeg-stream-0.6.0.tgz", - "integrity": "sha512-SnfSkBftW8Zhk12BbA5IXrCFovH2seOs84eafGq7/lLol38KrDz6xbRVqc8hby81L9SdMOAB4JJEpd7S9k1pQg==", - "requires": { - "debug": "^4.1.1" - }, - "dependencies": { - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", diff --git a/package.json b/package.json index 099272c..45ffe20 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "debug": "4.1.1", "ejs": "3.0.1", "express": "4.17.1", - "ffmpeg-stream": "0.6.0", "fs-extra": "8.1.0", "got": "11.3.0", "method-override": "3.0.0", diff --git a/public/js/index.js b/public/js/index.js index 0e8d378..83406ce 100644 --- a/public/js/index.js +++ b/public/js/index.js @@ -6,9 +6,14 @@ import { PlaceholderController, } from './schedule.js' +import VideoPlayer from './video-player.js' + const application = Stimulus.Application.start() + application.register('schedule', ScheduleController) application.register('schedule-event', ScheduleEventController) application.register('inline-editor', InlineEditorController) // form-based Inline Editor application.register('schedule-note', ScheduleNoteController) // similar, but with special validation application.register('placeholder', PlaceholderController) + +application.register('video-player', VideoPlayer) diff --git a/public/js/video-player.js b/public/js/video-player.js new file mode 100644 index 0000000..245c7a5 --- /dev/null +++ b/public/js/video-player.js @@ -0,0 +1,168 @@ +const { createMachine, assign, interpret } = XStateFSM + +function sum (a, b) { + return a + b +} + +const machineConfig = { + id: 'player', + initial: 'inactive', + context: { + segments: [], + curr: 0 + }, + states: { + inactive: { + entry: ['deactivate', 'setSrc'], + on: { CLICK: 'activating' } + }, + activating: { + entry: 'activate', + on: { ACTIVE: 'playing' } + }, + playing: { + entry: ['startTimer', 'play'], + exit: ['stopTimer'], + on: { + SEGMENT_ENDED: [ + { + cond: context => context.curr < context.segments.length - 1, + actions: ['nextSegment', 'setSrc'] + }, + { + target: 'inactive', + actions: ['reset'] + } + ] + } + } + } +} + +export default class VideoPlayer extends Stimulus.Controller { + static targets = [ 'video', 'segment', 'invitation', 'controls', 'status', 'progress', 'current', 'statusShot', 'statusTake' ] + + static UPDATE_INTERVAL_MS = 1000 / 20 // update the progress bar at 20 fps + + initialize () { + this.intervalId = null + this.service = interpret( + createMachine({ + ...machineConfig, + context: { + segments: this.getSegments(this.segmentTargets), + curr: 0 + } + }, { + actions: { + deactivate: this.deactivate.bind(this), + activate: this.activate.bind(this), + play: this.play.bind(this), + setSrc: this.setSrc.bind(this), + startTimer: this.startTimer.bind(this), + stopTimer: this.stopTimer.bind(this), + + // + // + // context assignment actions + // + reset: assign({ curr: 0 }), + nextSegment: assign({ curr: context => context.curr + 1 }) + } + }) + ).start() + + // for debugging: + // this.service.subscribe(state => console.log(state.value)) + } + + // + // + // Controller methods + // + getSegments (el) { + let segments = [] + for (let segment of el) { + const { href } = segment + const { takeId, takeNumber, sceneNumber, shotNumber, impromptu, posterframe, duration } = segment.dataset + segments.push({ + id: takeId, + takeNumber, + src: href, + posterframe, + sceneNumber, + shotNumber, + impromptu: impromptu == '' ? true : false, + duration: parseFloat(duration) + }) + } + return segments + } + + // + // + // State Machine actions + // + deactivate (context) { + this.invitationTarget.style.display = 'flex' + this.controlsTarget.style.opacity = 0.3 + this.videoTarget.pause() + this.statusTarget.innerText = 'Paused' + } + activate (context) { + this.invitationTarget.style.display = 'none' + this.controlsTarget.style.opacity = 1 + + this.service.send('ACTIVE') + } + play (context, event) { + this.videoTarget.play() + this.statusTarget.innerText = 'Playing' + } + setSrc (context) { + let { curr } = context + let take = context.segments[curr] + let { src } = take + + this.videoTarget.src = src + this.currentTarget.innerText = `${curr + 1} / ${context.segments.length}` + + this.statusShotTarget.innerText = `Shot ${take.impromptu ? 'i' : ''}${take.shotNumber}` + this.statusTakeTarget.innerText = `Take ${take.takeNumber}` + } + startTimer () { + this.intervalId = setInterval( + () => this.timeUpdate({ target: this.videoTarget }), + VideoPlayer.UPDATE_INTERVAL_MS + ) + } + stopTimer () { + clearTimeout(this.intervalId) + } + + // + // + // Controller events + // + startPlayback (event) { + this.service.send('CLICK') + } + ended (event) { + this.service.send('SEGMENT_ENDED') + } + timeUpdate (event) { + let video = event.target + + if (video.readyState) { + let { segments, curr } = this.service.state.context + let total = segments.map(s => s.duration).reduce(sum) + let passed = 0 + for (let i = 0; i < curr; i++) { + passed += segments[i].duration + } + let elapsed = passed + video.currentTime + let pct = (elapsed / total) * 100 + this.progressTarget.style.width = `${pct}%` + } + } +} diff --git a/public/js/xstate.fsm.umd.js b/public/js/xstate.fsm.umd.js new file mode 100644 index 0000000..1da4378 --- /dev/null +++ b/public/js/xstate.fsm.umd.js @@ -0,0 +1 @@ +var t,e;t=this,e=function(t){"use strict";var e;(e=t.InterpreterStatus||(t.InterpreterStatus={}))[e.NotStarted=0]="NotStarted",e[e.Running=1]="Running",e[e.Stopped=2]="Stopped";const n={type:"xstate.init"};function i(t){return void 0===t?[]:[].concat(t)}function s(t,e){return"string"==typeof(t="string"==typeof t&&e&&e[t]?e[t]:t)?{type:t}:"function"==typeof t?{type:t.name,exec:t}:t}function r(t){return e=>t===e}function o(t){return"string"==typeof t?{type:t}:t}function a(t,e){return{value:t,context:e,actions:[],changed:!1,matches:r(t)}}const c=(t,e)=>t.actions.forEach(({exec:n})=>n&&n(t.context,e));t.assign=function(t){return{type:"xstate.assign",assignment:t}},t.createMachine=function(t,e={}){const n={config:t,_options:e,initialState:{value:t.initial,actions:i(t.states[t.initial].entry).map(t=>s(t,e.actions)),context:t.context,matches:r(t.initial)},transition:(e,c)=>{const{value:u,context:f}="string"==typeof e?{value:e,context:t.context}:e,p=o(c),g=t.states[u];if(g.on){const e=i(g.on[p.type]);for(const i of e){if(void 0===i)return a(u,f);const{target:e=u,actions:o=[],cond:c=(()=>!0)}="string"==typeof i?{target:i}:i;let d=f;if(c(f,p)){const i=t.states[e];let a=!1;const c=[].concat(g.exit,o,i.entry).filter(t=>t).map(t=>s(t,n._options.actions)).filter(t=>{if("xstate.assign"===t.type){a=!0;let e=Object.assign({},d);return"function"==typeof t.assignment?e=t.assignment(d,p):Object.keys(t.assignment).forEach(n=>{e[n]="function"==typeof t.assignment[n]?t.assignment[n](d,p):t.assignment[n]}),d=e,!1}return!0});return{value:e,context:d,actions:c,changed:e!==u||c.length>0||a,matches:r(e)}}}}return a(u,f)}};return n},t.interpret=function(e){let i=e.initialState,s=t.InterpreterStatus.NotStarted;const r=new Set,a={_machine:e,send:n=>{s===t.InterpreterStatus.Running&&(i=e.transition(i,n),c(i,o(n)),r.forEach(t=>t(i)))},subscribe:t=>(r.add(t),t(i),{unsubscribe:()=>r.delete(t)}),start:()=>(s=t.InterpreterStatus.Running,c(i,n),a),stop:()=>(s=t.InterpreterStatus.Stopped,r.clear(),a),get state(){return i},get status(){return s}};return a},Object.defineProperty(t,"__esModule",{value:!0})},"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t=t||self).XStateFSM={}); diff --git a/server/index.js b/server/index.js index 5a77b30..32b8a9f 100644 --- a/server/index.js +++ b/server/index.js @@ -145,18 +145,15 @@ app.get('/status', status.index) // TODO await visualSlateRenderer.start() +rtspClient.start() bus .on('takes/create', async ({ id }) => { console.log('[server] RTSP client START recording stream for take', id) - try { - await rtspClient.startup({ uri: ZCAM_RTSP_URL, takeId: id }) - } catch (err) { - console.error('[server] RTSP client error', err) - } + rtspClient.send({ type: 'REC_START', src: ZCAM_RTSP_URL, takeId: id }) }) .on('takes/cut', () => { console.log('[server] RTSP client STOP recording stream') - rtspClient.shutdown() + rtspClient.send('REC_STOP') }) // Z Cam connections @@ -200,6 +197,7 @@ async function shutdown () { await webSocketServer.stop() await downloader.stop() await zcamWsRelay.stop() + await rtspClient.stop() if (livereload) { livereload.stop() } diff --git a/server/routes/scenes.js b/server/routes/scenes.js index e9a82df..f17ab3d 100644 --- a/server/routes/scenes.js +++ b/server/routes/scenes.js @@ -1,9 +1,15 @@ +const { spawnSync } = require('child_process') +const path = require('path') +const { UPLOADS_PATH } = require('../config') + const { get, all } = require('../db') const Scene = require('../decorators/scene') const Shot = require('../decorators/shot') const Take = require('../decorators/take') +const keyBy = id => (prev, curr) => (prev[curr[id]] = curr, prev) + exports.index = (req, res) => { let { projectId } = req.params @@ -12,13 +18,13 @@ exports.index = (req, res) => { let scenes = all(` SELECT scenes.*, - COUNT(shots.id) AS shots_count, - SUM(shots.duration) AS shots_duration, - COUNT(takes.id) AS takes_count + COUNT(DISTINCT shots.id) AS shots_count, + COUNT(DISTINCT takes.id) AS takes_count, + SUM(DISTINCT shots.duration) AS shots_duration FROM scenes - LEFT OUTER JOIN shots ON scenes.id = shots.scene_id - LEFT OUTER JOIN takes ON scenes.id = takes.scene_id + LEFT JOIN takes ON takes.scene_id = scenes.id + LEFT JOIN shots ON shots.scene_id = scenes.id WHERE scenes.project_id = ? GROUP BY 1 @@ -26,7 +32,11 @@ exports.index = (req, res) => { projectId ) - let shots = all('SELECT * FROM shots WHERE project_id = ?', projectId) + let shots = all( + `SELECT * FROM shots + WHERE project_id = ? + ORDER BY impromptu ASC, shot_number + `, project.id) res.render('scenes', { project, @@ -54,12 +64,11 @@ exports.show = (req, res) => { ) let shots = all( - `SELECT * - FROM shots - WHERE scene_id = ? - AND project_id = ?`, - sceneId, projectId - ) + `SELECT * FROM shots + WHERE project_id = ? + AND scene_id = ? + ORDER BY impromptu ASC, shot_number + `, project.id, scene.id) let { project_scenes_count } = get( `SELECT COUNT(id) as project_scenes_count @@ -68,48 +77,64 @@ exports.show = (req, res) => { projectId ) - let bestTakesByShotId = {} - for (let shot of shots) { - // best or most recent take - // TODO optimize queries - let mostRecent = get( - `SELECT * - FROM takes - WHERE shot_id = ? - AND project_id = ? - ORDER BY datetime(cut_at) - LIMIT 1`, - shot.id, - projectId - ) - let highestRated = get( - `SELECT * - FROM takes - WHERE rating IS NOT NULL - AND shot_id = ? - AND project_id = ? - ORDER BY rating - LIMIT 1`, - shot.id, - projectId - ) - let take = highestRated || mostRecent || null - // TODO optimize this - bestTakesByShotId[shot.id] = take - ? - take.downloaded - ? { - downloaded: take.downloaded, - src: new Take(take).filenameForThumbnail({ - ...{ scene_number } = scene, - ...{ shot_number, impromptu } = shot - }) - } - : { - downloaded: take.downloaded, - src: null - } - : null } + let bestTakesByShotId = all( + ` + SELECT + best_take.id, + best_take.downloaded, + best_take.rating, + best_take.cut_at, + best_take.metadata_json, + best_take.take_number, + + shots.id as 'shot_id', + shots.shot_number as 'shot_number', + shots.impromptu as 'impromptu', + + scenes.scene_number as 'scene_number' + FROM + shots + JOIN scenes ON scenes.id = shots.scene_id + + -- find the best take + -- e.g.: the single highest rated, most-recent take + JOIN takes as best_take ON (best_take.id = ( + SELECT + id + FROM + takes + WHERE + shot_id = shots.id + ORDER BY + rating DESC, + datetime(cut_at) DESC + LIMIT 1 + )) + AND + shots.scene_id = ? + ORDER BY + impromptu ASC, + shot_number + `, + scene.id + ) + .map(best => { + let take = new Take(best) + return { + ...take, + src: { + thumbnail: take.filenameForThumbnail({ + ...{ scene_number } = best, + ...{ shot_number, impromptu } = best + }), + stream: take.filenameForStream({ + ...{ scene_number } = best, + ...{ shot_number, impromptu } = best + }) + } + } + }) + .reduce(keyBy('shot_id'), {}) let takesCountByShotId = {} for (let shot of shots) { @@ -123,6 +148,14 @@ exports.show = (req, res) => { takesCountByShotId[shot.id] = takes_count } + let previewTakes = [] + for (let shot of shots) { + let take = bestTakesByShotId[shot.id] + if (take && take.downloaded) { + previewTakes.push(take) + } + } + res.render('scene', { project, scene: new Scene(scene), @@ -130,6 +163,8 @@ exports.show = (req, res) => { project_scenes_count, takesCountByShotId, - bestTakesByShotId + bestTakesByShotId, + + previewTakes }) } diff --git a/server/services/takes/create.js b/server/services/takes/create.js index dde47b3..e6112f5 100644 --- a/server/services/takes/create.js +++ b/server/services/takes/create.js @@ -13,12 +13,14 @@ module.exports = function create ({ projectId, sceneId, shotId, at }) { (project_id, scene_id, shot_id, take_number, ready_at, - downloaded) + downloaded, + metadata_json) VALUES (?, ?, ?, ?, ?, - 0 + 0, + '{}' )`, projectId, sceneId, shotId, take_number, diff --git a/server/systems/downloader.js b/server/systems/downloader.js index 7fb7a10..ee4d97a 100644 --- a/server/systems/downloader.js +++ b/server/systems/downloader.js @@ -347,8 +347,8 @@ const verifyFrameCount = (context, event) => (callback, onReceive) => { let expected = info.vcnt let actual = parseFloat(stdout.toString()) - // accept within +/- 2 frames - if (Math.abs(expected - actual) <= 2) { + // accept within +/- 10 frames + if (Math.abs(expected - actual) <= 10) { callback('SUCCESS') } else { callback({ @@ -511,7 +511,7 @@ const copyFilesAndMarkComplete = (context, event) => (callback, onReceive) => { `UPDATE takes SET downloaded = 1, - metadata_json = ? + metadata_json = json_patch(metadata_json, ?) WHERE id = ?`, JSON.stringify(metadata), take.id diff --git a/server/systems/rtsp-client.js b/server/systems/rtsp-client.js index 98d17b8..0eb74ed 100644 --- a/server/systems/rtsp-client.js +++ b/server/systems/rtsp-client.js @@ -1,58 +1,205 @@ -const { Converter } = require('ffmpeg-stream') const fs = require('fs-extra') const path = require('path') +const { createMachine, interpret, assign, forwardTo } = require('xstate') +const { spawn, spawnSync } = require('child_process') + +const debug = require('debug')('shotcore:rtsp-client') const { UPLOADS_PATH } = require('../config') -const { get } = require('../db') +const { run, get } = require('../db') const Take = require('../decorators/take') -// const { createStreamWithVisualSlate } = require('../systems/visual-slate') - -let converter -let running = false - -async function startup ({ uri, takeId }) { - if (running) { - console.warn('[rtsp-client] startup() called but it is already in progress') - return +const machine = { + id: 'rtsp-client', + initial: 'idle', + strict: true, + context: {}, + states: { + idle: { + on: { + REC_START: 'recording' + } + }, + recording: { + entry: ['setContextFromTakeById'], + invoke: { + id: 'recorder', + src: 'recordingService' + }, + on: { + REC_STOP: { actions: forwardTo('recorder') }, + ERROR: 'failure', + SUCCESS: 'processing' + } + }, + processing: { + invoke: { + id: 'processor', + src: 'processingService' + }, + on: { + SET_DURATION: { + actions: ['setDuration'] + }, + SUCCESS: 'idle' + }, + exit: ['updateTake', 'clearContext'] + }, + failure: { + entry: (context, event) => console.error(context, event) + }, + off: { + entry: ['clearContext'], + final: true + } + }, + on: { + OFF: 'off' } +} - console.log('[rtsp-client] startup()') - running = true - - let take = get('SELECT * FROM takes WHERE id = ?', takeId) +const getDataByTakeId = id => { + let take = get('SELECT * FROM takes WHERE id = ?', id) + take = new Take(take) let { scene_number } = get(`SELECT scene_number from scenes WHERE id = ?`, take.scene_id) let { shot_number, impromptu } = get(`SELECT shot_number, impromptu from shots WHERE id = ?`, take.shot_id) - let dirname = path.join('projects', take.project_id.toString(), 'takes') + let filename = take.filenameForStream({ + scene_number, + shot_number, impromptu + }) + + let dirname = path.join(UPLOADS_PATH, 'projects', take.project_id.toString(), 'takes') + let dst = path.join(dirname, filename) - let filename = Take.filenameForStream({ + return { + take: new Take(take), scene_number, shot_number, impromptu, - take_number: take.take_number, - id: take.id + + dst + } +} + +const getFileDuration = src => { + let { stdout, stderr } = spawnSync( + 'ffprobe', [ + '-v', 'error', + '-select_streams', 'v:0', + '-show_entries', 'format=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', + src + ] + ) + stdout = stdout.toString().trim() + stderr = stderr.toString().trim() + if (stderr) { + console.error(stderr) + throw new Error(`Error getting duration of file ${src}\n` + stderr) + } + return parseFloat(stdout) +} + +const recordingService = (context, event) => (callback, onReceive) => { + debug('recordingService') + debug({ context, event}) + + let complete = false + let closing = false + + let { src, dst } = context + + // TODO record to tmp folder and copy afterward + + debug('preparing to record') + debug('src:', src) + debug('dst:', dst) + + // TODO better error handling + if (fs.existsSync(dst)) { + throw new Error('File already exists!') + } + + fs.mkdirpSync(path.dirname(dst)) + + let child = spawn( + 'ffmpeg', [ + '-loglevel', 'error', + // input + '-i', src, + // transport + '-rtsp_transport', 'udp+tcp', + + // force start time to 0 + // NOTE: this may only be required for the zcam-mock-server RTSP server + // real Z Cam RTSP server might not need it? + '-vf', 'setpts=PTS-STARTPTS', + + // never overwrite + '-n', + // output + dst + ] + ) + + child.stdout.on('data', data => debug(data.toString())) + + child.stderr.on('data', data => console.error(data.toString())) + + child.on('error', err => { + callback({ type: 'ERROR', error: err }) }) - let filepath = path.join(dirname, filename) - fs.mkdirpSync(path.join(UPLOADS_PATH, dirname)) + child.on('close', (code, signal) => { + if (complete) { + console.warn('unexpected multiple close events from ffmpeg process') + return + } - converter = new Converter() + complete = true + if (signal || code !== 0) { + debug('close', { code, signal }) - console.log(`[rtsp-client] - from: ${uri}`) - converter - .createInputFromFile(uri, { rtsp_transport: 'udp+tcp' }) + if (closing && code === 255) { + return callback('SUCCESS') + } - console.log(`[rtsp-client] - to: ${UPLOADS_PATH}/${filepath}`) + if (signal) { + callback({ type: 'ERROR', error: new Error(`exited via signal ${signal}`) }) + } else { + callback({ type: 'ERROR', error: new Error(`exited with code ${code}`) }) + } + } + }) - converter - .createOutputToFile(path.join(UPLOADS_PATH, dirname, filename), { codec: 'copy' }) + onReceive(event => { + if (event.type === 'REC_STOP') { + debug('done! closing …') + closing = true + child.kill() + } + }) - await converter.run() + return () => { + if (complete == false) { + debug('recording interrupted. terminating early …') + child.kill() + } + } +} +const processingService = (context, event) => (callback, onReceive) => { + debug('calculating duration') + + let duration = getFileDuration(context.dst) + callback({ type: 'SET_DURATION', duration }) + + // TODO visual slate? + // // await createStreamWithVisualSlate({ - // inpath: path.join(UPLOADS_PATH, dirname, filename), + // inpath: path.join(UPLOADS_PATH, dirname, filename), // outpath: path.join(UPLOADS_PATH, dirname, filename), // // TODO don't hardcode // frameLengthInSeconds: 1001/24000, @@ -61,21 +208,82 @@ async function startup ({ uri, takeId }) { // height: 1080 // } // }) + + callback('SUCCESS') +} + +const updateTake = (context, event) => { + let { id } = context.take + let { duration } = context + let { dst } = context + + let { changes, lastInsertRowid } = run(` + UPDATE takes + SET metadata_json = json_patch(metadata_json, ?) + WHERE id = ? + `, + JSON.stringify( + { + preview: { + filename: path.basename(dst), + duration + } + } + ), + id) } -async function shutdown () { - console.log('[rtsp-client] shutdown()') - try { - console.log('[rtsp-client] killing current process …') - converter.kill() - } catch (err) { - // .kill() throws Error of { code: 1 } - // console.error('[rtsp-client] ERROR', err) +const service = interpret( + createMachine( + machine, + { + services: { + recordingService, + processingService + }, + actions: { + updateTake, + setContextFromTakeById: assign((context, event) => { + let { src } = event + return { + src, + ...getDataByTakeId(event.takeId) + } + }), + setDuration: assign({ duration: (context, event) => event.duration }), + clearContext: assign({}) + }, + guards: { + // + } + } + ) +) +.onTransition(event => debug( + '->', event.value, + event.context.take ? `take:${event.context.take.id}` : '', + event.context.duration ? `take:${event.context.duration}` : '' +)) + +async function start () { + service.start() + return true +} + +async function stop () { + if (service.initialized) { + service.send('OFF') + service.stop() } - running = false + return true +} + +function send (event) { + service.send(event) } module.exports = { - startup, - shutdown + start, + stop, + send } diff --git a/server/views/_video-player.ejs b/server/views/_video-player.ejs new file mode 100644 index 0000000..1514c42 --- /dev/null +++ b/server/views/_video-player.ejs @@ -0,0 +1,140 @@ +<% if (takes.length) { %> +