Merge pull request #42 from Lumen5/LU-2860-FrameNotFound

Frame not found fix

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumen5/framefusion",
-  "version": "0.0.22",
+  "version": "0.0.23",
   "type": "module",
   "scripts": {
     "docs": "typedoc framefusion.ts",

src/backends/beamcoder.ts

@@ -94,6 +94,7 @@ const createFilter = async({
 
 const STREAM_TYPE_VIDEO = 'video';
 const COLORSPACE_RGBA = 'rgba';
+const MAX_RECURSION = 5;
 
 /**
  * A simple extractor that uses beamcoder to extract frames from a video file.
@@ -158,6 +159,12 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
      */
     #packetReadCount = 0;
 
+    /**
+     * The number of times we've recursively read packets from the demuxer to complete the frame query
+     * @private
+     */
+    #recursiveReadCount = 0;
+
     /**
      * Encoder/Decoder construction is async, so it can't be put in a regular constructor.
      * Use and await this method to generate an extractor.
@@ -206,15 +213,10 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
     }
 
     /**
-     * Duration in seconds
+     * This is the duration of the first video stream in the file expressed in seconds.
      */
     get duration(): number {
-        const time_base = this.#demuxer.streams[this.#streamIndex].time_base;
-        const durations = this.#demuxer.streams.map(
-            stream => stream.duration * time_base[0] / time_base[1]
-        );
-
-        return Math.max(...durations);
+        return this.ptsToTime(this.#demuxer.streams[this.#streamIndex].duration);
     }
 
     /**
@@ -284,9 +286,13 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
 
     /**
      * Get the frame at the given presentation timestamp (PTS)
+     * @param targetPTS - the target presentation timestamp (PTS) we want to retrieve
+     * @param SeekPTSOffset - the offset to use when seeking to the targetPTS. This is used when we have trouble finding
+     * the targetPTS. We use it to further move away from the requested PTS to find a frame. The allows use to read
+     * additional packets and find a frame that is closer to the targetPTS.
      */
-    async _getFrameAtPts(targetPTS: number) {
-        VERBOSE && console.log('_getFrameAtPts', targetPTS, '-> duration', this.duration);
+    async _getFrameAtPts(targetPTS: number, SeekPTSOffset = 0): Promise<beamcoder.Frame> {
+        VERBOSE && console.log('_getFrameAtPts', targetPTS, SeekPTSOffset, '-> duration', this.duration);
         this.#packetReadCount = 0;
 
         // seek and create a decoder when retrieving a frame for the first time or when seeking backwards
@@ -296,17 +302,16 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
         // Example: when we got a frame a 0 and request a frame at t = 30s just after, we don't want to start reading all packets
         // until 30s.
         const RE_SEEK_THRESHOLD = 3; // 3 seconds - typically we have keyframes at shorter intervals
-        const timeDifference = this.ptsToTime(Math.abs(targetPTS - (this.#packet?.pts || 0)));
-
-        VERBOSE && console.log(`timeDifference: ${timeDifference}, targetPTS: ${targetPTS}, last packet pts: ${this.#packet?.pts}`);
+        const hasFrameWithinThreshold = this.#filteredFramesPacket.flat().some(frame => {
+            return this.ptsToTime(Math.abs(targetPTS - (frame as Frame).pts)) < RE_SEEK_THRESHOLD;
+        });
+        VERBOSE && console.log('hasPreviousTargetPTS', this.#previousTargetPTS === null, 'targetPTS is smaller', this.#previousTargetPTS > targetPTS, 'has frame within threshold', hasFrameWithinThreshold);
+        if (this.#previousTargetPTS === null || this.#previousTargetPTS > targetPTS || !hasFrameWithinThreshold) {
+            VERBOSE && console.log(`Seeking to ${targetPTS - SeekPTSOffset}`);
 
-        if (this.#previousTargetPTS === null ||
-            this.#previousTargetPTS > targetPTS ||
-            timeDifference > RE_SEEK_THRESHOLD) {
-            VERBOSE && console.log(`Seeking to ${targetPTS}`);
             await this.#demuxer.seek({
                 stream_index: 0, // even though we specify the stream index, it still seeks all streams
-                timestamp: targetPTS,
+                timestamp: targetPTS + SeekPTSOffset,
                 any: false,
             });
             await this.#createDecoder();
@@ -399,8 +404,20 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
             this.#packetReadCount++;
         }
 
+        // we read through all the available packets and frames, but we still don't have a frame. This can happen
+        // when our targetPTS is to close to the end of the video. In this case, we'll try to seek further away from
+        // the end of the video and try again. We've set up a MAX_RECURSION to prevent an infinite loop.
         if (!outputFrame) {
-            throw Error('No matching frame found');
+            if (MAX_RECURSION < this.#recursiveReadCount) {
+                throw Error('No matching frame found');
+            }
+            const TIME_OFFSET = 0.1; // time offset in seconds
+            const PTSOffset = this._timeToPTS(TIME_OFFSET);
+            this.#recursiveReadCount++;
+            outputFrame = await this._getFrameAtPts(targetPTS, SeekPTSOffset - PTSOffset);
+            if (outputFrame) {
+                this.#recursiveReadCount = 0;
+            }
         }
         VERBOSE && console.log('read', this.packetReadCount, 'packets');
 
@@ -440,6 +457,7 @@ export class BeamcoderExtractor extends BaseExtractor implements Extractor {
         if (decodedFrames && decodedFrames.frames.length !== 0) {
             frames = decodedFrames.frames;
         }
+        VERBOSE && console.log(`returning ${frames.length} decoded frames`);
 
         return { packet, frames };
     }

test/framefusion.test.ts

@@ -73,6 +73,19 @@ describe('FrameFusion', () => {
         await extractor.dispose();
     });
 
+    it('can get duration when audio stream is longer than video stream', async() => {
+        // Arrange
+        const extractor = await BeamcoderExtractor.create({
+            inputFileOrUrl: 'https://storage.googleapis.com/lumen5-prod-video/fixed_tmpRjwSJC.mp4',
+        });
+
+        // Act
+        expect(extractor.duration).to.equal(12.92);
+
+        // Cleanup
+        await extractor.dispose();
+    });
+
     it('only reads a few packets to get the next frame after a seek', async() => {
         const extractor = await BeamcoderExtractor.create({
             inputFileOrUrl: 'https://storage.googleapis.com/lumen5-prod-video/mvc-4k-new-orleans-a053c0340725rv-112014WA74Rf.mp4',
@@ -207,6 +220,41 @@ describe('FrameFusion', () => {
         await extractor.dispose();
     });
 
+    it('can get frame towards to end when decoder is flushed', async() => {
+        // Arrange
+        const extractor = await BeamcoderExtractor.create({
+            inputFileOrUrl: 'https://storage.googleapis.com/lumen5-prod-video/fixed_tmpRjwSJC.mp4',
+        });
+
+        // Act
+        await extractor.getFrameAtTime(12.866667);
+        const frame = await extractor.getFrameAtTime(12.9);
+
+        // Assert
+        expect(frame).to.not.be.null;
+        expect(frame.pts).to.equal(321000);
+
+        // Cleanup
+        await extractor.dispose();
+    });
+
+    it('can get frame towards to end when no packets are available', async() => {
+        // Arrange
+        const extractor = await BeamcoderExtractor.create({
+            inputFileOrUrl: 'https://storage.googleapis.com/lumen5-prod-video/fixed_tmpRjwSJC.mp4',
+        });
+
+        // Act
+        const frame = await extractor.getFrameAtTime(12.9);
+
+        // Assert
+        expect(frame).to.not.be.null;
+        expect(frame.pts).to.equal(321000);
+
+        // Cleanup
+        await extractor.dispose();
+    });
+
     it('can get frames at random times (forward and backward)', async() => {
         // Arrange
         const extractor = await BeamcoderExtractor.create({