From 9d0e84f85b5fadbcf4c693edfe29bc1079e7ffb1 Mon Sep 17 00:00:00 2001 From: Andrew Wason Date: Wed, 29 Nov 2023 15:05:41 -0500 Subject: [PATCH] WIP --- src/mediafx/image_clip.cpp | 2 +- src/mediafx/image_clip.h | 2 +- src/mediafx/rate_control.cpp | 19 ++++++++++++------- src/mediafx/rate_control.h | 1 + src/mediafx/video_clip.cpp | 14 +++++++++----- src/mediafx/video_clip.h | 3 ++- src/mediafx/visual_clip.cpp | 2 +- src/mediafx/visual_clip.h | 2 +- 8 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/mediafx/image_clip.cpp b/src/mediafx/image_clip.cpp index d3cdc69..e65d49d 100644 --- a/src/mediafx/image_clip.cpp +++ b/src/mediafx/image_clip.cpp @@ -47,7 +47,7 @@ void ImageClip::loadMedia(const QUrl& url) videoFrame.unmap(); } -bool ImageClip::prepareNextVideoFrame() +bool ImageClip::prepareNextVideoFrame(const Interval& globalTime) { // XXX timestamp frame based on nextClipTime() ? (in microseconds, not milliseconds) setCurrentVideoFrame(videoFrame); diff --git a/src/mediafx/image_clip.h b/src/mediafx/image_clip.h index 49f0a53..0313657 100644 --- a/src/mediafx/image_clip.h +++ b/src/mediafx/image_clip.h @@ -35,7 +35,7 @@ class ImageClip : public VisualClip { protected: void loadMedia(const QUrl&) override; - bool prepareNextVideoFrame() override; + bool prepareNextVideoFrame(const Interval& globalTime) override; void stop() override; diff --git a/src/mediafx/rate_control.cpp b/src/mediafx/rate_control.cpp index 8def02a..5aebbeb 100644 --- a/src/mediafx/rate_control.cpp +++ b/src/mediafx/rate_control.cpp @@ -35,6 +35,8 @@ // XXX need to keep buffer partially full, scale to keep 5 frames in buffer somehow +// track frame enqueue time, and growth rate of buffer, and tweak playback to control growth relative to desired size? +// /*XXX qint64 movingAverage(qint64 average, qint64 sample) { @@ -44,24 +46,25 @@ qint64 movingAverage(qint64 average, qint64 sample) return average; } */ +const int desiredBufferSize = 5; // XXX make member - fold into rateScalar qreal RateControl::playbackRate() const { - qreal rate = 10.0; - if (isFrameRequiredTimerValid && isEnqueueTimerValid) { + qreal rate = desiredBufferSize; + if (!bufferedFrames.isEmpty() && timeSinceFrameLastRequired && isEnqueueTimerValid) { auto timeSinceLastEnqueued = enqueueTimer.durationElapsed().count(); - auto timeSinceLastRequired = frameRequiredTimer.durationElapsed().count(); - rate = (timeSinceLastEnqueued / (qreal)timeSinceLastRequired) * rateScalar; - qCritical() << "rateScalar" << rateScalar << "rate" << rate << "frames" << bufferedFrames.size() << "timeSinceLastEnqueued" << timeSinceLastEnqueued << "timeSinceLastRequired" << timeSinceLastRequired; // XXX + rate = (timeSinceLastEnqueued / (qreal)timeSinceFrameLastRequired) + * (desiredBufferSize / (qreal)bufferedFrames.size()) + * rateScalar; + qCritical() << "rateScalar" << rateScalar << "rate" << rate << "frames" << bufferedFrames.size() << "timeSinceLastEnqueued" << timeSinceLastEnqueued << "timeSinceFrameLastRequired" << timeSinceFrameLastRequired; // XXX } return rate; } void RateControl::onVideoFrameRequired() { - // Valid when we are called the 2nd time if (frameRequiredTimer.isValid()) - isFrameRequiredTimerValid = true; + timeSinceFrameLastRequired = frameRequiredTimer.durationElapsed().count(); frameRequiredTimer.start(); } @@ -76,6 +79,7 @@ void RateControl::enqueue(const QVideoFrame& videoFrame) isEnqueueTimerValid = true; enqueueTimer.start(); bufferedFrames.enqueue(videoFrame); + qCritical() << "enqueue" << bufferedFrames.size(); // XXX } void RateControl::reset() @@ -84,4 +88,5 @@ void RateControl::reset() bufferedFrames.squeeze(); frameRequiredTimer.invalidate(); enqueueTimer.invalidate(); + timeSinceFrameLastRequired = 0; } \ No newline at end of file diff --git a/src/mediafx/rate_control.h b/src/mediafx/rate_control.h index 92a2850..9ae5c64 100644 --- a/src/mediafx/rate_control.h +++ b/src/mediafx/rate_control.h @@ -38,6 +38,7 @@ class RateControl : public QObject { private: QElapsedTimer frameRequiredTimer; + qint64 timeSinceFrameLastRequired = 0; bool isFrameRequiredTimerValid = false; QElapsedTimer enqueueTimer; bool isEnqueueTimerValid = false; diff --git a/src/mediafx/video_clip.cpp b/src/mediafx/video_clip.cpp index b848d03..68450cc 100644 --- a/src/mediafx/video_clip.cpp +++ b/src/mediafx/video_clip.cpp @@ -51,7 +51,9 @@ void VideoClip::loadMedia(const QUrl& url) void VideoClip::onRateControl() { - mediaPlayer.setPlaybackRate(rateControl.playbackRate()); + auto rate = rateControl.playbackRate(); + qCritical() << "onRateControl" << rate; // XXX + mediaPlayer.setPlaybackRate(rate); } void VideoClip::onVideoFrameChanged(const QVideoFrame& frame) @@ -69,7 +71,7 @@ void VideoClip::onVideoFrameChanged(const QVideoFrame& frame) QMetaObject::invokeMethod(this, &VideoClip::onRateControl, Qt::QueuedConnection); } -bool VideoClip::prepareNextVideoFrame() +bool VideoClip::prepareNextVideoFrame(const Interval& globalTime) { QMutexLocker locker(&mutex); auto nextFrameTime = nextClipTime(); @@ -81,9 +83,11 @@ bool VideoClip::prepareNextVideoFrame() if (videoFrame.isValid() && nextFrameTime.contains(videoFrame.startTime())) { return true; } - if (rateControl.isEmpty()) - qCritical() << "prepareNextVideoFrame BUFFER EMPTY"; // XXX - rateControl.onVideoFrameRequired(); + qCritical() << "prepareNextVideoFrame buffer empty" << rateControl.isEmpty(); // XXX + if (lastRequestedGlobalTime != globalTime) { + rateControl.onVideoFrameRequired(); + lastRequestedGlobalTime = globalTime; + } while (!rateControl.isEmpty()) { videoFrame = rateControl.dequeue(); if (nextFrameTime.contains(videoFrame.startTime())) { diff --git a/src/mediafx/video_clip.h b/src/mediafx/video_clip.h index eb39bc0..3072095 100644 --- a/src/mediafx/video_clip.h +++ b/src/mediafx/video_clip.h @@ -42,7 +42,7 @@ class VideoClip : public VisualClip { protected: void loadMedia(const QUrl&) override; - bool prepareNextVideoFrame() override; + bool prepareNextVideoFrame(const Interval& globalTime) override; void onActiveChanged(bool active) override; @@ -59,4 +59,5 @@ private slots: QVideoSink mediaPlayerSink; QMutex mutex; RateControl rateControl; + Interval lastRequestedGlobalTime; }; diff --git a/src/mediafx/visual_clip.cpp b/src/mediafx/visual_clip.cpp index 70129ba..0274805 100644 --- a/src/mediafx/visual_clip.cpp +++ b/src/mediafx/visual_clip.cpp @@ -32,7 +32,7 @@ void VisualClip::setVideoSinks(const QList& videoSinks) bool VisualClip::renderClip(const Interval& globalTime) { - if (!prepareNextVideoFrame()) + if (!prepareNextVideoFrame(globalTime)) return false; for (auto videoSink : videoSinks()) { videoSink->setVideoFrame(m_currentVideoFrame); diff --git a/src/mediafx/visual_clip.h b/src/mediafx/visual_clip.h index 544b87f..d8feab0 100644 --- a/src/mediafx/visual_clip.h +++ b/src/mediafx/visual_clip.h @@ -40,7 +40,7 @@ class VisualClip : public Clip { void setVideoSinks(const QList&); bool renderClip(const Interval& globalTime) override; - virtual bool prepareNextVideoFrame() = 0; + virtual bool prepareNextVideoFrame(const Interval& globalTime) = 0; QVideoFrame currentVideoFrame() const { return m_currentVideoFrame; }; void setCurrentVideoFrame(const QVideoFrame& videoFrame) { m_currentVideoFrame = videoFrame; };