From 68c010780a487ee22158b403195d5e38e4fa89ba Mon Sep 17 00:00:00 2001 From: Julian Waller Date: Fri, 5 Jan 2024 16:11:02 +0000 Subject: [PATCH 1/7] wip: inspired by https://github.com/nxtedition/casparcg/pull/16 --- src/modules/ffmpeg/CMakeLists.txt | 2 + src/modules/ffmpeg/util/audio_resampler.cpp | 38 ++++++ src/modules/ffmpeg/util/audio_resampler.h | 25 ++++ src/modules/html/CMakeLists.txt | 4 +- src/modules/html/producer/html_producer.cpp | 141 ++++++++++++++++++-- 5 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 src/modules/ffmpeg/util/audio_resampler.cpp create mode 100644 src/modules/ffmpeg/util/audio_resampler.h diff --git a/src/modules/ffmpeg/CMakeLists.txt b/src/modules/ffmpeg/CMakeLists.txt index 631792e27c..082ec919c7 100644 --- a/src/modules/ffmpeg/CMakeLists.txt +++ b/src/modules/ffmpeg/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES producer/av_producer.cpp producer/av_input.cpp util/av_util.cpp + util/audio_resampler.cpp producer/ffmpeg_producer.cpp consumer/ffmpeg_consumer.cpp @@ -15,6 +16,7 @@ set(HEADERS producer/av_producer.h producer/av_input.h util/av_util.h + util/audio_resampler.h producer/ffmpeg_producer.h consumer/ffmpeg_consumer.h diff --git a/src/modules/ffmpeg/util/audio_resampler.cpp b/src/modules/ffmpeg/util/audio_resampler.cpp new file mode 100644 index 0000000000..e7f07203d1 --- /dev/null +++ b/src/modules/ffmpeg/util/audio_resampler.cpp @@ -0,0 +1,38 @@ +#include "audio_resampler.h" +#include "av_assert.h" + +extern "C" { +#include +#include +} + +namespace caspar::ffmpeg { + + AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt) + : ctx(std::shared_ptr(swr_alloc_set_opts(nullptr, + AV_CH_LAYOUT_7POINT1, + AV_SAMPLE_FMT_S32, + sample_rate, + AV_CH_LAYOUT_7POINT1, + in_sample_fmt, + sample_rate, + 0, + nullptr), + [](SwrContext* ptr) { swr_free(&ptr); })) + { + if (!ctx) + FF_RET(AVERROR(ENOMEM), "swr_alloc_set_opts"); + + FF_RET(swr_init(ctx.get()), "swr_init"); + } + + caspar::array AudioResampler::convert(int frames, const void** src) + { + auto result = caspar::array(frames * 8 * sizeof(int32_t)); + auto ptr = result.data(); + auto ret = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); + + return result; + } + + }; // namespace caspar::ffmpeg \ No newline at end of file diff --git a/src/modules/ffmpeg/util/audio_resampler.h b/src/modules/ffmpeg/util/audio_resampler.h new file mode 100644 index 0000000000..a7a6316d7d --- /dev/null +++ b/src/modules/ffmpeg/util/audio_resampler.h @@ -0,0 +1,25 @@ +#include +#include + +extern "C" { +#include +} + +struct SwrContext; + +namespace caspar::ffmpeg { + + class AudioResampler + { + std::shared_ptr ctx; + + public: + AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt); + + AudioResampler(const AudioResampler&) = delete; + AudioResampler& operator=(const AudioResampler&) = delete; + + caspar::array convert(int frames, const void** src); + }; + + }; // namespace caspar::ffmpeg \ No newline at end of file diff --git a/src/modules/html/CMakeLists.txt b/src/modules/html/CMakeLists.txt index c155dc6c0b..3f2afe42ba 100644 --- a/src/modules/html/CMakeLists.txt +++ b/src/modules/html/CMakeLists.txt @@ -25,7 +25,9 @@ target_include_directories(html PRIVATE .. ../.. ${CEF_INCLUDE_PATH} - ) + ${FFMPEG_INCLUDE_PATH} +) +target_link_libraries(html ffmpeg) set_target_properties(html PROPERTIES FOLDER modules) source_group(sources\\producer producer/*) diff --git a/src/modules/html/producer/html_producer.cpp b/src/modules/html/producer/html_producer.cpp index 2ad42d07be..599eafd858 100644 --- a/src/modules/html/producer/html_producer.cpp +++ b/src/modules/html/producer/html_producer.cpp @@ -60,13 +60,75 @@ #include #include +#include + #include "../html.h" namespace caspar { namespace html { +inline std::int_least64_t now() +{ + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count(); +} + +struct presentation_frame +{ + std::int_least64_t timestamp = now(); + core::draw_frame frame = core::draw_frame::empty(); + bool has_video = false; + bool has_audio = false; + + explicit presentation_frame() + { + } + + presentation_frame(presentation_frame&& other) + noexcept : timestamp(other.timestamp),frame(std::move(other.frame)) + { + } + + presentation_frame(const presentation_frame&) = delete; + presentation_frame& operator=(const presentation_frame&) = delete; + + presentation_frame& operator=(presentation_frame&& rhs) + { + timestamp = rhs.timestamp; + frame = std::move(rhs.frame); + return *this; + } + + ~presentation_frame() {} + + void add_audio(core::mutable_frame audio) { + if (has_audio) return; + has_audio = true; + + if (frame) { + frame = core::draw_frame::over(frame, core::draw_frame(std::move (audio))); + } else { + frame = core::draw_frame(std::move (audio)); + } + } + + void add_video(core::draw_frame video) { + if (has_video) return; + has_video = true; + + if (frame) { + frame = core::draw_frame::over(frame, std::move (video)); + } else { + frame = std::move (video); + } + } +}; + + class html_client : public CefClient , public CefRenderHandler + , public CefAudioHandler , public CefLifeSpanHandler , public CefLoadHandler , public CefDisplayHandler @@ -85,11 +147,14 @@ class html_client bool gpu_enabled_; tbb::concurrent_queue javascript_before_load_; std::atomic loaded_; - std::queue> frames_; + std::queue frames_; + core::draw_frame last_generated_frame_; mutable std::mutex frames_mutex_; const size_t frames_max_size_ = 4; std::atomic closing_; + std::unique_ptr audioResampler_; + core::draw_frame last_frame_; std::int_least64_t last_frame_time_; @@ -167,15 +232,15 @@ class html_client // Check if the sole buffered frame is too young to have a partner field generated (with a tolerance) auto time_per_frame = (1000 * 1.5) / format_desc_.fps; - auto front_frame_is_too_young = (now_time - frames_.front().first) < time_per_frame; + auto front_frame_is_too_young = (now_time - frames_.front().timestamp) < time_per_frame; if (follows_gap_in_frames && front_frame_is_too_young) { return false; } } - last_frame_time_ = frames_.front().first; - last_frame_ = std::move(frames_.front().second); + last_frame_time_ = frames_.front().timestamp; + last_frame_ = std::move(frames_.front().frame); frames_.pop(); graph_->set_value("buffered-frames", (double)frames_.size() / frames_max_size_); @@ -245,12 +310,7 @@ class html_client } private: - std::int_least64_t now() - { - return std::chrono::duration_cast( - std::chrono::high_resolution_clock::now().time_since_epoch()) - .count(); - } + void GetViewRect(CefRefPtr browser, CefRect& rect) override { @@ -302,7 +362,13 @@ class html_client { std::lock_guard lock(frames_mutex_); - frames_.push(std::make_pair(now(), core::draw_frame(std::move(frame)))); + core::draw_frame new_frame = core::draw_frame(std::move(frame)); + last_generated_frame_ = new_frame; + + presentation_frame wrapped_frame; + wrapped_frame.add_video(std::move(new_frame)); + + frames_.push(std::move(wrapped_frame)); while (frames_.size() > 4) { frames_.pop(); graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame"); @@ -353,6 +419,8 @@ class html_client CefRefPtr GetRenderHandler() override { return this; } + CefRefPtr GetAudioHandler() override { return this; } + CefRefPtr GetLifeSpanHandler() override { return this; } CefRefPtr GetLoadHandler() override { return this; } @@ -378,7 +446,7 @@ class html_client { std::lock_guard lock(frames_mutex_); - frames_.push(std::make_pair(now(), core::draw_frame::empty())); + frames_.push(presentation_frame()); } { @@ -399,6 +467,55 @@ class html_client return false; } + bool GetAudioParameters(CefRefPtr browser, CefAudioParameters& params) override + { + params.channel_layout = CEF_CHANNEL_LAYOUT_7_1; + params.sample_rate = format_desc_.audio_sample_rate; + params.frames_per_buffer = format_desc_.audio_cadence[0]; + return format_desc_.audio_cadence.size() == 1; // TODO - handle 59.94 + } + + void OnAudioStreamStarted(CefRefPtr browser, const CefAudioParameters& params, int channels) override + { + audioResampler_ = std::make_unique(params.sample_rate, AV_SAMPLE_FMT_FLTP); + } + void OnAudioStreamPacket(CefRefPtr browser, const float** data, int samples, int64_t pts) override + { + if (!audioResampler_) return; + + auto audio = audioResampler_->convert(samples, reinterpret_cast(data)); + auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc()); + + { + std::lock_guard lock(frames_mutex_); + if (frames_.empty()) { + presentation_frame wrapped_frame; + + wrapped_frame.add_audio(std::move(audio_frame)); + if (last_generated_frame_) { + wrapped_frame.add_video(last_generated_frame_); + } + + frames_.push(std::move(wrapped_frame)); + } else { + if (!frames_.back().has_audio) { + frames_.back().add_audio(std::move(audio_frame)); + } else { + presentation_frame wrapped_frame; + wrapped_frame.add_audio(std::move(audio_frame)); + frames_.push(std::move(wrapped_frame)); + } + } + + } + } + void OnAudioStreamStopped(CefRefPtr browser) override { audioResampler_ = nullptr; } + void OnAudioStreamError(CefRefPtr browser, const CefString& message) override + { + CASPAR_LOG(info) << "[html_producer] OnAudioStreamError: \"" << message.ToString() << "\""; + audioResampler_ = nullptr; + } + void do_execute_javascript(const std::wstring& javascript) { html::begin_invoke([=] { From c3b4a0521ad9aab0961be4a281bdbc3a5303aa6d Mon Sep 17 00:00:00 2001 From: Julian Waller Date: Fri, 5 Jan 2024 16:34:23 +0000 Subject: [PATCH 2/7] wip: rework frame flow --- src/modules/html/producer/html_producer.cpp | 53 ++++++++++----------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/modules/html/producer/html_producer.cpp b/src/modules/html/producer/html_producer.cpp index 599eafd858..dfffb712ab 100644 --- a/src/modules/html/producer/html_producer.cpp +++ b/src/modules/html/producer/html_producer.cpp @@ -80,8 +80,12 @@ struct presentation_frame bool has_video = false; bool has_audio = false; - explicit presentation_frame() + explicit presentation_frame(core::draw_frame video = {}) { + if (video) { + frame = std::move(video); + has_video = true; + } } presentation_frame(presentation_frame&& other) @@ -255,12 +259,13 @@ class html_client { if (!try_pop(field)) { graph_->set_tag(diagnostics::tag_severity::SILENT, "late-frame"); + return core::draw_frame::still(last_frame_); + } else { + return last_frame_; } - - return last_frame_; } - core::draw_frame last_frame() const { return last_frame_; } + core::draw_frame last_frame() const { return core::draw_frame::still(last_frame_); } bool is_ready() const { @@ -365,10 +370,7 @@ class html_client core::draw_frame new_frame = core::draw_frame(std::move(frame)); last_generated_frame_ = new_frame; - presentation_frame wrapped_frame; - wrapped_frame.add_video(std::move(new_frame)); - - frames_.push(std::move(wrapped_frame)); + frames_.push(presentation_frame(std::move(new_frame))); while (frames_.size() > 4) { frames_.pop(); graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame"); @@ -483,31 +485,26 @@ class html_client { if (!audioResampler_) return; - auto audio = audioResampler_->convert(samples, reinterpret_cast(data)); - auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc()); - - { - std::lock_guard lock(frames_mutex_); - if (frames_.empty()) { - presentation_frame wrapped_frame; + auto audio = audioResampler_->convert(samples, reinterpret_cast(data)); + auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc()); + { + std::lock_guard lock(frames_mutex_); + if (frames_.empty()) { + presentation_frame wrapped_frame(last_generated_frame_); + wrapped_frame.add_audio(std::move(audio_frame)); + + frames_.push(std::move(wrapped_frame)); + } else { + if (!frames_.back().has_audio) { + frames_.back().add_audio(std::move(audio_frame)); + } else { + presentation_frame wrapped_frame(last_generated_frame_); wrapped_frame.add_audio(std::move(audio_frame)); - if (last_generated_frame_) { - wrapped_frame.add_video(last_generated_frame_); - } - frames_.push(std::move(wrapped_frame)); - } else { - if (!frames_.back().has_audio) { - frames_.back().add_audio(std::move(audio_frame)); - } else { - presentation_frame wrapped_frame; - wrapped_frame.add_audio(std::move(audio_frame)); - frames_.push(std::move(wrapped_frame)); - } } - } + } } void OnAudioStreamStopped(CefRefPtr browser) override { audioResampler_ = nullptr; } void OnAudioStreamError(CefRefPtr browser, const CefString& message) override From b503c2e3bbb1dfab6d3df2015f0dd0559538be37 Mon Sep 17 00:00:00 2001 From: Julian Waller Date: Fri, 5 Jan 2024 16:36:53 +0000 Subject: [PATCH 3/7] chore: format --- src/modules/ffmpeg/util/audio_resampler.cpp | 48 +++++++------- src/modules/ffmpeg/util/audio_resampler.h | 22 ++++--- src/modules/html/producer/html_producer.cpp | 69 +++++++++++---------- 3 files changed, 72 insertions(+), 67 deletions(-) diff --git a/src/modules/ffmpeg/util/audio_resampler.cpp b/src/modules/ffmpeg/util/audio_resampler.cpp index e7f07203d1..ffd6e25d80 100644 --- a/src/modules/ffmpeg/util/audio_resampler.cpp +++ b/src/modules/ffmpeg/util/audio_resampler.cpp @@ -8,31 +8,31 @@ extern "C" { namespace caspar::ffmpeg { - AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt) - : ctx(std::shared_ptr(swr_alloc_set_opts(nullptr, - AV_CH_LAYOUT_7POINT1, - AV_SAMPLE_FMT_S32, - sample_rate, - AV_CH_LAYOUT_7POINT1, - in_sample_fmt, - sample_rate, - 0, - nullptr), - [](SwrContext* ptr) { swr_free(&ptr); })) - { - if (!ctx) - FF_RET(AVERROR(ENOMEM), "swr_alloc_set_opts"); +AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt) + : ctx(std::shared_ptr(swr_alloc_set_opts(nullptr, + AV_CH_LAYOUT_7POINT1, + AV_SAMPLE_FMT_S32, + sample_rate, + AV_CH_LAYOUT_7POINT1, + in_sample_fmt, + sample_rate, + 0, + nullptr), + [](SwrContext* ptr) { swr_free(&ptr); })) +{ + if (!ctx) + FF_RET(AVERROR(ENOMEM), "swr_alloc_set_opts"); - FF_RET(swr_init(ctx.get()), "swr_init"); - } + FF_RET(swr_init(ctx.get()), "swr_init"); +} - caspar::array AudioResampler::convert(int frames, const void** src) - { - auto result = caspar::array(frames * 8 * sizeof(int32_t)); - auto ptr = result.data(); - auto ret = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); +caspar::array AudioResampler::convert(int frames, const void** src) +{ + auto result = caspar::array(frames * 8 * sizeof(int32_t)); + auto ptr = result.data(); + auto ret = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); - return result; - } + return result; +} - }; // namespace caspar::ffmpeg \ No newline at end of file +}; // namespace caspar::ffmpeg \ No newline at end of file diff --git a/src/modules/ffmpeg/util/audio_resampler.h b/src/modules/ffmpeg/util/audio_resampler.h index a7a6316d7d..3f91b44cef 100644 --- a/src/modules/ffmpeg/util/audio_resampler.h +++ b/src/modules/ffmpeg/util/audio_resampler.h @@ -1,6 +1,8 @@ #include #include +#pragma once + extern "C" { #include } @@ -9,17 +11,17 @@ struct SwrContext; namespace caspar::ffmpeg { - class AudioResampler - { - std::shared_ptr ctx; +class AudioResampler +{ + std::shared_ptr ctx; - public: - AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt); + public: + AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt); - AudioResampler(const AudioResampler&) = delete; - AudioResampler& operator=(const AudioResampler&) = delete; + AudioResampler(const AudioResampler&) = delete; + AudioResampler& operator=(const AudioResampler&) = delete; - caspar::array convert(int frames, const void** src); - }; + caspar::array convert(int frames, const void** src); +}; - }; // namespace caspar::ffmpeg \ No newline at end of file +}; // namespace caspar::ffmpeg \ No newline at end of file diff --git a/src/modules/html/producer/html_producer.cpp b/src/modules/html/producer/html_producer.cpp index dfffb712ab..f5b42bd61b 100644 --- a/src/modules/html/producer/html_producer.cpp +++ b/src/modules/html/producer/html_producer.cpp @@ -69,27 +69,28 @@ namespace caspar { namespace html { inline std::int_least64_t now() { return std::chrono::duration_cast( - std::chrono::high_resolution_clock::now().time_since_epoch()) - .count(); + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count(); } struct presentation_frame { std::int_least64_t timestamp = now(); - core::draw_frame frame = core::draw_frame::empty(); - bool has_video = false; - bool has_audio = false; + core::draw_frame frame = core::draw_frame::empty(); + bool has_video = false; + bool has_audio = false; explicit presentation_frame(core::draw_frame video = {}) { if (video) { - frame = std::move(video); + frame = std::move(video); has_video = true; } } - presentation_frame(presentation_frame&& other) - noexcept : timestamp(other.timestamp),frame(std::move(other.frame)) + presentation_frame(presentation_frame&& other) noexcept + : timestamp(other.timestamp) + , frame(std::move(other.frame)) { } @@ -105,30 +106,33 @@ struct presentation_frame ~presentation_frame() {} - void add_audio(core::mutable_frame audio) { - if (has_audio) return; + void add_audio(core::mutable_frame audio) + { + if (has_audio) + return; has_audio = true; if (frame) { - frame = core::draw_frame::over(frame, core::draw_frame(std::move (audio))); + frame = core::draw_frame::over(frame, core::draw_frame(std::move(audio))); } else { - frame = core::draw_frame(std::move (audio)); + frame = core::draw_frame(std::move(audio)); } } - void add_video(core::draw_frame video) { - if (has_video) return; + void add_video(core::draw_frame video) + { + if (has_video) + return; has_video = true; if (frame) { - frame = core::draw_frame::over(frame, std::move (video)); + frame = core::draw_frame::over(frame, std::move(video)); } else { - frame = std::move (video); + frame = std::move(video); } } }; - class html_client : public CefClient , public CefRenderHandler @@ -146,16 +150,16 @@ class html_client caspar::timer paint_timer_; caspar::timer test_timer_; - spl::shared_ptr frame_factory_; - core::video_format_desc format_desc_; - bool gpu_enabled_; - tbb::concurrent_queue javascript_before_load_; - std::atomic loaded_; - std::queue frames_; - core::draw_frame last_generated_frame_; - mutable std::mutex frames_mutex_; - const size_t frames_max_size_ = 4; - std::atomic closing_; + spl::shared_ptr frame_factory_; + core::video_format_desc format_desc_; + bool gpu_enabled_; + tbb::concurrent_queue javascript_before_load_; + std::atomic loaded_; + std::queue frames_; + core::draw_frame last_generated_frame_; + mutable std::mutex frames_mutex_; + const size_t frames_max_size_ = 4; + std::atomic closing_; std::unique_ptr audioResampler_; @@ -315,8 +319,6 @@ class html_client } private: - - void GetViewRect(CefRefPtr browser, CefRect& rect) override { CASPAR_ASSERT(CefCurrentlyOn(TID_UI)); @@ -368,7 +370,7 @@ class html_client std::lock_guard lock(frames_mutex_); core::draw_frame new_frame = core::draw_frame(std::move(frame)); - last_generated_frame_ = new_frame; + last_generated_frame_ = new_frame; frames_.push(presentation_frame(std::move(new_frame))); while (frames_.size() > 4) { @@ -421,7 +423,7 @@ class html_client CefRefPtr GetRenderHandler() override { return this; } - CefRefPtr GetAudioHandler() override { return this; } + CefRefPtr GetAudioHandler() override { return this; } CefRefPtr GetLifeSpanHandler() override { return this; } @@ -483,9 +485,10 @@ class html_client } void OnAudioStreamPacket(CefRefPtr browser, const float** data, int samples, int64_t pts) override { - if (!audioResampler_) return; + if (!audioResampler_) + return; - auto audio = audioResampler_->convert(samples, reinterpret_cast(data)); + auto audio = audioResampler_->convert(samples, reinterpret_cast(data)); auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc()); { From 4356399d3eadf977c84327cd8181606a0c1447a1 Mon Sep 17 00:00:00 2001 From: Julian Waller Date: Wed, 5 Jun 2024 21:42:23 +0100 Subject: [PATCH 4/7] fix build error --- src/modules/ffmpeg/util/audio_resampler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/ffmpeg/util/audio_resampler.cpp b/src/modules/ffmpeg/util/audio_resampler.cpp index ffd6e25d80..e5d52ddc88 100644 --- a/src/modules/ffmpeg/util/audio_resampler.cpp +++ b/src/modules/ffmpeg/util/audio_resampler.cpp @@ -30,7 +30,7 @@ caspar::array AudioResampler::convert(int frames, const void** src) { auto result = caspar::array(frames * 8 * sizeof(int32_t)); auto ptr = result.data(); - auto ret = swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); + swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); return result; } From 46bf2f37468a797d9a0924d9888c1b0da03732d4 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Fri, 6 Dec 2024 21:13:22 +0100 Subject: [PATCH 5/7] fix build error after rebase --- src/modules/ffmpeg/util/audio_resampler.cpp | 30 +++++++++++---------- src/modules/ffmpeg/util/audio_resampler.h | 2 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/modules/ffmpeg/util/audio_resampler.cpp b/src/modules/ffmpeg/util/audio_resampler.cpp index e5d52ddc88..95549fa807 100644 --- a/src/modules/ffmpeg/util/audio_resampler.cpp +++ b/src/modules/ffmpeg/util/audio_resampler.cpp @@ -8,20 +8,22 @@ extern "C" { namespace caspar::ffmpeg { -AudioResampler::AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt) - : ctx(std::shared_ptr(swr_alloc_set_opts(nullptr, - AV_CH_LAYOUT_7POINT1, - AV_SAMPLE_FMT_S32, - sample_rate, - AV_CH_LAYOUT_7POINT1, - in_sample_fmt, - sample_rate, - 0, - nullptr), - [](SwrContext* ptr) { swr_free(&ptr); })) +AudioResampler::AudioResampler(int sample_rate, AVSampleFormat in_sample_fmt) { - if (!ctx) - FF_RET(AVERROR(ENOMEM), "swr_alloc_set_opts"); + AVChannelLayout channel_layout = AV_CHANNEL_LAYOUT_7POINT1; + + SwrContext* raw_ctx = nullptr; + FF(swr_alloc_set_opts2(&raw_ctx, + &channel_layout, + AV_SAMPLE_FMT_S32, + sample_rate, + &channel_layout, + in_sample_fmt, + sample_rate, + 0, + nullptr)); + + ctx = std::shared_ptr(raw_ctx, [](SwrContext* ptr) { swr_free(&ptr); }); FF_RET(swr_init(ctx.get()), "swr_init"); } @@ -35,4 +37,4 @@ caspar::array AudioResampler::convert(int frames, const void** src) return result; } -}; // namespace caspar::ffmpeg \ No newline at end of file +}; // namespace caspar::ffmpeg diff --git a/src/modules/ffmpeg/util/audio_resampler.h b/src/modules/ffmpeg/util/audio_resampler.h index 3f91b44cef..5d08f34f22 100644 --- a/src/modules/ffmpeg/util/audio_resampler.h +++ b/src/modules/ffmpeg/util/audio_resampler.h @@ -16,7 +16,7 @@ class AudioResampler std::shared_ptr ctx; public: - AudioResampler(int64_t sample_rate, AVSampleFormat in_sample_fmt); + AudioResampler(int sample_rate, AVSampleFormat in_sample_fmt); AudioResampler(const AudioResampler&) = delete; AudioResampler& operator=(const AudioResampler&) = delete; From 5497cf12c98037a57f1e8523ae0cbbcb492376c3 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Fri, 6 Dec 2024 21:17:12 +0100 Subject: [PATCH 6/7] fix audio issue related to 16 vs 8 channels --- src/modules/ffmpeg/util/audio_resampler.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/modules/ffmpeg/util/audio_resampler.cpp b/src/modules/ffmpeg/util/audio_resampler.cpp index 95549fa807..7e6a7bb820 100644 --- a/src/modules/ffmpeg/util/audio_resampler.cpp +++ b/src/modules/ffmpeg/util/audio_resampler.cpp @@ -11,10 +11,11 @@ namespace caspar::ffmpeg { AudioResampler::AudioResampler(int sample_rate, AVSampleFormat in_sample_fmt) { AVChannelLayout channel_layout = AV_CHANNEL_LAYOUT_7POINT1; - + AVChannelLayout channel_layout_out = AV_CHANNEL_LAYOUT_HEXADECAGONAL; + SwrContext* raw_ctx = nullptr; FF(swr_alloc_set_opts2(&raw_ctx, - &channel_layout, + &channel_layout_out, AV_SAMPLE_FMT_S32, sample_rate, &channel_layout, @@ -30,7 +31,7 @@ AudioResampler::AudioResampler(int sample_rate, AVSampleFormat in_sample_fmt) caspar::array AudioResampler::convert(int frames, const void** src) { - auto result = caspar::array(frames * 8 * sizeof(int32_t)); + auto result = caspar::array(frames * 16 * sizeof(int32_t)); auto ptr = result.data(); swr_convert(ctx.get(), (uint8_t**)&ptr, frames, reinterpret_cast(src), frames); From 9856962ab54aa20446e24951e80145e75a53678f Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Fri, 6 Dec 2024 23:34:23 +0100 Subject: [PATCH 7/7] reafactor frame flow --- src/modules/html/producer/html_producer.cpp | 93 ++++++++------------- 1 file changed, 37 insertions(+), 56 deletions(-) diff --git a/src/modules/html/producer/html_producer.cpp b/src/modules/html/producer/html_producer.cpp index f5b42bd61b..5fc33621e7 100644 --- a/src/modules/html/producer/html_producer.cpp +++ b/src/modules/html/producer/html_producer.cpp @@ -75,17 +75,13 @@ inline std::int_least64_t now() struct presentation_frame { - std::int_least64_t timestamp = now(); - core::draw_frame frame = core::draw_frame::empty(); - bool has_video = false; - bool has_audio = false; + std::int_least64_t timestamp; + core::draw_frame frame; - explicit presentation_frame(core::draw_frame video = {}) + explicit presentation_frame(core::draw_frame frame = {}, std::int_least64_t ts = now()) noexcept + : timestamp(ts) + , frame(std::move(frame)) { - if (video) { - frame = std::move(video); - has_video = true; - } } presentation_frame(presentation_frame&& other) noexcept @@ -97,7 +93,7 @@ struct presentation_frame presentation_frame(const presentation_frame&) = delete; presentation_frame& operator=(const presentation_frame&) = delete; - presentation_frame& operator=(presentation_frame&& rhs) + presentation_frame& operator=(presentation_frame&& rhs) noexcept { timestamp = rhs.timestamp; frame = std::move(rhs.frame); @@ -105,32 +101,6 @@ struct presentation_frame } ~presentation_frame() {} - - void add_audio(core::mutable_frame audio) - { - if (has_audio) - return; - has_audio = true; - - if (frame) { - frame = core::draw_frame::over(frame, core::draw_frame(std::move(audio))); - } else { - frame = core::draw_frame(std::move(audio)); - } - } - - void add_video(core::draw_frame video) - { - if (has_video) - return; - has_video = true; - - if (frame) { - frame = core::draw_frame::over(frame, std::move(video)); - } else { - frame = std::move(video); - } - } }; class html_client @@ -156,13 +126,15 @@ class html_client tbb::concurrent_queue javascript_before_load_; std::atomic loaded_; std::queue frames_; - core::draw_frame last_generated_frame_; + std::queue audio_frames_; mutable std::mutex frames_mutex_; + mutable std::mutex audio_frames_mutex_; const size_t frames_max_size_ = 4; std::atomic closing_; std::unique_ptr audioResampler_; + core::draw_frame last_video_frame_; core::draw_frame last_frame_; std::int_least64_t last_frame_time_; @@ -219,8 +191,21 @@ class html_client bool try_pop(const core::video_field field) { + bool result = false; std::lock_guard lock(frames_mutex_); + core::draw_frame audio_frame; + uint64_t audio_frame_timestamp = 0; + + { + std::lock_guard audio_lock(audio_frames_mutex_); + if (!audio_frames_.empty()) { + audio_frame_timestamp = audio_frames_.front().timestamp; + audio_frame = core::draw_frame(std::move(audio_frames_.front().frame)); + audio_frames_.pop(); + } + } + if (!frames_.empty()) { /* * CEF in gpu-enabled mode only sends frames when something changes, and interlaced channels @@ -248,15 +233,22 @@ class html_client } last_frame_time_ = frames_.front().timestamp; - last_frame_ = std::move(frames_.front().frame); + last_video_frame_ = std::move(frames_.front().frame); + last_frame_ = last_video_frame_; frames_.pop(); graph_->set_value("buffered-frames", (double)frames_.size() / frames_max_size_); - return true; + result = true; } - return false; + if (audio_frame) { + last_frame_time_ = audio_frame_timestamp; + last_frame_ = core::draw_frame::over(last_video_frame_, audio_frame); + result = true; + } + + return result; } core::draw_frame receive(const core::video_field field) @@ -370,10 +362,9 @@ class html_client std::lock_guard lock(frames_mutex_); core::draw_frame new_frame = core::draw_frame(std::move(frame)); - last_generated_frame_ = new_frame; frames_.push(presentation_frame(std::move(new_frame))); - while (frames_.size() > 4) { + while (frames_.size() > frames_max_size_) { frames_.pop(); graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame"); } @@ -492,21 +483,11 @@ class html_client auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc()); { - std::lock_guard lock(frames_mutex_); - if (frames_.empty()) { - presentation_frame wrapped_frame(last_generated_frame_); - wrapped_frame.add_audio(std::move(audio_frame)); - - frames_.push(std::move(wrapped_frame)); - } else { - if (!frames_.back().has_audio) { - frames_.back().add_audio(std::move(audio_frame)); - } else { - presentation_frame wrapped_frame(last_generated_frame_); - wrapped_frame.add_audio(std::move(audio_frame)); - frames_.push(std::move(wrapped_frame)); - } + std::lock_guard lock(audio_frames_mutex_); + while (audio_frames_.size() >= frames_max_size_) { + audio_frames_.pop(); } + audio_frames_.push(presentation_frame(core::draw_frame(std::move(audio_frame)))); } } void OnAudioStreamStopped(CefRefPtr browser) override { audioResampler_ = nullptr; }