Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Text to speech in reader mode added. #19213

Merged
merged 5 commits into from
Sep 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 31 additions & 23 deletions browser/speedreader/page_distiller.cc
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,20 @@ void PageDistiller::GetDistilledText(DistillContentCallback callback) {
weak_factory_.GetWeakPtr(), std::move(callback)));
}

void PageDistiller::GetTextToSpeak(TextToSpeechContentCallback callback) {
if (state_ != State::kDistilled) {
return std::move(callback).Run(base::Value());
}

constexpr const char16_t kGetTextToSpeak[] = uR"js( extractTextToSpeak() )js";

web_contents_->GetPrimaryMainFrame()->ExecuteJavaScriptInIsolatedWorld(
kGetTextToSpeak,
base::BindOnce(&PageDistiller::OnGetTextToSpeak,
weak_factory_.GetWeakPtr(), std::move(callback)),
ISOLATED_WORLD_ID_BRAVE_INTERNAL);
}

void PageDistiller::UpdateState(State state) {
state_ = state;
for (auto& observer : observers_) {
Expand All @@ -62,22 +76,17 @@ void PageDistiller::StartDistill(DistillContentCallback callback) {
return std::move(callback).Run(false, {});
}

if (state_ == State::kDistilled) {
constexpr const char16_t kScript[] = uR"js( extractText() )js";
web_contents_->GetPrimaryMainFrame()->ExecuteJavaScriptInIsolatedWorld(
kScript,
base::BindOnce(&PageDistiller::OnGetText, weak_factory_.GetWeakPtr(),
std::move(callback)),
ISOLATED_WORLD_ID_BRAVE_INTERNAL);
} else {
constexpr const char16_t kScript[] =
uR"js( document.documentElement.outerHTML )js";
web_contents_->GetPrimaryMainFrame()->ExecuteJavaScriptInIsolatedWorld(
kScript,
base::BindOnce(&PageDistiller::OnGetOuterHTML,
weak_factory_.GetWeakPtr(), std::move(callback)),
ISOLATED_WORLD_ID_BRAVE_INTERNAL);
}
constexpr const char16_t kGetDocumentSource[] =
uR"js( document.documentElement.outerHTML )js";

constexpr const char16_t kGetBodySource[] =
uR"js( document.body.outerHTML )js";

web_contents_->GetPrimaryMainFrame()->ExecuteJavaScriptInIsolatedWorld(
(state_ != State::kDistilled) ? kGetDocumentSource : kGetBodySource,
base::BindOnce(&PageDistiller::OnGetOuterHTML, weak_factory_.GetWeakPtr(),
std::move(callback)),
ISOLATED_WORLD_ID_BRAVE_INTERNAL);
}

void PageDistiller::OnGetOuterHTML(DistillContentCallback callback,
Expand All @@ -104,14 +113,12 @@ void PageDistiller::OnGetOuterHTML(DistillContentCallback callback,
}
}

void PageDistiller::OnGetText(DistillContentCallback callback,
base::Value result) {
if (!web_contents_ || !result.is_dict() ||
!result.GetDict().FindString("content")) {
return std::move(callback).Run(false, {});
void PageDistiller::OnGetTextToSpeak(TextToSpeechContentCallback callback,
base::Value result) {
if (!result.is_dict()) {
return std::move(callback).Run(base::Value());
}
std::move(callback).Run(true,
std::move(*result.GetDict().FindString("content")));
std::move(callback).Run(std::move(result));
}

void PageDistiller::OnPageDistilled(DistillContentCallback callback,
Expand Down Expand Up @@ -147,6 +154,7 @@ void PageDistiller::ExtractText(DistillContentCallback callback,
return std::move(callback).Run(false, {});
}

re2::RE2::GlobalReplace(&html_content, "<[^>]*>", " ");
std::move(callback).Run(true, html_content);
}

Expand Down
5 changes: 4 additions & 1 deletion browser/speedreader/page_distiller.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class PageDistiller {

using DistillContentCallback =
base::OnceCallback<void(bool success, std::string content)>;
using TextToSpeechContentCallback = base::OnceCallback<void(base::Value)>;

State GetState() const;

Expand All @@ -49,6 +50,7 @@ class PageDistiller {

void GetDistilledHTML(DistillContentCallback callback);
void GetDistilledText(DistillContentCallback callback);
void GetTextToSpeak(TextToSpeechContentCallback callback);

protected:
explicit PageDistiller(content::WebContents* web_contents);
Expand All @@ -60,7 +62,8 @@ class PageDistiller {
private:
void StartDistill(DistillContentCallback callback);
void OnGetOuterHTML(DistillContentCallback callback, base::Value result);
void OnGetText(DistillContentCallback callback, base::Value result);
void OnGetTextToSpeak(TextToSpeechContentCallback callback,
base::Value result);
void OnPageDistilled(DistillContentCallback callback,
DistillationResult result,
std::string original_data,
Expand Down
48 changes: 35 additions & 13 deletions browser/speedreader/speedreader_tab_helper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include "brave/components/speedreader/speedreader_rewriter_service.h"
#include "brave/components/speedreader/speedreader_service.h"
#include "brave/components/speedreader/speedreader_util.h"
#include "brave/components/speedreader/tts_player.h"
#include "chrome/common/chrome_isolated_world_ids.h"
#include "components/dom_distiller/content/browser/distillable_page_utils.h"
#include "components/grit/brave_components_resources.h"
Expand All @@ -51,6 +52,11 @@ namespace speedreader {
std::u16string GetSpeedreaderData(
std::initializer_list<std::pair<base::StringPiece, int>> resources) {
std::u16string result = u"speedreaderData = {";

if (kSpeedreaderTTS.Get()) {
result += u"ttsEnabled: true,";
}

for (const auto& r : resources) {
auto text = brave_l10n::GetLocalizedResourceUTF16String(r.second);
// Make sure that the text doesn't contain js injection
Expand Down Expand Up @@ -204,6 +210,17 @@ void SpeedreaderTabHelper::OnShowOriginalPage() {
TransitStateTo(DistillStates::ViewOriginal());
}

void SpeedreaderTabHelper::OnTtsPlayPause(int paragraph_index) {
auto& tts_controller =
speedreader::TtsPlayer::GetInstance()->GetControllerFor(web_contents());
if (tts_controller.IsPlaying() &&
tts_controller.IsPlayingRequestedWebContents(paragraph_index)) {
tts_controller.Pause();
} else {
tts_controller.Play(paragraph_index);
}
}

void SpeedreaderTabHelper::ClearPersistedData() {
if (auto* entry = web_contents()->GetController().GetLastCommittedEntry()) {
SpeedreaderExtendedInfoHandler::ClearPersistedData(entry);
Expand Down Expand Up @@ -234,8 +251,10 @@ void SpeedreaderTabHelper::ProcessNavigation(

auto* rewriter_service =
g_brave_browser_process->speedreader_rewriter_service();
auto* nav_entry = navigation_handle->GetNavigationEntry();

const bool url_looks_readable =
rewriter_service &&
nav_entry && !nav_entry->IsViewSourceMode() && rewriter_service &&
rewriter_service->URLLooksReadable(navigation_handle->GetURL());

const bool enabled_for_site =
Expand Down Expand Up @@ -302,19 +321,22 @@ void SpeedreaderTabHelper::DidStopLoading() {

void SpeedreaderTabHelper::DOMContentLoaded(
content::RenderFrameHost* render_frame_host) {
if (!render_frame_host->IsInPrimaryMainFrame()) {
if (!render_frame_host->IsInPrimaryMainFrame() ||
!DistillStates::IsDistilled(distill_state_)) {
return;
}
UpdateUI();

if (!IsPageDistillationAllowed()) {
return;
} else {
UpdateUI();
}

static base::NoDestructor<std::u16string> kSpeedreaderData(GetSpeedreaderData(
{{"showOriginalLinkText", IDS_READER_MODE_SHOW_ORIGINAL_PAGE_LINK},
{"minutesText", IDS_READER_MODE_MINUTES_TEXT}}));
static base::NoDestructor<std::u16string> kSpeedreaderData(
GetSpeedreaderData({
{"showOriginalLinkText", IDS_READER_MODE_SHOW_ORIGINAL_PAGE_LINK},
{"minutesText", IDS_READER_MODE_MINUTES_TEXT},
#if defined(IDS_READER_MODE_TEXT_TO_SPEECH_PLAY_PAUSE)
{
"playButtonTitle", IDS_READER_MODE_TEXT_TO_SPEECH_PLAY_PAUSE
}
#endif
}));

static base::NoDestructor<std::u16string> kJsScript(base::UTF8ToUTF16(
ui::ResourceBundle::GetSharedInstance().LoadDataResourceString(
Expand Down Expand Up @@ -448,15 +470,15 @@ void SpeedreaderTabHelper::SetDocumentAttribute(const std::string& attribute,
}

void SpeedreaderTabHelper::OnGetDocumentSource(bool success, std::string html) {
if (!success) {
if (!success || html.empty()) {
// TODO(boocmp): Show error dialog [Distillation failed on this page].
TransitStateTo(DistillStates::DistillReverting(
DistillStates::DistillReverting::Reason::kError, false));
TransitStateTo(DistillStates::ViewOriginal());
return;
}

single_show_content_.swap(html);
single_show_content_ = std::move(html);
TransitStateTo(
DistillStates::Distilling(DistillStates::Distilling::Reason::kManual));
}
Expand Down
1 change: 1 addition & 0 deletions browser/speedreader/speedreader_tab_helper.h
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ class SpeedreaderTabHelper

// mojom::SpeedreaderHost:
void OnShowOriginalPage() override;
void OnTtsPlayPause(int index) override;

private:
friend class content::WebContentsUserData<SpeedreaderTabHelper>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <memory>
#include <utility>

#include "base/strings/string_number_conversions.h"
#include "brave/browser/speedreader/speedreader_service_factory.h"
#include "brave/browser/speedreader/speedreader_tab_helper.h"
#include "brave/browser/ui/brave_browser_window.h"
Expand All @@ -21,6 +22,7 @@
#include "chrome/browser/ui/browser_window.h"
#include "chrome/browser/ui/color/chrome_color_id.h"
#include "chrome/browser/ui/tabs/tab_strip_model.h"
#include "chrome/common/chrome_isolated_world_ids.h"
#include "ui/color/color_provider.h"

#if BUILDFLAG(ENABLE_AI_CHAT)
Expand All @@ -35,14 +37,13 @@ class TtsPlayerDelegate : public speedreader::TtsPlayer::Delegate {

void RequestReadingContent(
content::WebContents* web_contents,
base::OnceCallback<void(bool success, std::string content)> result_cb)
override {
base::OnceCallback<void(base::Value content)> result_cb) override {
auto* page_distiller =
speedreader::SpeedreaderTabHelper::GetPageDistiller(web_contents);
if (page_distiller) {
page_distiller->GetDistilledText(std::move(result_cb));
page_distiller->GetTextToSpeak(std::move(result_cb));
} else {
std::move(result_cb).Run(false, {});
std::move(result_cb).Run(base::Value());
}
}
};
Expand Down Expand Up @@ -72,6 +73,11 @@ SpeedreaderToolbarDataHandlerImpl::SpeedreaderToolbarDataHandlerImpl(

speedreader::TtsPlayer::GetInstance()->set_delegate(
std::make_unique<TtsPlayerDelegate>());

const auto& tts_settings = GetSpeedreaderService()->GetTtsSettings();
speedreader::TtsPlayer::GetInstance()->SetSpeed(
static_cast<double>(tts_settings.speed) / 100.0);
speedreader::TtsPlayer::GetInstance()->SetVoice(tts_settings.voice);
}

SpeedreaderToolbarDataHandlerImpl::~SpeedreaderToolbarDataHandlerImpl() =
Expand Down Expand Up @@ -243,9 +249,24 @@ void SpeedreaderToolbarDataHandlerImpl::OnReadingStop(

void SpeedreaderToolbarDataHandlerImpl::OnReadingProgress(
content::WebContents* web_contents,
const std::string& element_id,
int paragraph_index,
int char_index,
int length) {}
int length) {
if (!web_contents) {
return;
}

constexpr const char16_t kHighlight[] = uR"js( highlightText($1, $2, $3) )js";

const auto script = base::ReplaceStringPlaceholders(
kHighlight,
{base::NumberToString16(paragraph_index),
base::NumberToString16(char_index), base::NumberToString16(length)},
nullptr);

web_contents->GetPrimaryMainFrame()->ExecuteJavaScriptInIsolatedWorld(
script, base::DoNothing(), ISOLATED_WORLD_ID_BRAVE_INTERNAL);
iefremov marked this conversation as resolved.
Show resolved Hide resolved
}

void SpeedreaderToolbarDataHandlerImpl::OnTabStripModelChanged(
TabStripModel* tab_strip_model,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
#ifndef BRAVE_BROWSER_UI_WEBUI_SPEEDREADER_SPEEDREADER_TOOLBAR_DATA_HANDLER_IMPL_H_
#define BRAVE_BROWSER_UI_WEBUI_SPEEDREADER_SPEEDREADER_TOOLBAR_DATA_HANDLER_IMPL_H_

#include <string>

#include "base/scoped_observation.h"
#include "brave/browser/speedreader/speedreader_tab_helper.h"
#include "brave/components/speedreader/common/speedreader_toolbar.mojom.h"
Expand Down Expand Up @@ -87,7 +85,7 @@ class SpeedreaderToolbarDataHandlerImpl
void OnReadingStart(content::WebContents* web_contents) override;
void OnReadingStop(content::WebContents* web_contents) override;
void OnReadingProgress(content::WebContents* web_contents,
const std::string& element_id,
int paragraph_index,
int char_index,
int length) override;

Expand Down
4 changes: 4 additions & 0 deletions browser/ui/webui/speedreader/speedreader_toolbar_ui.cc
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "brave/components/constants/webui_url_constants.h"
#include "brave/components/l10n/common/localization_util.h"
#include "brave/components/speedreader/common/constants.h"
#include "brave/components/speedreader/common/features.h"
#include "brave/components/speedreader/resources/panel/grit/brave_speedreader_toolbar_generated_map.h"
#include "build/build_config.h"
#include "chrome/browser/profiles/profile.h"
Expand Down Expand Up @@ -52,6 +53,9 @@ SpeedreaderToolbarUI::SpeedreaderToolbarUI(content::WebUI* web_ui,
#else
source->AddBoolean("aiChatFeatureEnabled", false);
#endif
source->AddBoolean("ttsEnabled",
speedreader::features::IsSpeedreaderEnabled() &&
speedreader::kSpeedreaderTTS.Get());
}

SpeedreaderToolbarUI::~SpeedreaderToolbarUI() = default;
Expand Down
3 changes: 3 additions & 0 deletions components/speedreader/common/features.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,7 @@ BASE_FEATURE(kSpeedreaderFeature,
const base::FeatureParam<int> kSpeedreaderMinOutLengthParam{
&kSpeedreaderFeature, "min_out_length", 1000};

const base::FeatureParam<bool> kSpeedreaderTTS{&kSpeedreaderFeature, "tts",
false};

} // namespace speedreader
1 change: 1 addition & 0 deletions components/speedreader/common/features.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
namespace speedreader {
BASE_DECLARE_FEATURE(kSpeedreaderFeature);
extern const base::FeatureParam<int> kSpeedreaderMinOutLengthParam;
extern const base ::FeatureParam<bool> kSpeedreaderTTS;
} // namespace speedreader

#endif // BRAVE_COMPONENTS_SPEEDREADER_COMMON_FEATURES_H_
3 changes: 3 additions & 0 deletions components/speedreader/common/speedreader.mojom
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,7 @@ module speedreader.mojom;
interface SpeedreaderHost {
// The browser handler for clicking on the "Show original page" link.
OnShowOriginalPage();

// The browser handler for clicking on the "Play/Pause" button.
OnTtsPlayPause(int32 paragraph_index);
};
Loading