From e8c7aff94311dcb29a8219fcf9b7d46d073a8232 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Sun, 20 Oct 2024 09:47:57 +0800 Subject: [PATCH 01/34] Implement empty chat side panel toggle This commit introduces a toolbar button that allows users to toggle the visibility of the chat side panel. The button provides a visual cue by highlighting with a rounded background when the chat panel is active. --- chrome/app/chrome_command_ids.h | 1 + chrome/app/generated_resources.grd | 12 ++++ chrome/browser/ui/actions/chrome_action_id.h | 1 + chrome/browser/ui/browser_actions.cc | 4 ++ .../browser/ui/browser_element_identifiers.cc | 1 + chrome/browser/ui/views/side_panel/BUILD.gn | 4 ++ .../ai_chat/ai_chat_side_panel_coordinator.cc | 59 +++++++++++++++++++ .../ai_chat/ai_chat_side_panel_coordinator.h | 31 ++++++++++ .../side_panel/ai_chat_side_panel_web_view.cc | 37 ++++++++++++ .../side_panel/ai_chat_side_panel_web_view.h | 27 +++++++++ .../side_panel/side_panel_coordinator.cc | 15 +++-- .../ui/views/side_panel/side_panel_entry_id.h | 1 + .../ui/views/side_panel/side_panel_util.cc | 17 +++++- .../ui/views/side_panel/side_panel_util.h | 4 +- chrome/browser/ui/views/toolbar/BUILD.gn | 2 + .../views/toolbar/ai_chat_toolbar_button.cc | 34 +++++++++++ .../ui/views/toolbar/ai_chat_toolbar_button.h | 26 ++++++++ .../ui/views/toolbar/toolbar_ink_drop_util.cc | 6 +- .../browser/ui/views/toolbar/toolbar_view.cc | 39 +++++++++++- .../browser/ui/views/toolbar/toolbar_view.h | 11 ++++ 20 files changed, 320 insertions(+), 12 deletions(-) create mode 100644 chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc create mode 100644 chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h create mode 100644 chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc create mode 100644 chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h create mode 100644 chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc create mode 100644 chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.h diff --git a/chrome/app/chrome_command_ids.h b/chrome/app/chrome_command_ids.h index 9097e34ba86031..2ea5c36ac48935 100644 --- a/chrome/app/chrome_command_ids.h +++ b/chrome/app/chrome_command_ids.h @@ -269,6 +269,7 @@ #define IDC_SHOW_GOOGLE_LENS_SHORTCUT 40282 #define IDC_SHOW_CUSTOMIZE_CHROME_SIDE_PANEL 40283 #define IDC_SHOW_CUSTOMIZE_CHROME_TOOLBAR 40284 +#define IDC_SHOW_AI_CHAT 40999 // Spell-check // Insert any additional suggestions before _LAST; these have to be consecutive. diff --git a/chrome/app/generated_resources.grd b/chrome/app/generated_resources.grd index ef588a13b86630..d479b3f86f2d02 100644 --- a/chrome/app/generated_resources.grd +++ b/chrome/app/generated_resources.grd @@ -8965,6 +8965,18 @@ Keep your key file in a safe place. You will need it to create new versions of y + + + + Chat + + + + + Chat + + + Side Panel Resize Handle diff --git a/chrome/browser/ui/actions/chrome_action_id.h b/chrome/browser/ui/actions/chrome_action_id.h index 3c8c3fd5b4d2c4..5477ec4f5067c1 100644 --- a/chrome/browser/ui/actions/chrome_action_id.h +++ b/chrome/browser/ui/actions/chrome_action_id.h @@ -550,6 +550,7 @@ E(kActionSidePanelShowLensOverlayResults, IDC_CONTENT_CONTEXT_LENS_OVERLAY) \ E(kActionSidePanelShowReadAnything) \ E(kActionSidePanelShowReadingList, IDC_READING_LIST_MENU_SHOW_UI) \ + E(kActionAIChat, IDC_SHOW_AI_CHAT) \ E(kActionSidePanelShowSearchCompanion, IDC_SHOW_SEARCH_COMPANION) \ E(kActionSidePanelShowShoppingInsights) \ E(kActionSidePanelShowSideSearch) \ diff --git a/chrome/browser/ui/browser_actions.cc b/chrome/browser/ui/browser_actions.cc index a354a5b785589b..677abd6318ac29 100644 --- a/chrome/browser/ui/browser_actions.cc +++ b/chrome/browser/ui/browser_actions.cc @@ -143,6 +143,10 @@ void BrowserActions::InitializeBrowserActions() { IDS_READ_LATER_TITLE, IDS_READ_LATER_TITLE, kReadingListIcon, kActionSidePanelShowReadingList, browser, true), + SidePanelAction(SidePanelEntryId::kAIChat, + IDS_AI_CHAT_TITLE, IDS_AI_CHAT_TITLE, + kSpeakerIcon, kActionAIChat, + browser, false), SidePanelAction(SidePanelEntryId::kAboutThisSite, IDS_PAGE_INFO_ABOUT_THIS_PAGE_TITLE, IDS_PAGE_INFO_ABOUT_THIS_PAGE_TITLE, diff --git a/chrome/browser/ui/browser_element_identifiers.cc b/chrome/browser/ui/browser_element_identifiers.cc index ddc76506180884..4a0c59be55371f 100644 --- a/chrome/browser/ui/browser_element_identifiers.cc +++ b/chrome/browser/ui/browser_element_identifiers.cc @@ -9,6 +9,7 @@ // Please keep this list alphabetized. DEFINE_ELEMENT_IDENTIFIER_VALUE(kAddCurrentTabToReadingListElementId); +DECLARE_ELEMENT_IDENTIFIER_VALUE(kAIChatSidePanelElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE( kAnonymizedUrlCollectionPersonalizationSettingId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kAppUninstallDialogOkButtonId); diff --git a/chrome/browser/ui/views/side_panel/BUILD.gn b/chrome/browser/ui/views/side_panel/BUILD.gn index f737fdd6743633..6b6b4e78723821 100644 --- a/chrome/browser/ui/views/side_panel/BUILD.gn +++ b/chrome/browser/ui/views/side_panel/BUILD.gn @@ -13,6 +13,10 @@ source_set("side_panel_enums") { source_set("side_panel") { sources = [ + "ai_chat_side_panel_web_view.cc", + "ai_chat_side_panel_web_view.h", + "ai_chat/ai_chat_side_panel_coordinator.cc", + "ai_chat/ai_chat_side_panel_coordinator.h", "bookmarks/bookmarks_side_panel_coordinator.cc", "bookmarks/bookmarks_side_panel_coordinator.h", "companion/companion_side_panel_controller_utils.h", diff --git a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc new file mode 100644 index 00000000000000..4281e534ea8d41 --- /dev/null +++ b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc @@ -0,0 +1,59 @@ +#include "ai_chat_side_panel_coordinator.h" + +#include + +#include "chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h" +#include "base/functional/callback.h" +#include "chrome/app/vector_icons/vector_icons.h" +#include "chrome/browser/ui/browser.h" +#include "chrome/browser/ui/views/frame/browser_view.h" +#include "chrome/browser/ui/views/side_panel/side_panel_entry.h" +#include "chrome/browser/ui/views/side_panel/side_panel_registry.h" +#include "chrome/grit/generated_resources.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/base/models/image_model.h" +#include "ui/base/ui_base_features.h" +#include "ui/views/vector_icons.h" +#include "chrome/browser/ui/views/toolbar/toolbar_view.h" + +AIChatSidePanelCoordinator::AIChatSidePanelCoordinator(Browser* browser) + : BrowserUserData(*browser) {} + +AIChatSidePanelCoordinator::~AIChatSidePanelCoordinator() = default; + +void AIChatSidePanelCoordinator::CreateAndRegisterEntry(SidePanelRegistry* global_registry) { + global_registry->Register(std::make_unique( + SidePanelEntry::Id::kAIChat, + base::BindRepeating( + &AIChatSidePanelCoordinator::CreateAIChatWebView, + base::Unretained(this)))); +} + +std::unique_ptr +AIChatSidePanelCoordinator::CreateAIChatWebView() { + return std::make_unique(&GetBrowser()); +} + +void AIChatSidePanelCoordinator::UpdateOpeningPanelId(SidePanelEntryId panel_id) { + auto* browser = &GetBrowser() ; + auto* browser_view = BrowserView::GetBrowserViewForBrowser(browser); + if (browser_view) { + auto* toolbar = browser_view->toolbar(); + if (panel_id != SidePanelEntry::Id::kAIChat) { + toolbar->ResetHighlightForAIChatButton(); + } else { + toolbar->AddHighlightForAIChatButton(); + } + } +} + +void AIChatSidePanelCoordinator::UpdateClosingPanelId(SidePanelEntryId panel_id) { + auto* browser = &GetBrowser() ; + auto* browser_view = BrowserView::GetBrowserViewForBrowser(browser); + if (browser_view && panel_id == SidePanelEntry::Id::kAIChat) { + auto* toolbar = browser_view->toolbar(); + toolbar->ResetHighlightForAIChatButton(); + } +} + +BROWSER_USER_DATA_KEY_IMPL(AIChatSidePanelCoordinator); diff --git a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h new file mode 100644 index 00000000000000..c4b349a54e7248 --- /dev/null +++ b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h @@ -0,0 +1,31 @@ +#ifndef CHROMIUM_AI_CHAT_SIDE_PANEL_COORDINATOR_H +#define CHROMIUM_AI_CHAT_SIDE_PANEL_COORDINATOR_H + +#include "chrome/browser/ui/browser_user_data.h" +#include "chrome/browser/ui/views/side_panel/side_panel_entry_id.h" + +class Browser; +class SidePanelRegistry; + +namespace views { + class View; +} // namespace views + +class AIChatSidePanelCoordinator : public BrowserUserData { + public: + explicit AIChatSidePanelCoordinator(Browser* browser); + ~AIChatSidePanelCoordinator() override; + + void CreateAndRegisterEntry(SidePanelRegistry* global_registry); + void UpdateOpeningPanelId(SidePanelEntryId panel_id); + void UpdateClosingPanelId(SidePanelEntryId panel_id); + + private: + friend class BrowserUserData; + + std::unique_ptr CreateAIChatWebView(); + + BROWSER_USER_DATA_KEY_DECL(); +}; + +#endif //CHROMIUM_AI_CHAT_SIDE_PANEL_COORDINATOR_H diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc new file mode 100644 index 00000000000000..3ac44130a1c268 --- /dev/null +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc @@ -0,0 +1,37 @@ +#include "ai_chat_side_panel_web_view.h" + +#include +#include "ui/views/view.h" +#include "ui/views/controls/label.h" +#include "ui/views/layout/flex_layout.h" +#include "ui/views/layout/layout_types.h" +#include "chrome/browser/extensions/api/bookmark_manager_private/bookmark_manager_private_api.h" +#include "chrome/browser/profiles/profile.h" +#include "chrome/browser/ui/bookmarks/bookmark_utils.h" +#include "chrome/browser/ui/browser.h" +#include "chrome/browser/ui/browser_element_identifiers.h" +#include "chrome/browser/ui/ui_features.h" +#include "chrome/browser/ui/views/frame/browser_view.h" +#include "chrome/common/webui_url_constants.h" +#include "chrome/grit/generated_resources.h" +#include "ui/base/metadata/metadata_impl_macros.h" +#include "ui/views/view_class_properties.h" + +AIChatSidePanelWebView::AIChatSidePanelWebView(Browser *browser) : browser_(browser) { + auto* layout = SetLayoutManager(std::make_unique()); + + // Configure layout properties to center the label both vertically and horizontally + layout->SetOrientation(views::LayoutOrientation::kVertical) // Stack items vertically + .SetMainAxisAlignment(views::LayoutAlignment::kCenter) // Align in the center of the main axis (vertical) + .SetCrossAxisAlignment(views::LayoutAlignment::kCenter); // Align in the center of the cross axis (horizontal) + + // Create a label to add to the layout + auto* label1 = new views::Label(u"Chat"); + auto* label2 = new views::Label(u"coming soon..."); + + // Add the label to the view + AddChildView(label1); + AddChildView(label2); +} + +AIChatSidePanelWebView::~AIChatSidePanelWebView() = default; diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h new file mode 100644 index 00000000000000..5ce6d2e39bcdb7 --- /dev/null +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h @@ -0,0 +1,27 @@ +#ifndef CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H +#define CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H + +#include "ui/views/view.h" +#include "base/functional/callback_forward.h" +#include "base/memory/raw_ptr.h" +#include "chrome/browser/ui/tabs/tab_strip_model_observer.h" +#include "chrome/browser/ui/views/side_panel/side_panel_web_ui_view.h" +#include "chrome/browser/ui/webui/side_panel/reading_list/reading_list_ui.h" +#include "ui/base/metadata/metadata_header_macros.h" +#include "ui/views/controls/webview/webview.h" + +class Browser; + +class AIChatSidePanelWebView : public views::View { + public: + AIChatSidePanelWebView(Browser* browser); + AIChatSidePanelWebView(const AIChatSidePanelWebView&) = delete; + AIChatSidePanelWebView& operator=(const AIChatSidePanelWebView&) = + delete; + ~AIChatSidePanelWebView() override; + + private: + const raw_ptr browser_; + base::WeakPtrFactory weak_factory_{this}; +}; +#endif //CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H \ No newline at end of file diff --git a/chrome/browser/ui/views/side_panel/side_panel_coordinator.cc b/chrome/browser/ui/views/side_panel/side_panel_coordinator.cc index 56029058ace006..dfa3b8c6094052 100644 --- a/chrome/browser/ui/views/side_panel/side_panel_coordinator.cc +++ b/chrome/browser/ui/views/side_panel/side_panel_coordinator.cc @@ -394,10 +394,12 @@ void SidePanelCoordinator::UpdatePinState() { updated_pin_state = !actions_model->IsActionPinned(*extension_id); actions_model->SetActionVisibility(*extension_id, updated_pin_state); } else { - PinnedToolbarActionsModel* const actions_model = - PinnedToolbarActionsModel::Get(profile); - - updated_pin_state = !actions_model->Contains(action_id.value()); + PinnedToolbarActionsModel* const actions_model = PinnedToolbarActionsModel::Get(profile); + if (current_entry_->key().id() == SidePanelEntryId::kAIChat) { + updated_pin_state = false; + } else { + updated_pin_state = !actions_model->Contains(action_id.value()); + } actions_model->UpdatePinnedState(action_id.value(), updated_pin_state); } @@ -835,6 +837,9 @@ void SidePanelCoordinator::NotifyPinnedContainerOfActiveStateChange( std::optional action_id = SidePanelEntryIdToActionId(key.id()); CHECK(action_id.has_value()); + if (key.id() == SidePanelEntryId::kAIChat) { + is_active = false; + } toolbar_container->UpdateActionState(*action_id, is_active); } } @@ -1103,7 +1108,7 @@ void SidePanelCoordinator::OnViewVisibilityChanged(views::View* observed_view, if (!content_wrapper->children().empty()) { content_wrapper->RemoveChildViewT(content_wrapper->children().front()); } - SidePanelUtil::RecordSidePanelClosed(opened_timestamp_); + SidePanelUtil::RecordSidePanelClosed(browser_view_->browser(), previous_entry->key().id(), opened_timestamp_); view_state_observers_.Notify( &SidePanelViewStateObserver::OnSidePanelDidClose); diff --git a/chrome/browser/ui/views/side_panel/side_panel_entry_id.h b/chrome/browser/ui/views/side_panel/side_panel_entry_id.h index fe3eaaa62ebf4b..1cb52752d07bf7 100644 --- a/chrome/browser/ui/views/side_panel/side_panel_entry_id.h +++ b/chrome/browser/ui/views/side_panel/side_panel_entry_id.h @@ -19,6 +19,7 @@ // since we cannot autogenerate this in actions.xml. #define SIDE_PANEL_ENTRY_IDS(V) \ /* Global Entries */ \ + V(kAIChat, kActionAIChat, "AIChat") \ V(kReadingList, kActionSidePanelShowReadingList, "ReadingList") \ V(kBookmarks, kActionSidePanelShowBookmarks, "Bookmarks") \ V(kHistoryClusters, kActionSidePanelShowHistoryCluster, "HistoryClusters") \ diff --git a/chrome/browser/ui/views/side_panel/side_panel_util.cc b/chrome/browser/ui/views/side_panel/side_panel_util.cc index 037c5c34e68cf7..3514641e531a3f 100644 --- a/chrome/browser/ui/views/side_panel/side_panel_util.cc +++ b/chrome/browser/ui/views/side_panel/side_panel_util.cc @@ -31,6 +31,7 @@ #include "ui/accessibility/accessibility_features.h" #include "ui/actions/actions.h" #include "chrome/browser/ui/views/side_panel/extensions/extension_side_panel_manager.h" +#include "chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h" // static void SidePanelUtil::PopulateGlobalEntries(Browser* browser, @@ -39,6 +40,10 @@ void SidePanelUtil::PopulateGlobalEntries(Browser* browser, ReadingListSidePanelCoordinator::GetOrCreateForBrowser(browser) ->CreateAndRegisterEntry(window_registry); + // Add ai chat + AIChatSidePanelCoordinator::GetOrCreateForBrowser(browser) + ->CreateAndRegisterEntry(window_registry); + // Add bookmarks. BookmarksSidePanelCoordinator::GetOrCreateForBrowser(browser) ->CreateAndRegisterEntry(window_registry); @@ -87,11 +92,17 @@ void SidePanelUtil::RecordSidePanelShowOrChangeEntryTrigger( } } -void SidePanelUtil::RecordSidePanelClosed(base::TimeTicks opened_timestamp) { +void SidePanelUtil::RecordSidePanelClosed(Browser* browser, + SidePanelEntry::Id id, + base::TimeTicks opened_timestamp) { base::RecordAction(base::UserMetricsAction("SidePanel.Hide")); base::UmaHistogramLongTimes("SidePanel.OpenDuration", base::TimeTicks::Now() - opened_timestamp); + + + auto* ai_chat_coordinator = AIChatSidePanelCoordinator::GetOrCreateForBrowser(browser); + ai_chat_coordinator->UpdateClosingPanelId(id); } void SidePanelUtil::RecordSidePanelResizeMetrics(SidePanelEntry::Id id, @@ -147,6 +158,10 @@ void SidePanelUtil::RecordEntryShowTriggeredMetrics( Browser* browser, SidePanelEntry::Id id, std::optional trigger) { + + auto* ai_chat_coordinator = AIChatSidePanelCoordinator::GetOrCreateForBrowser(browser); + ai_chat_coordinator->UpdateOpeningPanelId(id); + if (trigger.has_value()) { base::UmaHistogramEnumeration( base::StrCat({"SidePanel.", SidePanelEntryIdToHistogramName(id), diff --git a/chrome/browser/ui/views/side_panel/side_panel_util.h b/chrome/browser/ui/views/side_panel/side_panel_util.h index 334f193376a4a4..eb56c3b53a79f4 100644 --- a/chrome/browser/ui/views/side_panel/side_panel_util.h +++ b/chrome/browser/ui/views/side_panel/side_panel_util.h @@ -39,7 +39,9 @@ class SidePanelUtil { static void RecordSidePanelOpen(std::optional trigger); static void RecordSidePanelShowOrChangeEntryTrigger( std::optional trigger); - static void RecordSidePanelClosed(base::TimeTicks opened_timestamp); + static void RecordSidePanelClosed(Browser* browser, + SidePanelEntry::Id id, + base::TimeTicks opened_timestamp); static void RecordSidePanelResizeMetrics(SidePanelEntry::Id id, int side_panel_contents_width, int browser_window_width); diff --git a/chrome/browser/ui/views/toolbar/BUILD.gn b/chrome/browser/ui/views/toolbar/BUILD.gn index c322ee82949f27..dea8aeb060f358 100644 --- a/chrome/browser/ui/views/toolbar/BUILD.gn +++ b/chrome/browser/ui/views/toolbar/BUILD.gn @@ -60,6 +60,8 @@ source_set("toolbar") { "toolbar_ink_drop_util.h", "toolbar_view.cc", "toolbar_view.h", + "ai_chat_toolbar_button.cc", + "ai_chat_toolbar_button.h" ] public_deps = [ "//base", diff --git a/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc new file mode 100644 index 00000000000000..ca4ca764c8d47d --- /dev/null +++ b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc @@ -0,0 +1,34 @@ +#include "ai_chat_toolbar_button.h" +#include "ui/base/metadata/metadata_header_macros.h" +#include "chrome/app/vector_icons/vector_icons.h" +#include "components/vector_icons/vector_icons.h" +#include "chrome/browser/ui/browser.h" +#include "chrome/browser/ui/view_ids.h" +#include "chrome/grit/generated_resources.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/base/metadata/metadata_impl_macros.h" +#include "ui/views/view.h" +#include "chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.h" + +AIChatToolbarButton::AIChatToolbarButton(PressedCallback callback) + : ToolbarButton(std::move(callback)) { + // Todo: to add localized text and vector icon + SetTooltipText(u"Chat"); + SetHorizontalAlignment(gfx::ALIGN_CENTER); + SetVectorIcon(kSpeakerIcon); + SetVisible(true); + ConfigureInkDropForToolbar(this); +} + +AIChatToolbarButton::~AIChatToolbarButton() = default; + +void AIChatToolbarButton::AddHighlight() { + anchor_higlight_ = AddAnchorHighlight(); +} + +void AIChatToolbarButton::ResetHighlight() { + anchor_higlight_.reset(); +} + +BEGIN_METADATA(AIChatToolbarButton) +END_METADATA diff --git a/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.h b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.h new file mode 100644 index 00000000000000..7cad8463a7bc06 --- /dev/null +++ b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.h @@ -0,0 +1,26 @@ +#ifndef CHROMIUM_AI_CHAT_TOOLBAR_BUTTON_H +#define CHROMIUM_AI_CHAT_TOOLBAR_BUTTON_H + +#include "base/memory/raw_ptr.h" +#include "chrome/browser/ui/views/toolbar/toolbar_button.h" +#include "ui/base/metadata/metadata_header_macros.h" + +class Browser; + +class AIChatToolbarButton : public ToolbarButton { + METADATA_HEADER(AIChatToolbarButton, ToolbarButton) + + public: + AIChatToolbarButton(PressedCallback callback); + AIChatToolbarButton(const AIChatToolbarButton&) = delete; + AIChatToolbarButton& operator=(const AIChatToolbarButton&) = delete; + ~AIChatToolbarButton() override; + + void AddHighlight(); + void ResetHighlight(); + + private: + std::optional anchor_higlight_; +}; + +#endif //CHROMIUM_AI_CHAT_TOOLBAR_BUTTON_H \ No newline at end of file diff --git a/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc b/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc index e6c3026e63cd85..a6f79cf9a3973a 100644 --- a/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc +++ b/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc @@ -40,10 +40,12 @@ class ToolbarButtonHighlightPathGenerator rect.Inset(GetToolbarInkDropInsets(view)); const int radii = ChromeLayoutProvider::Get()->GetCornerRadiusMetric( - views::Emphasis::kMaximum, rect.size()); + views::Emphasis::kHigh, rect.size() + ); SkPath path; - path.addRoundRect(gfx::RectToSkRect(rect), radii, radii); + // path.addRoundRect(gfx::RectToSkRect(rect), radii, radii); + path.addRoundRect(gfx::RectToSkRect( gfx::Rect( rect)), radii, radii); return path; } }; diff --git a/chrome/browser/ui/views/toolbar/toolbar_view.cc b/chrome/browser/ui/views/toolbar/toolbar_view.cc index 1b494d20b63426..a528d03fdafa0b 100644 --- a/chrome/browser/ui/views/toolbar/toolbar_view.cc +++ b/chrome/browser/ui/views/toolbar/toolbar_view.cc @@ -123,6 +123,14 @@ #include "ui/views/widget/tooltip_manager.h" #include "ui/views/widget/widget.h" #include "ui/views/window/non_client_view.h" +#include "chrome/browser/ui/views/side_panel/side_panel_coordinator.h" +#include "chrome/browser/ui/views/side_panel/side_panel_entry_key.h" +#include "chrome/browser/ui/views/side_panel/side_panel_enums.h" +#include "chrome/browser/ui/views/side_panel/side_panel_entry_id.h" +#include "chrome/browser/ui/views/side_panel/side_panel_ui.h" +#include "ui/views/background.h" +#include "ui/views/view.h" +#include "ui/gfx/color_palette.h" #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) #include "chrome/browser/recovery/recovery_install_global_error_factory.h" @@ -352,6 +360,10 @@ void ToolbarView::Init() { std::unique_ptr reload = std::make_unique(browser_->command_controller()); + std::unique_ptr ai_chat_button = std::make_unique( + base::BindRepeating( + &ToolbarView::AIChatButtonPressed, base::Unretained(this))); + PrefService* const prefs = browser_->profile()->GetPrefs(); std::unique_ptr home = std::make_unique( base::BindRepeating(callback, browser_, IDC_HOME), prefs); @@ -506,11 +518,12 @@ void ToolbarView::Init() { #endif // BUILDFLAG(ENABLE_WEBUI_TAB_STRIP) if (base::FeatureList::IsEnabled(features::kResponsiveToolbar)) { - overflow_button_ = - container_view_->AddChildView(std::make_unique()); - overflow_button_->SetVisible(false); + overflow_button_ = container_view_->AddChildView(std::make_unique()); + overflow_button_->SetVisible(false); } + ai_chat_button_ = container_view_->AddChildView(std::move(ai_chat_button)); + auto app_menu_button = std::make_unique(this); app_menu_button->SetFlipCanvasOnPaintForRTLUI(true); app_menu_button->GetViewAccessibility().SetName( @@ -559,6 +572,14 @@ void ToolbarView::Init() { initialized_ = true; } +void ToolbarView::ResetHighlightForAIChatButton() { + ai_chat_button_->ResetHighlight(); +} + +void ToolbarView::AddHighlightForAIChatButton() { + ai_chat_button_->AddHighlight(); +} + void ToolbarView::AnimationEnded(const gfx::Animation* animation) { if (animation->GetCurrentValue() == 0) SetToolbarVisibility(false); @@ -932,6 +953,18 @@ void ToolbarView::NewTabButtonPressed(const ui::Event& event) { NewTabTypes::NEW_TAB_ENUM_COUNT); } +void ToolbarView::AIChatButtonPressed(const ui::Event& event) { + is_ai_chat_button_active_ = !is_ai_chat_button_active_; + if (is_ai_chat_button_active_) { + ai_chat_button_->AddHighlight(); + } else { + ai_chat_button_->ResetHighlight(); + } + auto key = SidePanelEntryKey(SidePanelEntryId::kAIChat); + auto *side_panel = browser_view_ -> browser()->GetFeatures().side_panel_ui(); + side_panel->Toggle(key, SidePanelOpenTrigger::kToolbarButton); +} + bool ToolbarView::AcceleratorPressed(const ui::Accelerator& accelerator) { const views::View* focused_view = focus_manager()->GetFocusedView(); if (focused_view && (focused_view->GetID() == VIEW_ID_OMNIBOX)) diff --git a/chrome/browser/ui/views/toolbar/toolbar_view.h b/chrome/browser/ui/views/toolbar/toolbar_view.h index b02224437ad20b..48ea7acc62bbb1 100644 --- a/chrome/browser/ui/views/toolbar/toolbar_view.h +++ b/chrome/browser/ui/views/toolbar/toolbar_view.h @@ -24,6 +24,7 @@ #include "chrome/browser/ui/views/location_bar/location_bar_view.h" #include "chrome/browser/ui/views/profiles/avatar_toolbar_button.h" #include "chrome/browser/ui/views/toolbar/overflow_button.h" +#include "chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.h" #include "components/prefs/pref_member.h" #include "ui/base/accelerators/accelerator.h" #include "ui/base/metadata/metadata_header_macros.h" @@ -217,6 +218,9 @@ class ToolbarView : public views::AccessiblePaneView, bool AcceleratorPressed(const ui::Accelerator& acc) override; void ChildPreferredSizeChanged(views::View* child) override; + void ResetHighlightForAIChatButton(); + void AddHighlightForAIChatButton(); + friend class AvatarToolbarButtonBrowserTest; protected: @@ -289,6 +293,10 @@ class ToolbarView : public views::AccessiblePaneView, void NewTabButtonPressed(const ui::Event& event); + + bool is_ai_chat_button_active_ = false; + void AIChatButtonPressed(const ui::Event& event); + gfx::SlideAnimation size_animation_{this}; // Controls. Most of these can be null, e.g. in popup windows. Only @@ -362,6 +370,8 @@ class ToolbarView : public views::AccessiblePaneView, // `toolbar_controller_`. raw_ptr overflow_button_ = nullptr; + raw_ptr ai_chat_button_ = nullptr; + // There are two situations where background_view_left_ and // background_view_right_ need be repainted: window active state change and // theme change. active_state_subscription_ handles the former, and the latter @@ -372,6 +382,7 @@ class ToolbarView : public views::AccessiblePaneView, // Listens to changes to active state to update background_view_right_ and // background_view_left_, as their background depends on active state. base::CallbackListSubscription active_state_subscription_; + }; extern const ui::ClassProperty* const kActionItemUnderlineIndicatorKey; From 6a8c6bfba2fc676f24f929457ff75af366336783 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 21 Oct 2024 06:04:04 +0800 Subject: [PATCH 02/34] Add side panel and web UI integration for chat functionality. Side panel can be toggled from the toolbar and displays AI chat alongside current web content. --- .../chrome_browser_interface_binders.cc | 5 + chrome/browser/resources/BUILD.gn | 1 + chrome/browser/resources/chat/BUILD.gn | 22 +++ chrome/browser/resources/chat/chat.css | 5 + chrome/browser/resources/chat/chat.html | 14 ++ .../browser/resources/chat/chat_api_proxy.ts | 35 ++++ chrome/browser/resources/chat/chat_app.css | 160 ++++++++++++++++++ .../browser/resources/chat/chat_app.html.ts | 15 ++ chrome/browser/resources/chat/chat_app.ts | 44 +++++ chrome/browser/ui/BUILD.gn | 5 + .../browser/ui/browser_element_identifiers.cc | 1 + .../browser/ui/browser_element_identifiers.h | 1 + chrome/browser/ui/views/frame/browser_view.cc | 2 + .../ai_chat/ai_chat_side_panel_coordinator.cc | 3 +- .../side_panel/ai_chat_side_panel_web_view.cc | 42 +++-- .../side_panel/ai_chat_side_panel_web_view.h | 10 +- chrome/browser/ui/webui/chat/BUILD.gn | 13 ++ chrome/browser/ui/webui/chat/chat.mojom | 23 +++ .../ui/webui/chat/chat_page_handler.cc | 25 +++ .../browser/ui/webui/chat/chat_page_handler.h | 30 ++++ chrome/browser/ui/webui/chat/chat_ui.cc | 46 +++++ chrome/browser/ui/webui/chat/chat_ui.h | 61 +++++++ .../browser/ui/webui/chrome_web_ui_configs.cc | 2 + chrome/chrome_paks.gni | 2 + chrome/common/webui_url_constants.h | 2 + third_party/lit/v3_0/BUILD.gn | 1 + tools/gritsettings/resource_ids.spec | 4 + .../histograms/metadata/others/histograms.xml | 1 + .../histograms/metadata/page/histograms.xml | 1 + 29 files changed, 557 insertions(+), 19 deletions(-) create mode 100644 chrome/browser/resources/chat/BUILD.gn create mode 100644 chrome/browser/resources/chat/chat.css create mode 100644 chrome/browser/resources/chat/chat.html create mode 100644 chrome/browser/resources/chat/chat_api_proxy.ts create mode 100644 chrome/browser/resources/chat/chat_app.css create mode 100644 chrome/browser/resources/chat/chat_app.html.ts create mode 100644 chrome/browser/resources/chat/chat_app.ts create mode 100644 chrome/browser/ui/webui/chat/BUILD.gn create mode 100644 chrome/browser/ui/webui/chat/chat.mojom create mode 100644 chrome/browser/ui/webui/chat/chat_page_handler.cc create mode 100644 chrome/browser/ui/webui/chat/chat_page_handler.h create mode 100644 chrome/browser/ui/webui/chat/chat_ui.cc create mode 100644 chrome/browser/ui/webui/chat/chat_ui.h diff --git a/chrome/browser/chrome_browser_interface_binders.cc b/chrome/browser/chrome_browser_interface_binders.cc index fefcf21bf3ffda..964b157a29cd10 100644 --- a/chrome/browser/chrome_browser_interface_binders.cc +++ b/chrome/browser/chrome_browser_interface_binders.cc @@ -52,6 +52,8 @@ #include "chrome/browser/ui/webui/suggest_internals/suggest_internals_ui.h" #include "chrome/browser/ui/webui/usb_internals/usb_internals.mojom.h" #include "chrome/browser/ui/webui/usb_internals/usb_internals_ui.h" +#include "chrome/browser/ui/webui/chat/chat.mojom.h" +#include "chrome/browser/ui/webui/chat/chat_ui.h" #include "chrome/browser/web_applications/web_app_utils.h" #include "chrome/common/buildflags.h" #include "chrome/common/chrome_features.h" @@ -1374,6 +1376,9 @@ void PopulateChromeWebUIFrameBinders( side_panel::mojom::BookmarksPageHandlerFactory, BookmarksSidePanelUI>( map); + RegisterWebUIControllerInterfaceBinder< + chat::mojom::PageHandlerFactory, ChatUI>(map); + RegisterWebUIControllerInterfaceBinder< shopping_service::mojom::ShoppingServiceHandlerFactory, BookmarksSidePanelUI, commerce::ProductSpecificationsUI, diff --git a/chrome/browser/resources/BUILD.gn b/chrome/browser/resources/BUILD.gn index 8f314e71ac57e5..dd6b62e21aae41 100644 --- a/chrome/browser/resources/BUILD.gn +++ b/chrome/browser/resources/BUILD.gn @@ -69,6 +69,7 @@ group("resources") { "web_app_internals:resources", "webui_gallery:resources", "whats_new:resources", + "chat:resources", ] if (is_chrome_branded) { public_deps += [ diff --git a/chrome/browser/resources/chat/BUILD.gn b/chrome/browser/resources/chat/BUILD.gn new file mode 100644 index 00000000000000..b1cf9f71e9434e --- /dev/null +++ b/chrome/browser/resources/chat/BUILD.gn @@ -0,0 +1,22 @@ +import("//ui/webui/resources/tools/build_webui.gni") + +build_webui("build") { + grd_prefix = "chat" + + static_files = [ "chat.html", "chat.css" ] + + non_web_component_files = [ "chat_api_proxy.ts", "chat_app.ts", "chat_app.html.ts" ] + css_files = [ "chat_app.css" ] + + # Enable the proper webui_context_type depending on whether implementing + # a chrome:// or chrome-untrusted:// page. + webui_context_type = "trusted" + ts_deps = [ + "//third_party/lit/v3_0:build_ts", + "//ui/webui/resources/js:build_ts", + "//ui/webui/resources/mojo:build_ts", + ] + + mojo_files_deps = [ "//chrome/browser/ui/webui/chat:mojo_bindings_ts__generator" ] + mojo_files = [ "$root_gen_dir/chrome/browser/ui/webui/chat/chat.mojom-webui.ts",] +} \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat.css b/chrome/browser/resources/chat/chat.css new file mode 100644 index 00000000000000..2d6b08d4b48301 --- /dev/null +++ b/chrome/browser/resources/chat/chat.css @@ -0,0 +1,5 @@ + html, + body { + background-color: hsla(0, 0%, 0%, 0); + margin: 0; + } \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat.html b/chrome/browser/resources/chat/chat.html new file mode 100644 index 00000000000000..9aa0e342f292be --- /dev/null +++ b/chrome/browser/resources/chat/chat.html @@ -0,0 +1,14 @@ + + + + + Chat + + + + + + + + + \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat_api_proxy.ts b/chrome/browser/resources/chat/chat_api_proxy.ts new file mode 100644 index 00000000000000..28ebdac007482c --- /dev/null +++ b/chrome/browser/resources/chat/chat_api_proxy.ts @@ -0,0 +1,35 @@ +import {PageCallbackRouter, PageHandlerFactory, PageHandlerRemote} from './chat.mojom-webui.js'; +import type {PageHandlerInterface} from './chat.mojom-webui.js'; + +export interface ChatApiProxy { + getCallbackRouter(): PageCallbackRouter; + handler: PageHandlerInterface; +} + +let instance: ChatApiProxy|null = null; + +export class ChatApiProxyImpl implements ChatApiProxy { + private readonly callbackRouter: PageCallbackRouter = new PageCallbackRouter(); + handler: PageHandlerRemote = new PageHandlerRemote(); + + constructor() { + this.callbackRouter = new PageCallbackRouter(); + this.handler = new PageHandlerRemote(); + const factory = PageHandlerFactory.getRemote(); + factory.createPageHandler( + this.callbackRouter.$.bindNewPipeAndPassRemote(), + this.handler.$.bindNewPipeAndPassReceiver()); + } + + static getInstance(): ChatApiProxy { + return instance || (instance = new ChatApiProxyImpl()); + } + + static setInstance(proxy: ChatApiProxy) { + instance = proxy; + } + + getCallbackRouter() { + return this.callbackRouter; + } +} \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat_app.css b/chrome/browser/resources/chat/chat_app.css new file mode 100644 index 00000000000000..db15933072aaf3 --- /dev/null +++ b/chrome/browser/resources/chat/chat_app.css @@ -0,0 +1,160 @@ +/* #css_wrapper_metadata_start + * #type=style-lit + * #scheme=relative + * #css_wrapper_metadata_end */ + + +@import url('https://fonts.googleapis.com/css2?family=Poppins:wght@400;500;600&display=swap'); +* { + margin: 0; + padding: 0; + box-sizing: border-box; + font-family: "Poppins", sans-serif; +} + +:root { + --text-color: #343541; + --icon-color: #a9a9bc; + --icon-hover-bg: #f1f1f3; + --placeholder-color: #6c6c6c; + --outgoing-chat-bg: #FFFFFF; + --incoming-chat-bg: #F7F7F8; + --outgoing-chat-border: #FFFFFF; + --incoming-chat-border: #D9D9E3; +} + +#container { + display: flex; + flex-direction: column; + height: calc(100vh - 8px); + background-color: transparent; + margin: 4px; + padding: 8px; +} + +.top { + flex: 0 0 auto; + display: flex; + font-size: 1rem; + border-radius: 12px; + border: 1px solid #e3e3e3; + padding: 12px; + max-width: fit-content; +} + +.middle { + flex: 1; + display: flex; + flex-direction: column; + /* Expands to fill the remaining space */ +} + +.bottom { + flex: 0 0 auto; + display: flex; + /* Takes only the height of its content */ +} + +.typing-container { + position: fixed; + bottom: 0; + width: 100%; + display: flex; + padding: 20px 10px; + justify-content: center; + background: var(--outgoing-chat-bg); + border-top: 1px solid var(--incoming-chat-border); +} + +.typing-container .typing-content { + display: flex; + max-width: 950px; + width: 100%; + align-items: flex-end; +} + +.typing-container .typing-textarea { + width: 100%; + display: flex; + position: relative; +} + +.typing-textarea textarea { + resize: none; + height: 55px; + width: 100%; + padding: 15px 45px 15px 20px; + color: var(--text-color); + font-size: 1rem; + border-radius: 12px; + max-height: 250px; + overflow-y: auto; + border: 1px solid #e3e3e3; +} + +.typing-textarea textarea::placeholder { + color: var(--placeholder-color); +} + +.typing-textarea textarea::placeholder { + color: var(--placeholder-color); +} + +.typing-content span { + width: 55px; + height: 55px; + display: flex; + border-radius: 12px; + font-size: 1rem; + align-items: center; + justify-content: center; + color: var(--icon-color); +} + +.typing-textarea span { + position: absolute; + right: 0; + bottom: 0; + visibility: hidden; +} + +span.material-symbols-rounded { + font-size: 1.25rem !important; +} + +.typing-content span { + width: 55px; + height: 55px; + display: flex; + border-radius: 4px; + font-size: 1.35rem; + align-items: center; + justify-content: center; + color: var(--icon-color); +} + +.typing-textarea span { + position: absolute; + right: 0; + bottom: 0; + visibility: hidden; +} + +.typing-textarea textarea:valid~span { + visibility: visible; +} + +.typing-controls { + display: flex; +} + +.typing-controls span { + margin-left: 7px; + font-size: 1rem; + background: var(--incoming-chat-bg); + outline: 1px solid var(--incoming-chat-border); +} + +.typing-controls span:hover { + background: var(--icon-hover-bg); +} \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat_app.html.ts b/chrome/browser/resources/chat/chat_app.html.ts new file mode 100644 index 00000000000000..f7f7d134dce1ed --- /dev/null +++ b/chrome/browser/resources/chat/chat_app.html.ts @@ -0,0 +1,15 @@ +import {ChatAppElement} from "./chat_app"; +import {html} from '//resources/lit/v3_0/lit.rollup.js'; + +export function getHtml(this: ChatAppElement) { + return html` +
+
+
+
+
+ + send +
+
`; +} \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat_app.ts b/chrome/browser/resources/chat/chat_app.ts new file mode 100644 index 00000000000000..00799d0515fe32 --- /dev/null +++ b/chrome/browser/resources/chat/chat_app.ts @@ -0,0 +1,44 @@ +import './strings.m.js'; + +import {CrLitElement} from '//resources/lit/v3_0/lit.rollup.js'; +import {getCss} from './chat_app.css.js'; +import {getHtml} from './chat_app.html.js'; +import type {ChatApiProxy} from "./chat_api_proxy.js"; +import {ChatApiProxyImpl} from "./chat_api_proxy.js"; + +export class ChatAppElement extends CrLitElement { + private chatApiProxy_: ChatApiProxy = ChatApiProxyImpl.getInstance(); + constructor() { + super(); + setTimeout(() => this.chatApiProxy_.handler.showUI(), 0); + } + static get is() { + return 'chat-app'; + } + + static override get styles() { + return getCss(); + } + + // override connectedCallback() { + // super.connectedCallback(); + // setTimeout(() => this.chatApiProxy_.handler.showUI(), 0); + // } + // + // override disconnectedCallback() { + // super.disconnectedCallback(); + // } + + override render() { + return getHtml.bind(this)(); + } + +} + +declare global { + interface HTMLElementTagNameMap { + 'chat-app': ChatAppElement; + } +} + +customElements.define(ChatAppElement.is, ChatAppElement); \ No newline at end of file diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index c1c10d6d72313a..413b05b0823b4a 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -280,6 +280,10 @@ static_library("ui") { "webui/version/version_ui.h", "webui/webui_load_timer.cc", "webui/webui_load_timer.h", + "webui/chat/chat_ui.cc", + "webui/chat/chat_ui.h", + "webui/chat/chat_page_handler.cc", + "webui/chat/chat_page_handler.h", ] if (enable_session_service) { @@ -1743,6 +1747,7 @@ static_library("ui") { "//chrome/browser/ui/views/side_panel:side_panel_enums", "//chrome/browser/ui/views/toolbar", "//chrome/browser/ui/webui/cr_components/theme_color_picker", + "//chrome/browser/ui/webui/chat:mojo_bindings", "//chrome/browser/ui/webui/searchbox", "//chrome/browser/ui/webui/settings", "//chrome/browser/ui/webui/settings:impl", diff --git a/chrome/browser/ui/browser_element_identifiers.cc b/chrome/browser/ui/browser_element_identifiers.cc index 4a0c59be55371f..1e58a5439a988a 100644 --- a/chrome/browser/ui/browser_element_identifiers.cc +++ b/chrome/browser/ui/browser_element_identifiers.cc @@ -27,6 +27,7 @@ DEFINE_ELEMENT_IDENTIFIER_VALUE(kBookmarkBarElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kBookmarkSidePanelWebViewElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kBookmarkStarViewElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kBrowserViewElementId); +DEFINE_ELEMENT_IDENTIFIER_VALUE(kChatSidePanelWebViewElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kConstrainedDialogWebViewElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kCookieControlsIconElementId); DEFINE_ELEMENT_IDENTIFIER_VALUE(kCustomizeChromeSidePanelWebViewElementId); diff --git a/chrome/browser/ui/browser_element_identifiers.h b/chrome/browser/ui/browser_element_identifiers.h index 536cba8cfb5bd9..c1455c2eb9c2fa 100644 --- a/chrome/browser/ui/browser_element_identifiers.h +++ b/chrome/browser/ui/browser_element_identifiers.h @@ -35,6 +35,7 @@ DECLARE_ELEMENT_IDENTIFIER_VALUE(kBookmarkBarElementId); DECLARE_ELEMENT_IDENTIFIER_VALUE(kBookmarkSidePanelWebViewElementId); DECLARE_ELEMENT_IDENTIFIER_VALUE(kBookmarkStarViewElementId); DECLARE_ELEMENT_IDENTIFIER_VALUE(kBrowserViewElementId); +DECLARE_ELEMENT_IDENTIFIER_VALUE(kChatSidePanelWebViewElementId); DECLARE_ELEMENT_IDENTIFIER_VALUE(kConstrainedDialogWebViewElementId); DECLARE_ELEMENT_IDENTIFIER_VALUE(kCookieControlsIconElementId); DECLARE_ELEMENT_IDENTIFIER_VALUE(kCustomizeChromeSidePanelWebViewElementId); diff --git a/chrome/browser/ui/views/frame/browser_view.cc b/chrome/browser/ui/views/frame/browser_view.cc index 8c08f4c3f10faa..f11df05b576951 100644 --- a/chrome/browser/ui/views/frame/browser_view.cc +++ b/chrome/browser/ui/views/frame/browser_view.cc @@ -1034,6 +1034,8 @@ BrowserView::BrowserView(std::unique_ptr browser) unified_side_panel_ = AddChildView(std::make_unique( this, is_right_aligned ? SidePanel::HorizontalAlignment::kRight : SidePanel::HorizontalAlignment::kLeft)); + unified_side_panel_->SetPanelWidth(500); + unified_side_panel_->SetBackgroundRadii(gfx::RoundedCornersF(0.f, 0.f, 0.f, 0.f)); left_aligned_side_panel_separator_ = AddChildView(std::make_unique()); side_panel_rounded_corner_ = diff --git a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc index 4281e534ea8d41..277e7474642c44 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc +++ b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc @@ -31,7 +31,8 @@ void AIChatSidePanelCoordinator::CreateAndRegisterEntry(SidePanelRegistry* globa std::unique_ptr AIChatSidePanelCoordinator::CreateAIChatWebView() { - return std::make_unique(&GetBrowser()); + return std::make_unique(&GetBrowser(), + base::RepeatingClosure()); } void AIChatSidePanelCoordinator::UpdateOpeningPanelId(SidePanelEntryId panel_id) { diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc index 3ac44130a1c268..32ef52f951425e 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc @@ -4,10 +4,9 @@ #include "ui/views/view.h" #include "ui/views/controls/label.h" #include "ui/views/layout/flex_layout.h" +#include "ui/views/layout/fill_layout.h" #include "ui/views/layout/layout_types.h" -#include "chrome/browser/extensions/api/bookmark_manager_private/bookmark_manager_private_api.h" #include "chrome/browser/profiles/profile.h" -#include "chrome/browser/ui/bookmarks/bookmark_utils.h" #include "chrome/browser/ui/browser.h" #include "chrome/browser/ui/browser_element_identifiers.h" #include "chrome/browser/ui/ui_features.h" @@ -16,22 +15,37 @@ #include "chrome/grit/generated_resources.h" #include "ui/base/metadata/metadata_impl_macros.h" #include "ui/views/view_class_properties.h" +#include "ui/views/controls/webview/webview.h" +#include "url/gurl.h" -AIChatSidePanelWebView::AIChatSidePanelWebView(Browser *browser) : browser_(browser) { - auto* layout = SetLayoutManager(std::make_unique()); +using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; +BEGIN_TEMPLATE_METADATA(SidePanelWebUIViewT_ChatUI, SidePanelWebUIViewT) +END_METADATA - // Configure layout properties to center the label both vertically and horizontally - layout->SetOrientation(views::LayoutOrientation::kVertical) // Stack items vertically - .SetMainAxisAlignment(views::LayoutAlignment::kCenter) // Align in the center of the main axis (vertical) - .SetCrossAxisAlignment(views::LayoutAlignment::kCenter); // Align in the center of the cross axis (horizontal) +AIChatSidePanelWebView::AIChatSidePanelWebView( + Browser *browser, + base::RepeatingClosure close_cb) + : SidePanelWebUIViewT( + base::BindRepeating( + &AIChatSidePanelWebView::UpdateActiveURLToActiveTab, + base::Unretained(this)), + close_cb, + std::make_unique>( + GURL(chrome::kChromeUIChatURL), + browser->profile(), + IDS_AI_CHAT_TITLE, + /*esc_closes_ui=*/false)), + browser_(browser) { - // Create a label to add to the layout - auto* label1 = new views::Label(u"Chat"); - auto* label2 = new views::Label(u"coming soon..."); + SetProperty(views::kElementIdentifierKey, + kChatSidePanelWebViewElementId); +} - // Add the label to the view - AddChildView(label1); - AddChildView(label2); +void AIChatSidePanelWebView::UpdateActiveURLToActiveTab() { + LOG(INFO) << "To get current url of the tab"; } AIChatSidePanelWebView::~AIChatSidePanelWebView() = default; + +BEGIN_METADATA(AIChatSidePanelWebView) +END_METADATA \ No newline at end of file diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h index 5ce6d2e39bcdb7..5b2a422cad2651 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h @@ -6,20 +6,22 @@ #include "base/memory/raw_ptr.h" #include "chrome/browser/ui/tabs/tab_strip_model_observer.h" #include "chrome/browser/ui/views/side_panel/side_panel_web_ui_view.h" -#include "chrome/browser/ui/webui/side_panel/reading_list/reading_list_ui.h" #include "ui/base/metadata/metadata_header_macros.h" #include "ui/views/controls/webview/webview.h" +#include "chrome/browser/ui/webui/chat/chat_ui.h" class Browser; -class AIChatSidePanelWebView : public views::View { +class AIChatSidePanelWebView : public SidePanelWebUIViewT { + using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; + METADATA_HEADER(AIChatSidePanelWebView, SidePanelWebUIViewT_ChatUI) public: - AIChatSidePanelWebView(Browser* browser); + AIChatSidePanelWebView(Browser* browser, base::RepeatingClosure close_cb); AIChatSidePanelWebView(const AIChatSidePanelWebView&) = delete; AIChatSidePanelWebView& operator=(const AIChatSidePanelWebView&) = delete; ~AIChatSidePanelWebView() override; - + void UpdateActiveURLToActiveTab(); private: const raw_ptr browser_; base::WeakPtrFactory weak_factory_{this}; diff --git a/chrome/browser/ui/webui/chat/BUILD.gn b/chrome/browser/ui/webui/chat/BUILD.gn new file mode 100644 index 00000000000000..d892a802b58a83 --- /dev/null +++ b/chrome/browser/ui/webui/chat/BUILD.gn @@ -0,0 +1,13 @@ +import("//mojo/public/tools/bindings/mojom.gni") + +assert(!is_android) + +mojom("mojo_bindings") { + sources = [ "chat.mojom" ] + webui_module_path = "/" + public_deps = [ + "//mojo/public/mojom/base", + "//ui/base/mojom", + "//url/mojom:url_mojom_gurl", + ] +} \ No newline at end of file diff --git a/chrome/browser/ui/webui/chat/chat.mojom b/chrome/browser/ui/webui/chat/chat.mojom new file mode 100644 index 00000000000000..284cc9ed0301e0 --- /dev/null +++ b/chrome/browser/ui/webui/chat/chat.mojom @@ -0,0 +1,23 @@ +module chat.mojom; + +import "url/mojom/url.mojom"; + +// Factory ensures that the Page and PageHandler interfaces are always created +// together without requiring an initialization call from the WebUI to the +// handler. +interface PageHandlerFactory { + CreatePageHandler(pending_remote page, + pending_receiver handler); +}; + +// Called from TS side of chat (Renderer -> Browser) +interface PageHandler { + // Notify the backend that the UI is ready to be shown. + ShowUI(); + + // Notify the backend that the dialog should be closed. + CloseUI(); +}; + +// Called from C++ side of chat (Browser -> Renderer) +interface Page {}; \ No newline at end of file diff --git a/chrome/browser/ui/webui/chat/chat_page_handler.cc b/chrome/browser/ui/webui/chat/chat_page_handler.cc new file mode 100644 index 00000000000000..454ebdc65e2d8e --- /dev/null +++ b/chrome/browser/ui/webui/chat/chat_page_handler.cc @@ -0,0 +1,25 @@ +#include "chat_page_handler.h" + +ChatPageHandler::ChatPageHandler( + mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui) + : receiver_(this, std::move(receiver)), + page_(std::move(page)), + chat_ui_(chat_ui) { +} + +ChatPageHandler::~ChatPageHandler() = default; + +void ChatPageHandler::ShowUI() { + auto embedder = chat_ui_->embedder(); + if (embedder) { + embedder->ShowUI(); + } +} + +void ChatPageHandler::CloseUI() { + auto embedder = chat_ui_->embedder(); + if (embedder) + embedder->CloseUI(); +} \ No newline at end of file diff --git a/chrome/browser/ui/webui/chat/chat_page_handler.h b/chrome/browser/ui/webui/chat/chat_page_handler.h new file mode 100644 index 00000000000000..eb80ff6aaff47a --- /dev/null +++ b/chrome/browser/ui/webui/chat/chat_page_handler.h @@ -0,0 +1,30 @@ +#ifndef CHROMIUM_CHAT_PAGE_HANDLER_H +#define CHROMIUM_CHAT_PAGE_HANDLER_H + +#include "chrome/browser/ui/webui/chat/chat.mojom.h" +#include "mojo/public/cpp/bindings/receiver.h" +#include "mojo/public/cpp/bindings/remote.h" +#include "chrome/browser/ui/webui/chat/chat_ui.h" + +class ChatPageHandler : public chat::mojom::PageHandler { +public: + ChatPageHandler( + mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui + ); + + ChatPageHandler(const ChatPageHandler&) = delete; + ChatPageHandler& operator=(const ChatPageHandler&) = delete; + + ~ChatPageHandler() override; + + void ShowUI() override; + void CloseUI() override; + +private: + mojo::Receiver receiver_; + mojo::Remote page_; + const raw_ptr chat_ui_; +}; +#endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/browser/ui/webui/chat/chat_ui.cc b/chrome/browser/ui/webui/chat/chat_ui.cc new file mode 100644 index 00000000000000..861daab458e1df --- /dev/null +++ b/chrome/browser/ui/webui/chat/chat_ui.cc @@ -0,0 +1,46 @@ +#include "chat_ui.h" +#include "chrome/browser/ui/webui/chat/chat_page_handler.h" +#include "chrome/browser/ui/webui/webui_util.h" +#include "chrome/common/webui_url_constants.h" +#include "content/public/common/url_constants.h" +#include "content/public/browser/browser_context.h" +#include "content/public/browser/web_contents.h" +#include "content/public/browser/web_ui.h" +#include "content/public/browser/web_ui_data_source.h" +#include "content/public/browser/web_ui_controller.h" +#include "content/public/browser/webui_config.h" +#include "chrome/grit/chat_resources.h" +#include "chrome/grit/chat_resources_map.h" +#include "ui/webui/mojo_web_ui_controller.h" +#include "chrome/browser/profiles/profile.h" + +#pragma allow_unsafe_buffers + +ChatUI::ChatUI(content::WebUI* web_ui) + : TopChromeWebUIController(web_ui) { + Profile* const profile = Profile::FromWebUI(web_ui); + content::WebUIDataSource* source = content::WebUIDataSource::CreateAndAdd( + profile, chrome::kChromeUIChatHost); + webui::SetupWebUIDataSource( + source, + base::make_span(kChatResources, kChatResourcesSize), + IDR_CHAT_CHAT_HTML); +} + +ChatUI::~ChatUI() = default; + +WEB_UI_CONTROLLER_TYPE_IMPL(ChatUI) + +void ChatUI::BindInterface( + mojo::PendingReceiver receiver) { + page_factory_receiver_.reset(); + page_factory_receiver_.Bind(std::move(receiver)); +} + +void ChatUI::CreatePageHandler( + mojo::PendingRemote page, + mojo::PendingReceiver receiver) { + DCHECK(page); + page_handler_ = std::make_unique( + std::move(receiver), std::move(page), this); +} \ No newline at end of file diff --git a/chrome/browser/ui/webui/chat/chat_ui.h b/chrome/browser/ui/webui/chat/chat_ui.h new file mode 100644 index 00000000000000..115ed55b8209c9 --- /dev/null +++ b/chrome/browser/ui/webui/chat/chat_ui.h @@ -0,0 +1,61 @@ +#ifndef CHROMIUM_CHAT_UI_H +#define CHROMIUM_CHAT_UI_H + +#include "chrome/browser/ui/webui/chat/chat.mojom.h" +#include "content/public/browser/web_ui_controller.h" +#include "content/public/browser/webui_config.h" +#include "chrome/common/webui_url_constants.h" +#include "content/public/common/url_constants.h" +#include "ui/webui/mojo_web_ui_controller.h" +#include "mojo/public/cpp/bindings/pending_receiver.h" +#include "mojo/public/cpp/bindings/pending_remote.h" +#include "mojo/public/cpp/bindings/receiver.h" +#include "chrome/browser/ui/webui/top_chrome/top_chrome_web_ui_controller.h" +#include "chrome/browser/ui/webui/top_chrome/top_chrome_webui_config.h" + +// Forward declaration so that config definition can come before controller. +class ChatUI; + +class ChatUIConfig : public DefaultTopChromeWebUIConfig { +public: + ChatUIConfig() + : DefaultTopChromeWebUIConfig(content::kChromeUIScheme, + chrome::kChromeUIChatHost) {} +}; + +class ChatPageHandler; + +class ChatUI : public TopChromeWebUIController, + public chat::mojom::PageHandlerFactory { +public: + explicit ChatUI(content::WebUI *web_ui); + + ChatUI(const ChatUI &) = delete; + + ChatUI &operator=(const ChatUI &) = delete; + + ~ChatUI() override; + + // Instantiates the implementor of the mojom::PageHandlerFactory mojo + // interface passing the pending receiver that will be internally bound. + void BindInterface( + mojo::PendingReceiver receiver); + + static constexpr std::string + + GetWebUIName() { return "Chat"; } + +private: + // chat::mojom::PageHandlerFactory: + void CreatePageHandler( + mojo::PendingRemote page, + mojo::PendingReceiver receiver) override; + + std::unique_ptr page_handler_; + + mojo::Receiver page_factory_receiver_{ + this}; + + WEB_UI_CONTROLLER_TYPE_DECL(); +}; +#endif //CHROMIUM_CHAT_UI_H diff --git a/chrome/browser/ui/webui/chrome_web_ui_configs.cc b/chrome/browser/ui/webui/chrome_web_ui_configs.cc index 6d46b2bea8b206..666add07bf465a 100644 --- a/chrome/browser/ui/webui/chrome_web_ui_configs.cc +++ b/chrome/browser/ui/webui/chrome_web_ui_configs.cc @@ -145,6 +145,7 @@ #include "chrome/browser/ui/webui/app_settings/web_app_settings_ui.h" #include "chrome/browser/ui/webui/browser_switch/browser_switch_ui.h" #include "chrome/browser/ui/webui/whats_new/whats_new_ui.h" +#include "chrome/browser/ui/webui/chat/chat_ui.h" #endif // BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) || \ @@ -252,6 +253,7 @@ void RegisterChromeWebUIConfigs() { map.AddWebUIConfig( std::make_unique< privacy_sandbox_internals::PrivacySandboxInternalsUIConfig>()); + map.AddWebUIConfig(std::make_unique()); #if BUILDFLAG(ENABLE_NACL) map.AddWebUIConfig(std::make_unique()); diff --git a/chrome/chrome_paks.gni b/chrome/chrome_paks.gni index 4f99bf3e0ee600..68298f14e123b2 100644 --- a/chrome/chrome_paks.gni +++ b/chrome/chrome_paks.gni @@ -130,12 +130,14 @@ template("chrome_extra_paks") { "$root_gen_dir/third_party/blink/public/resources/blink_resources.pak", "$root_gen_dir/third_party/blink/public/resources/inspector_overlay_resources.pak", "$root_gen_dir/ui/resources/webui_resources.pak", + "$root_gen_dir/chrome/chat_resources.pak", ] deps = [ "//base/tracing/protos:chrome_track_event_resources", "//chrome/app/theme:chrome_unscaled_resources", "//chrome/browser:resources", "//chrome/browser/resources:resources", + "//chrome/browser/resources/chat:resources", "//chrome/common:resources", "//components/autofill/core/browser:autofill_address_rewriter_resources", "//components/resources", diff --git a/chrome/common/webui_url_constants.h b/chrome/common/webui_url_constants.h index 9fa26adc16d4fa..b143e07c35cde2 100644 --- a/chrome/common/webui_url_constants.h +++ b/chrome/common/webui_url_constants.h @@ -713,6 +713,8 @@ inline constexpr char kSiteDetailsSubpage[] = "content/siteDetails"; inline constexpr char kSyncSetupSubPage[] = "syncSetup"; inline constexpr char kTriggeredResetProfileSettingsSubPage[] = "triggeredResetProfileSettings"; +inline constexpr char kChromeUIChatURL[] = "chrome://chat/"; +inline constexpr char kChromeUIChatHost[] = "chat"; #if BUILDFLAG(IS_WIN) // TODO(crbug.com/40647483): Remove when issue is resolved. diff --git a/third_party/lit/v3_0/BUILD.gn b/third_party/lit/v3_0/BUILD.gn index 634d839caab86c..68dd6cf818a5f2 100644 --- a/third_party/lit/v3_0/BUILD.gn +++ b/third_party/lit/v3_0/BUILD.gn @@ -70,6 +70,7 @@ ts_library("build_ts") { "//chrome/test/data/webui/signin:build_ts", "//chrome/test/data/webui/tab_search:build_ts", "//chrome/test/data/webui/welcome:build_ts", + "//chrome/browser/resources/chat:build_ts", "//components/flags_ui/resources:build_ts", "//content/browser/resources/traces_internals:build_ts", "//ui/webui/resources/cr_components/certificate_manager:build_ts", diff --git a/tools/gritsettings/resource_ids.spec b/tools/gritsettings/resource_ids.spec index 38cf9d8fd803f4..ad205ae76a06a5 100644 --- a/tools/gritsettings/resource_ids.spec +++ b/tools/gritsettings/resource_ids.spec @@ -648,6 +648,10 @@ "META": {"sizes": {"includes": [10]}}, "includes": [5200], }, + "<(SHARED_INTERMEDIATE_DIR)/chrome/browser/resources/chat/resources.grd": { + "META": {"sizes": {"includes": [20]}}, + "includes": [5480], + }, # END chrome/ WebUI resources section # START chrome/ miscellaneous section. diff --git a/tools/metrics/histograms/metadata/others/histograms.xml b/tools/metrics/histograms/metadata/others/histograms.xml index 279040555faff1..2b7bc34d8e5b0c 100644 --- a/tools/metrics/histograms/metadata/others/histograms.xml +++ b/tools/metrics/histograms/metadata/others/histograms.xml @@ -206,6 +206,7 @@ chromium-metrics-reviews@google.com. + diff --git a/tools/metrics/histograms/metadata/page/histograms.xml b/tools/metrics/histograms/metadata/page/histograms.xml index a2ec77cd1cfca9..c88038a0b3ad4e 100644 --- a/tools/metrics/histograms/metadata/page/histograms.xml +++ b/tools/metrics/histograms/metadata/page/histograms.xml @@ -241,6 +241,7 @@ chromium-metrics-reviews@google.com. + From c800168da77be217fc2b9a0612261ca4c025ed38 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 21 Oct 2024 19:04:41 +0800 Subject: [PATCH 03/34] Move chat related files into side_panel folder --- chrome/browser/chrome_browser_interface_binders.cc | 4 ++-- chrome/browser/resources/BUILD.gn | 2 +- .../browser/resources/{ => side_panel}/chat/BUILD.gn | 6 +++--- .../browser/resources/{ => side_panel}/chat/chat.css | 0 .../browser/resources/{ => side_panel}/chat/chat.html | 0 .../resources/{ => side_panel}/chat/chat_api_proxy.ts | 0 .../resources/{ => side_panel}/chat/chat_app.css | 0 .../resources/{ => side_panel}/chat/chat_app.html.ts | 0 .../resources/{ => side_panel}/chat/chat_app.ts | 0 chrome/browser/ui/BUILD.gn | 10 +++++----- chrome/browser/ui/views/frame/browser_view.cc | 1 - .../ui/views/side_panel/ai_chat_side_panel_web_view.h | 2 +- chrome/browser/ui/webui/chrome_web_ui_configs.cc | 2 +- chrome/browser/ui/webui/{ => side_panel}/chat/BUILD.gn | 0 .../browser/ui/webui/{ => side_panel}/chat/chat.mojom | 0 .../webui/{ => side_panel}/chat/chat_page_handler.cc | 0 .../ui/webui/{ => side_panel}/chat/chat_page_handler.h | 4 ++-- .../browser/ui/webui/{ => side_panel}/chat/chat_ui.cc | 10 +++++----- .../browser/ui/webui/{ => side_panel}/chat/chat_ui.h | 2 +- chrome/chrome_paks.gni | 3 +-- third_party/lit/v3_0/BUILD.gn | 2 +- tools/gritsettings/resource_ids.spec | 8 ++++---- 22 files changed, 27 insertions(+), 29 deletions(-) rename chrome/browser/resources/{ => side_panel}/chat/BUILD.gn (71%) rename chrome/browser/resources/{ => side_panel}/chat/chat.css (100%) rename chrome/browser/resources/{ => side_panel}/chat/chat.html (100%) rename chrome/browser/resources/{ => side_panel}/chat/chat_api_proxy.ts (100%) rename chrome/browser/resources/{ => side_panel}/chat/chat_app.css (100%) rename chrome/browser/resources/{ => side_panel}/chat/chat_app.html.ts (100%) rename chrome/browser/resources/{ => side_panel}/chat/chat_app.ts (100%) rename chrome/browser/ui/webui/{ => side_panel}/chat/BUILD.gn (100%) rename chrome/browser/ui/webui/{ => side_panel}/chat/chat.mojom (100%) rename chrome/browser/ui/webui/{ => side_panel}/chat/chat_page_handler.cc (100%) rename chrome/browser/ui/webui/{ => side_panel}/chat/chat_page_handler.h (88%) rename chrome/browser/ui/webui/{ => side_panel}/chat/chat_ui.cc (84%) rename chrome/browser/ui/webui/{ => side_panel}/chat/chat_ui.h (96%) diff --git a/chrome/browser/chrome_browser_interface_binders.cc b/chrome/browser/chrome_browser_interface_binders.cc index 964b157a29cd10..34ccbaf582d15e 100644 --- a/chrome/browser/chrome_browser_interface_binders.cc +++ b/chrome/browser/chrome_browser_interface_binders.cc @@ -52,8 +52,8 @@ #include "chrome/browser/ui/webui/suggest_internals/suggest_internals_ui.h" #include "chrome/browser/ui/webui/usb_internals/usb_internals.mojom.h" #include "chrome/browser/ui/webui/usb_internals/usb_internals_ui.h" -#include "chrome/browser/ui/webui/chat/chat.mojom.h" -#include "chrome/browser/ui/webui/chat/chat_ui.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat_ui.h" #include "chrome/browser/web_applications/web_app_utils.h" #include "chrome/common/buildflags.h" #include "chrome/common/chrome_features.h" diff --git a/chrome/browser/resources/BUILD.gn b/chrome/browser/resources/BUILD.gn index dd6b62e21aae41..be73d47afa1bfb 100644 --- a/chrome/browser/resources/BUILD.gn +++ b/chrome/browser/resources/BUILD.gn @@ -61,6 +61,7 @@ group("resources") { "side_panel/history_clusters:resources", "side_panel/read_anything:resources", "side_panel/reading_list:resources", + "side_panel/chat:resources", "side_panel/shared:resources", "signin:resources", "suggest_internals:resources", @@ -69,7 +70,6 @@ group("resources") { "web_app_internals:resources", "webui_gallery:resources", "whats_new:resources", - "chat:resources", ] if (is_chrome_branded) { public_deps += [ diff --git a/chrome/browser/resources/chat/BUILD.gn b/chrome/browser/resources/side_panel/chat/BUILD.gn similarity index 71% rename from chrome/browser/resources/chat/BUILD.gn rename to chrome/browser/resources/side_panel/chat/BUILD.gn index b1cf9f71e9434e..11c6929579e30e 100644 --- a/chrome/browser/resources/chat/BUILD.gn +++ b/chrome/browser/resources/side_panel/chat/BUILD.gn @@ -1,7 +1,7 @@ import("//ui/webui/resources/tools/build_webui.gni") build_webui("build") { - grd_prefix = "chat" + grd_prefix = "side_panel_chat" static_files = [ "chat.html", "chat.css" ] @@ -17,6 +17,6 @@ build_webui("build") { "//ui/webui/resources/mojo:build_ts", ] - mojo_files_deps = [ "//chrome/browser/ui/webui/chat:mojo_bindings_ts__generator" ] - mojo_files = [ "$root_gen_dir/chrome/browser/ui/webui/chat/chat.mojom-webui.ts",] + mojo_files_deps = [ "//chrome/browser/ui/webui/side_panel/chat:mojo_bindings_ts__generator" ] + mojo_files = [ "$root_gen_dir/chrome/browser/ui/webui/side_panel/chat/chat.mojom-webui.ts",] } \ No newline at end of file diff --git a/chrome/browser/resources/chat/chat.css b/chrome/browser/resources/side_panel/chat/chat.css similarity index 100% rename from chrome/browser/resources/chat/chat.css rename to chrome/browser/resources/side_panel/chat/chat.css diff --git a/chrome/browser/resources/chat/chat.html b/chrome/browser/resources/side_panel/chat/chat.html similarity index 100% rename from chrome/browser/resources/chat/chat.html rename to chrome/browser/resources/side_panel/chat/chat.html diff --git a/chrome/browser/resources/chat/chat_api_proxy.ts b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts similarity index 100% rename from chrome/browser/resources/chat/chat_api_proxy.ts rename to chrome/browser/resources/side_panel/chat/chat_api_proxy.ts diff --git a/chrome/browser/resources/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css similarity index 100% rename from chrome/browser/resources/chat/chat_app.css rename to chrome/browser/resources/side_panel/chat/chat_app.css diff --git a/chrome/browser/resources/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts similarity index 100% rename from chrome/browser/resources/chat/chat_app.html.ts rename to chrome/browser/resources/side_panel/chat/chat_app.html.ts diff --git a/chrome/browser/resources/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts similarity index 100% rename from chrome/browser/resources/chat/chat_app.ts rename to chrome/browser/resources/side_panel/chat/chat_app.ts diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index 413b05b0823b4a..63580a48de0fa3 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -280,10 +280,6 @@ static_library("ui") { "webui/version/version_ui.h", "webui/webui_load_timer.cc", "webui/webui_load_timer.h", - "webui/chat/chat_ui.cc", - "webui/chat/chat_ui.h", - "webui/chat/chat_page_handler.cc", - "webui/chat/chat_page_handler.h", ] if (enable_session_service) { @@ -1599,6 +1595,10 @@ static_library("ui") { "webui/side_panel/reading_list/reading_list_page_handler.h", "webui/side_panel/reading_list/reading_list_ui.cc", "webui/side_panel/reading_list/reading_list_ui.h", + "webui/side_panel/chat/chat_ui.cc", + "webui/side_panel/chat/chat_ui.h", + "webui/side_panel/chat/chat_page_handler.cc", + "webui/side_panel/chat/chat_page_handler.h", "webui/suggest_internals/suggest_internals_handler.cc", "webui/suggest_internals/suggest_internals_handler.h", "webui/suggest_internals/suggest_internals_ui.cc", @@ -1747,7 +1747,7 @@ static_library("ui") { "//chrome/browser/ui/views/side_panel:side_panel_enums", "//chrome/browser/ui/views/toolbar", "//chrome/browser/ui/webui/cr_components/theme_color_picker", - "//chrome/browser/ui/webui/chat:mojo_bindings", + "//chrome/browser/ui/webui/side_panel/chat:mojo_bindings", "//chrome/browser/ui/webui/searchbox", "//chrome/browser/ui/webui/settings", "//chrome/browser/ui/webui/settings:impl", diff --git a/chrome/browser/ui/views/frame/browser_view.cc b/chrome/browser/ui/views/frame/browser_view.cc index f11df05b576951..efa788bda4e199 100644 --- a/chrome/browser/ui/views/frame/browser_view.cc +++ b/chrome/browser/ui/views/frame/browser_view.cc @@ -1035,7 +1035,6 @@ BrowserView::BrowserView(std::unique_ptr browser) this, is_right_aligned ? SidePanel::HorizontalAlignment::kRight : SidePanel::HorizontalAlignment::kLeft)); unified_side_panel_->SetPanelWidth(500); - unified_side_panel_->SetBackgroundRadii(gfx::RoundedCornersF(0.f, 0.f, 0.f, 0.f)); left_aligned_side_panel_separator_ = AddChildView(std::make_unique()); side_panel_rounded_corner_ = diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h index 5b2a422cad2651..d2c90357b9b241 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h @@ -8,7 +8,7 @@ #include "chrome/browser/ui/views/side_panel/side_panel_web_ui_view.h" #include "ui/base/metadata/metadata_header_macros.h" #include "ui/views/controls/webview/webview.h" -#include "chrome/browser/ui/webui/chat/chat_ui.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat_ui.h" class Browser; diff --git a/chrome/browser/ui/webui/chrome_web_ui_configs.cc b/chrome/browser/ui/webui/chrome_web_ui_configs.cc index 666add07bf465a..cfe958a58fc8e1 100644 --- a/chrome/browser/ui/webui/chrome_web_ui_configs.cc +++ b/chrome/browser/ui/webui/chrome_web_ui_configs.cc @@ -145,7 +145,7 @@ #include "chrome/browser/ui/webui/app_settings/web_app_settings_ui.h" #include "chrome/browser/ui/webui/browser_switch/browser_switch_ui.h" #include "chrome/browser/ui/webui/whats_new/whats_new_ui.h" -#include "chrome/browser/ui/webui/chat/chat_ui.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat_ui.h" #endif // BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) || \ diff --git a/chrome/browser/ui/webui/chat/BUILD.gn b/chrome/browser/ui/webui/side_panel/chat/BUILD.gn similarity index 100% rename from chrome/browser/ui/webui/chat/BUILD.gn rename to chrome/browser/ui/webui/side_panel/chat/BUILD.gn diff --git a/chrome/browser/ui/webui/chat/chat.mojom b/chrome/browser/ui/webui/side_panel/chat/chat.mojom similarity index 100% rename from chrome/browser/ui/webui/chat/chat.mojom rename to chrome/browser/ui/webui/side_panel/chat/chat.mojom diff --git a/chrome/browser/ui/webui/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc similarity index 100% rename from chrome/browser/ui/webui/chat/chat_page_handler.cc rename to chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc diff --git a/chrome/browser/ui/webui/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h similarity index 88% rename from chrome/browser/ui/webui/chat/chat_page_handler.h rename to chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index eb80ff6aaff47a..c40e48c0ad9b6f 100644 --- a/chrome/browser/ui/webui/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -1,10 +1,10 @@ #ifndef CHROMIUM_CHAT_PAGE_HANDLER_H #define CHROMIUM_CHAT_PAGE_HANDLER_H -#include "chrome/browser/ui/webui/chat/chat.mojom.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/remote.h" -#include "chrome/browser/ui/webui/chat/chat_ui.h" +#include "chat_ui.h" class ChatPageHandler : public chat::mojom::PageHandler { public: diff --git a/chrome/browser/ui/webui/chat/chat_ui.cc b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc similarity index 84% rename from chrome/browser/ui/webui/chat/chat_ui.cc rename to chrome/browser/ui/webui/side_panel/chat/chat_ui.cc index 861daab458e1df..861fd36e48cf83 100644 --- a/chrome/browser/ui/webui/chat/chat_ui.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc @@ -1,5 +1,5 @@ #include "chat_ui.h" -#include "chrome/browser/ui/webui/chat/chat_page_handler.h" +#include "chat_page_handler.h" #include "chrome/browser/ui/webui/webui_util.h" #include "chrome/common/webui_url_constants.h" #include "content/public/common/url_constants.h" @@ -9,8 +9,8 @@ #include "content/public/browser/web_ui_data_source.h" #include "content/public/browser/web_ui_controller.h" #include "content/public/browser/webui_config.h" -#include "chrome/grit/chat_resources.h" -#include "chrome/grit/chat_resources_map.h" +#include "chrome/grit/side_panel_chat_resources.h" +#include "chrome/grit/side_panel_chat_resources_map.h" #include "ui/webui/mojo_web_ui_controller.h" #include "chrome/browser/profiles/profile.h" @@ -23,8 +23,8 @@ ChatUI::ChatUI(content::WebUI* web_ui) profile, chrome::kChromeUIChatHost); webui::SetupWebUIDataSource( source, - base::make_span(kChatResources, kChatResourcesSize), - IDR_CHAT_CHAT_HTML); + base::make_span(kSidePanelChatResources, kSidePanelChatResourcesSize), + IDR_SIDE_PANEL_CHAT_CHAT_HTML); } ChatUI::~ChatUI() = default; diff --git a/chrome/browser/ui/webui/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h similarity index 96% rename from chrome/browser/ui/webui/chat/chat_ui.h rename to chrome/browser/ui/webui/side_panel/chat/chat_ui.h index 115ed55b8209c9..5aaec0835a245c 100644 --- a/chrome/browser/ui/webui/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -1,7 +1,7 @@ #ifndef CHROMIUM_CHAT_UI_H #define CHROMIUM_CHAT_UI_H -#include "chrome/browser/ui/webui/chat/chat.mojom.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" #include "content/public/browser/web_ui_controller.h" #include "content/public/browser/webui_config.h" #include "chrome/common/webui_url_constants.h" diff --git a/chrome/chrome_paks.gni b/chrome/chrome_paks.gni index 68298f14e123b2..03ac57f6ea0aad 100644 --- a/chrome/chrome_paks.gni +++ b/chrome/chrome_paks.gni @@ -130,14 +130,12 @@ template("chrome_extra_paks") { "$root_gen_dir/third_party/blink/public/resources/blink_resources.pak", "$root_gen_dir/third_party/blink/public/resources/inspector_overlay_resources.pak", "$root_gen_dir/ui/resources/webui_resources.pak", - "$root_gen_dir/chrome/chat_resources.pak", ] deps = [ "//base/tracing/protos:chrome_track_event_resources", "//chrome/app/theme:chrome_unscaled_resources", "//chrome/browser:resources", "//chrome/browser/resources:resources", - "//chrome/browser/resources/chat:resources", "//chrome/common:resources", "//components/autofill/core/browser:autofill_address_rewriter_resources", "//components/resources", @@ -195,6 +193,7 @@ template("chrome_extra_paks") { "$root_gen_dir/chrome/side_panel_history_clusters_resources.pak", "$root_gen_dir/chrome/side_panel_read_anything_resources.pak", "$root_gen_dir/chrome/side_panel_reading_list_resources.pak", + "$root_gen_dir/chrome/side_panel_chat_resources.pak", "$root_gen_dir/chrome/side_panel_shared_resources.pak", "$root_gen_dir/chrome/signin_resources.pak", "$root_gen_dir/chrome/suggest_internals_resources.pak", diff --git a/third_party/lit/v3_0/BUILD.gn b/third_party/lit/v3_0/BUILD.gn index 68dd6cf818a5f2..c7a3249b433dc0 100644 --- a/third_party/lit/v3_0/BUILD.gn +++ b/third_party/lit/v3_0/BUILD.gn @@ -49,6 +49,7 @@ ts_library("build_ts") { "//chrome/browser/resources/side_panel/history_clusters:build_ts", "//chrome/browser/resources/side_panel/read_anything:build_ts", "//chrome/browser/resources/side_panel/reading_list:build_ts", + "//chrome/browser/resources/side_panel/chat:build_ts", "//chrome/browser/resources/side_panel/shared:build_ts", "//chrome/browser/resources/signin:build_ts", "//chrome/browser/resources/signin/batch_upload:build_ts", @@ -70,7 +71,6 @@ ts_library("build_ts") { "//chrome/test/data/webui/signin:build_ts", "//chrome/test/data/webui/tab_search:build_ts", "//chrome/test/data/webui/welcome:build_ts", - "//chrome/browser/resources/chat:build_ts", "//components/flags_ui/resources:build_ts", "//content/browser/resources/traces_internals:build_ts", "//ui/webui/resources/cr_components/certificate_manager:build_ts", diff --git a/tools/gritsettings/resource_ids.spec b/tools/gritsettings/resource_ids.spec index ad205ae76a06a5..33c794971e4c9a 100644 --- a/tools/gritsettings/resource_ids.spec +++ b/tools/gritsettings/resource_ids.spec @@ -588,6 +588,10 @@ "META": {"sizes": {"includes": [15],}}, "includes": [4900], }, + "<(SHARED_INTERMEDIATE_DIR)/chrome/browser/resources/side_panel/chat/resources.grd": { + "META": {"sizes": {"includes": [20]}}, + "includes": [4910], + }, "<(SHARED_INTERMEDIATE_DIR)/chrome/browser/resources/side_panel/shared/resources.grd": { "META": {"sizes": {"includes": [15],}}, "includes": [4920], @@ -648,10 +652,6 @@ "META": {"sizes": {"includes": [10]}}, "includes": [5200], }, - "<(SHARED_INTERMEDIATE_DIR)/chrome/browser/resources/chat/resources.grd": { - "META": {"sizes": {"includes": [20]}}, - "includes": [5480], - }, # END chrome/ WebUI resources section # START chrome/ miscellaneous section. From a6431a7c1d7b4b2ea06e595b22cf8a286e770326 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 23 Oct 2024 03:54:44 +0800 Subject: [PATCH 04/34] Observe active tab info and pump it to WebUI for using chat context --- .../side_panel/chat/chat_api_proxy.ts | 20 +++- .../side_panel/chat/chat_app.html.ts | 4 +- .../resources/side_panel/chat/chat_app.ts | 45 +++++-- .../reading_list/reading_list_app.ts | 6 +- .../side_panel/ai_chat_side_panel_web_view.cc | 110 ++++++++++++++---- .../side_panel/ai_chat_side_panel_web_view.h | 44 ++++--- .../ui/webui/side_panel/chat/chat.mojom | 25 +++- .../side_panel/chat/chat_page_handler.cc | 45 +++++-- .../webui/side_panel/chat/chat_page_handler.h | 33 ++++-- .../ui/webui/side_panel/chat/chat_ui.cc | 10 +- .../ui/webui/side_panel/chat/chat_ui.h | 11 +- 11 files changed, 265 insertions(+), 88 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts index 28ebdac007482c..c6558617a554d1 100644 --- a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts +++ b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts @@ -1,9 +1,11 @@ -import {PageCallbackRouter, PageHandlerFactory, PageHandlerRemote} from './chat.mojom-webui.js'; -import type {PageHandlerInterface} from './chat.mojom-webui.js'; +import {PageCallbackRouter, PageHandlerFactory, PageHandlerRemote, SiteInfo} + from "./chat.mojom-webui.js"; export interface ChatApiProxy { + getSiteInfo():Promise<{siteInfo: SiteInfo}> + showUI(): void; + closeUI(): void; getCallbackRouter(): PageCallbackRouter; - handler: PageHandlerInterface; } let instance: ChatApiProxy|null = null; @@ -32,4 +34,16 @@ export class ChatApiProxyImpl implements ChatApiProxy { getCallbackRouter() { return this.callbackRouter; } + + showUI() { + this.handler.showUI(); + } + + closeUI(){ + this.handler.closeUI(); + } + + getSiteInfo() { + return this.handler.getSiteInfo(); + } } \ No newline at end of file diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index f7f7d134dce1ed..72b5c2e759f00f 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -4,7 +4,9 @@ import {html} from '//resources/lit/v3_0/lit.rollup.js'; export function getHtml(this: ChatAppElement) { return html`
-
+
+
${this.siteInfo_.url}
+
${this.siteInfo_.title}
diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 00799d0515fe32..daf9df0ce15ccf 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -6,11 +6,17 @@ import {getHtml} from './chat_app.html.js'; import type {ChatApiProxy} from "./chat_api_proxy.js"; import {ChatApiProxyImpl} from "./chat_api_proxy.js"; +import {SiteInfo} + from "./chat.mojom-webui.js"; export class ChatAppElement extends CrLitElement { private chatApiProxy_: ChatApiProxy = ChatApiProxyImpl.getInstance(); + + + private listenerIds_: number[] = []; + protected siteInfo_ : SiteInfo = { url : "url", title : "title"}; + constructor() { super(); - setTimeout(() => this.chatApiProxy_.handler.showUI(), 0); } static get is() { return 'chat-app'; @@ -20,14 +26,35 @@ export class ChatAppElement extends CrLitElement { return getCss(); } - // override connectedCallback() { - // super.connectedCallback(); - // setTimeout(() => this.chatApiProxy_.handler.showUI(), 0); - // } - // - // override disconnectedCallback() { - // super.disconnectedCallback(); - // } + static override get properties() { + return { + siteInfo_: {type: Object}, + }; + } + + + +private async updateSiteInfo(siteInfo: SiteInfo) { + this.siteInfo_ = siteInfo; + await this.updateComplete; + } + + override connectedCallback() { + super.connectedCallback(); + setTimeout(() => this.chatApiProxy_.showUI(), 0); + + this.listenerIds_.push( + this.chatApiProxy_.getCallbackRouter().onSiteInfoChanged.addListener( + (siteInfo: SiteInfo) => + this.updateSiteInfo(siteInfo))); + + } + + override disconnectedCallback() { + super.disconnectedCallback(); + this.listenerIds_.forEach( + id => this.chatApiProxy_.getCallbackRouter().removeListener(id)); + } override render() { return getHtml.bind(this)(); diff --git a/chrome/browser/resources/side_panel/reading_list/reading_list_app.ts b/chrome/browser/resources/side_panel/reading_list/reading_list_app.ts index 87debbe4418a69..9b26ac5a4898a5 100644 --- a/chrome/browser/resources/side_panel/reading_list/reading_list_app.ts +++ b/chrome/browser/resources/side_panel/reading_list/reading_list_app.ts @@ -59,9 +59,9 @@ export class ReadingListAppElement extends ReadingListAppElementBase { static override get properties() { return { - unreadItems_: {type: Array}, - readItems_: {type: Array}, - currentPageActionButtonState_: {type: Number}, + unreadItems_: {type: Array}, + readItems_: {type: Array}, + currentPageActionButtonState_: {type: Number}, buttonRipples: {type: Boolean}, loadingContent_: {type: Boolean}, }; diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc index 32ef52f951425e..8560018055e775 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc @@ -1,12 +1,12 @@ #include "ai_chat_side_panel_web_view.h" #include -#include "ui/views/view.h" -#include "ui/views/controls/label.h" -#include "ui/views/layout/flex_layout.h" -#include "ui/views/layout/fill_layout.h" -#include "ui/views/layout/layout_types.h" +#include + +#include "base/strings/string_util.h" +#include "base/strings/utf_string_conversions.h" #include "chrome/browser/profiles/profile.h" +#include "chrome/browser/ui/bookmarks/bookmark_utils.h" #include "chrome/browser/ui/browser.h" #include "chrome/browser/ui/browser_element_identifiers.h" #include "chrome/browser/ui/ui_features.h" @@ -14,35 +14,95 @@ #include "chrome/common/webui_url_constants.h" #include "chrome/grit/generated_resources.h" #include "ui/base/metadata/metadata_impl_macros.h" -#include "ui/views/view_class_properties.h" +#include "ui/views/controls/label.h" #include "ui/views/controls/webview/webview.h" +#include "ui/views/layout/fill_layout.h" +#include "ui/views/layout/flex_layout.h" +#include "ui/views/layout/layout_types.h" +#include "ui/views/view.h" +#include "ui/views/view_class_properties.h" #include "url/gurl.h" using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; BEGIN_TEMPLATE_METADATA(SidePanelWebUIViewT_ChatUI, SidePanelWebUIViewT) END_METADATA -AIChatSidePanelWebView::AIChatSidePanelWebView( - Browser *browser, - base::RepeatingClosure close_cb) - : SidePanelWebUIViewT( - base::BindRepeating( - &AIChatSidePanelWebView::UpdateActiveURLToActiveTab, - base::Unretained(this)), - close_cb, - std::make_unique>( - GURL(chrome::kChromeUIChatURL), - browser->profile(), - IDS_AI_CHAT_TITLE, - /*esc_closes_ui=*/false)), - browser_(browser) { - - SetProperty(views::kElementIdentifierKey, - kChatSidePanelWebViewElementId); +AIChatSidePanelWebView::AIChatSidePanelWebView(Browser* browser, + base::RepeatingClosure close_cb) + : SidePanelWebUIViewT( + base::BindRepeating( + &AIChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab, + base::Unretained(this)), + close_cb, + std::make_unique>( + GURL(chrome::kChromeUIChatURL), + browser->profile(), + IDS_AI_CHAT_TITLE, + /*esc_closes_ui=*/false)), + browser_(browser) { + SetProperty(views::kElementIdentifierKey, kChatSidePanelWebViewElementId); + browser_->tab_strip_model()->AddObserver(this); +} + +void AIChatSidePanelWebView::OnTabStripModelChanged( + TabStripModel* tab_strip_model, + const TabStripModelChange& change, + const TabStripSelectionChange& selection) { + if (GetVisible() && selection.active_tab_changed()) { + UpdateActiveSiteInfo(tab_strip_model->GetActiveWebContents()); + } +} + +void AIChatSidePanelWebView::TabChangedAt(content::WebContents* contents, + int index, + TabChangeType change_type) { + if (GetVisible() && index == browser_->tab_strip_model()->active_index() && + change_type == TabChangeType::kAll) { + UpdateActiveSiteInfo(browser_->tab_strip_model()->GetWebContentsAt(index)); + } +} + +void AIChatSidePanelWebView::UpdateActiveSiteInfo( + content::WebContents* contents) { + auto* controller = contents_wrapper()->GetWebUIController(); + if (!controller || !contents) { + return; + } + + // auto title = base::UTF16ToUTF8( web_contents_->GetTitle()); + // std::string url; + // const GURL gurl = web_contents_->GetLastCommittedURL(); + // if (gurl.SchemeIsHTTPOrHTTPS()) { + // url = gurl.spec(); + // } + // chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + // site_info->title = title; + // site_info->url = url; + // + auto title = contents->GetTitle(); + std::string url; + const GURL gurl = contents->GetLastCommittedURL(); + if (gurl.SchemeIsHTTPOrHTTPS()) { + url = gurl.spec(); + } + // bool ConversationDriver::IsContentAssociationPossible() { + // const GURL url = GetPageURL(); + // + // if (!base::Contains(kAllowedSchemes, url.scheme())) { + // return false; + // } + // + // return true; + // } + chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + site_info->title = base::UTF16ToUTF8(title); + site_info->url = url; + + controller->GetAs()->SetSiteInfo(site_info.Clone()); } -void AIChatSidePanelWebView::UpdateActiveURLToActiveTab() { - LOG(INFO) << "To get current url of the tab"; +void AIChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab() { + UpdateActiveSiteInfo(browser_->tab_strip_model()->GetActiveWebContents()); } AIChatSidePanelWebView::~AIChatSidePanelWebView() = default; diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h index d2c90357b9b241..90730f1f933695 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h @@ -1,29 +1,43 @@ #ifndef CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H #define CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H -#include "ui/views/view.h" #include "base/functional/callback_forward.h" #include "base/memory/raw_ptr.h" #include "chrome/browser/ui/tabs/tab_strip_model_observer.h" #include "chrome/browser/ui/views/side_panel/side_panel_web_ui_view.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat_ui.h" #include "ui/base/metadata/metadata_header_macros.h" #include "ui/views/controls/webview/webview.h" -#include "chrome/browser/ui/webui/side_panel/chat/chat_ui.h" +#include "ui/views/view.h" class Browser; -class AIChatSidePanelWebView : public SidePanelWebUIViewT { - using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; - METADATA_HEADER(AIChatSidePanelWebView, SidePanelWebUIViewT_ChatUI) - public: - AIChatSidePanelWebView(Browser* browser, base::RepeatingClosure close_cb); - AIChatSidePanelWebView(const AIChatSidePanelWebView&) = delete; - AIChatSidePanelWebView& operator=(const AIChatSidePanelWebView&) = - delete; - ~AIChatSidePanelWebView() override; - void UpdateActiveURLToActiveTab(); - private: - const raw_ptr browser_; - base::WeakPtrFactory weak_factory_{this}; +class AIChatSidePanelWebView : public SidePanelWebUIViewT, + public TabStripModelObserver { + using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; + METADATA_HEADER(AIChatSidePanelWebView, SidePanelWebUIViewT_ChatUI) + public: + AIChatSidePanelWebView(Browser* browser, base::RepeatingClosure close_cb); + AIChatSidePanelWebView(const AIChatSidePanelWebView&) = delete; + AIChatSidePanelWebView& operator=(const AIChatSidePanelWebView&) = delete; + ~AIChatSidePanelWebView() override; + + // TabStripModelObserver: + void OnTabStripModelChanged( + TabStripModel* tab_strip_model, + const TabStripModelChange& change, + const TabStripSelectionChange& selection) override; + + void TabChangedAt(content::WebContents* contents, + int index, + TabChangeType change_type) override; + + void UpdateActiveSiteInfo(content::WebContents* contents); + void UpdateActiveSiteInfoToActiveTab(); + + private: + const raw_ptr browser_; + base::WeakPtrFactory weak_factory_{this}; }; #endif //CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H \ No newline at end of file diff --git a/chrome/browser/ui/webui/side_panel/chat/chat.mojom b/chrome/browser/ui/webui/side_panel/chat/chat.mojom index 284cc9ed0301e0..ab6e0d9aee8597 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat.mojom +++ b/chrome/browser/ui/webui/side_panel/chat/chat.mojom @@ -2,16 +2,26 @@ module chat.mojom; import "url/mojom/url.mojom"; -// Factory ensures that the Page and PageHandler interfaces are always created -// together without requiring an initialization call from the WebUI to the -// handler. +struct SiteInfo { + // The title of the currently active tab if it has opening page + string? title; + + // The url of the currently active tab if it has opening page + string? url; +}; + +// Used by the WebUI page to bootstrap bidirectional communication. interface PageHandlerFactory { CreatePageHandler(pending_remote page, pending_receiver handler); }; -// Called from TS side of chat (Renderer -> Browser) +// Browser-side handler for requests from WebUI page. +// Renderer -> Browser interface PageHandler { + + GetSiteInfo() => (SiteInfo site_info); + // Notify the backend that the UI is ready to be shown. ShowUI(); @@ -19,5 +29,8 @@ interface PageHandler { CloseUI(); }; -// Called from C++ side of chat (Browser -> Renderer) -interface Page {}; \ No newline at end of file +// WebUI-side handler for requests from the browser. +// Browser -> Renderer +interface Page { + OnSiteInfoChanged(SiteInfo info); +}; \ No newline at end of file diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 454ebdc65e2d8e..ed43fe6b5de8bc 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -1,13 +1,23 @@ #include "chat_page_handler.h" +#include + +#include "base/strings/string_util.h" +#include "base/strings/utf_string_conversions.h" +#include "content/public/browser/web_contents.h" +#include "content/public/browser/web_ui.h" +#include "url/gurl.h" + ChatPageHandler::ChatPageHandler( - mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui) - : receiver_(this, std::move(receiver)), - page_(std::move(page)), - chat_ui_(chat_ui) { -} + mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui) + : receiver_(this, std::move(receiver)), + page_(std::move(page)), + chat_ui_(chat_ui), + web_ui_(web_ui), + web_contents_(web_ui->GetWebContents()) {} ChatPageHandler::~ChatPageHandler() = default; @@ -22,4 +32,23 @@ void ChatPageHandler::CloseUI() { auto embedder = chat_ui_->embedder(); if (embedder) embedder->CloseUI(); -} \ No newline at end of file +} + +void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { + if (page_.is_bound()) { + page_->OnSiteInfoChanged(std::move(site_info)); + } +} + +void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { + auto title = base::UTF16ToUTF8(web_contents_->GetTitle()); + std::string url; + const GURL gurl = web_contents_->GetLastCommittedURL(); + if (gurl.SchemeIsHTTPOrHTTPS()) { + url = gurl.spec(); + } + chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + site_info->title = title; + site_info->url = url; + std::move(callback).Run(site_info.Clone()); +} diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index c40e48c0ad9b6f..9d4ee0f2bc375f 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -1,30 +1,41 @@ #ifndef CHROMIUM_CHAT_PAGE_HANDLER_H #define CHROMIUM_CHAT_PAGE_HANDLER_H +#include "base/memory/raw_ptr.h" +#include "chat_ui.h" #include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" +#include "mojo/public/cpp/bindings/pending_receiver.h" +#include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/remote.h" -#include "chat_ui.h" + +namespace content { +class WebContents; +class WebUI; +} // namespace content class ChatPageHandler : public chat::mojom::PageHandler { public: - ChatPageHandler( - mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui - ); + ChatPageHandler(mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui); - ChatPageHandler(const ChatPageHandler&) = delete; - ChatPageHandler& operator=(const ChatPageHandler&) = delete; + ChatPageHandler(const ChatPageHandler&) = delete; + ChatPageHandler& operator=(const ChatPageHandler&) = delete; - ~ChatPageHandler() override; + ~ChatPageHandler() override; - void ShowUI() override; - void CloseUI() override; + void ShowUI() override; + void CloseUI() override; + void GetSiteInfo(GetSiteInfoCallback callback) override; + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); private: mojo::Receiver receiver_; mojo::Remote page_; const raw_ptr chat_ui_; + const raw_ptr web_ui_; + raw_ptr web_contents_; }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc index 861fd36e48cf83..5ebea1f16c8f64 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc @@ -42,5 +42,11 @@ void ChatUI::CreatePageHandler( mojo::PendingReceiver receiver) { DCHECK(page); page_handler_ = std::make_unique( - std::move(receiver), std::move(page), this); -} \ No newline at end of file + std::move(receiver), std::move(page), this, web_ui()); +} + +void ChatUI::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { + if (page_handler_) { + page_handler_->SetSiteInfo(std::move(site_info)); + } +} diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h index 5aaec0835a245c..2a04af4369ae6f 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -2,16 +2,16 @@ #define CHROMIUM_CHAT_UI_H #include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" +#include "chrome/browser/ui/webui/top_chrome/top_chrome_web_ui_controller.h" +#include "chrome/browser/ui/webui/top_chrome/top_chrome_webui_config.h" +#include "chrome/common/webui_url_constants.h" #include "content/public/browser/web_ui_controller.h" #include "content/public/browser/webui_config.h" -#include "chrome/common/webui_url_constants.h" #include "content/public/common/url_constants.h" -#include "ui/webui/mojo_web_ui_controller.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" -#include "chrome/browser/ui/webui/top_chrome/top_chrome_web_ui_controller.h" -#include "chrome/browser/ui/webui/top_chrome/top_chrome_webui_config.h" +#include "ui/webui/mojo_web_ui_controller.h" // Forward declaration so that config definition can come before controller. class ChatUI; @@ -44,8 +44,9 @@ class ChatUI : public TopChromeWebUIController, static constexpr std::string GetWebUIName() { return "Chat"; } + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); -private: + private: // chat::mojom::PageHandlerFactory: void CreatePageHandler( mojo::PendingRemote page, From c4a22e83bf275de704e58f5ee5c8ed0144ce7458 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 23 Oct 2024 15:29:29 +0800 Subject: [PATCH 05/34] Add chat icon and replace icon of browser_app_menu_button --- chrome/app/vector_icons/BUILD.gn | 2 ++ chrome/app/vector_icons/chat.icon | 13 ++++++++ chrome/app/vector_icons/more_horiz.icon | 32 +++++++++++++++++++ .../views/toolbar/ai_chat_toolbar_button.cc | 2 +- .../views/toolbar/browser_app_menu_button.cc | 2 +- 5 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 chrome/app/vector_icons/chat.icon create mode 100644 chrome/app/vector_icons/more_horiz.icon diff --git a/chrome/app/vector_icons/BUILD.gn b/chrome/app/vector_icons/BUILD.gn index 22370837a74523..837e19475e1ae9 100644 --- a/chrome/app/vector_icons/BUILD.gn +++ b/chrome/app/vector_icons/BUILD.gn @@ -40,6 +40,7 @@ aggregate_vector_icons("chrome_vector_icons") { "browser_tools_update.icon", "browser_tools_update_chrome_refresh.icon", "cast_chrome_refresh.icon", + "chat.icon", "chevron_right.icon", "chevron_right_chrome_refresh.icon", "chromium_minimize.icon", @@ -119,6 +120,7 @@ aggregate_vector_icons("chrome_vector_icons") { "menu_book_chrome_refresh.icon", "mixed_content.icon", "more_tools_menu.icon", + "more_horiz.icon", "move_group_to_new_window_refresh.icon", "my_location.icon", "name_window.icon", diff --git a/chrome/app/vector_icons/chat.icon b/chrome/app/vector_icons/chat.icon new file mode 100644 index 00000000000000..0475a1d4db2918 --- /dev/null +++ b/chrome/app/vector_icons/chat.icon @@ -0,0 +1,13 @@ +CANVAS_DIMENSIONS, 18, +FILL_RULE_NONZERO, +MOVE_TO, 0, 8.33f, +CUBIC_TO, 4.5f, 8.33f, 8.33f, 4.5f, 8.33f, 0, +H_LINE_TO, 9.67f, +CUBIC_TO, 9.67f, 4.5f, 13.5f, 8.33f, 18, 8.33f, +V_LINE_TO, 9.67f, +CUBIC_TO, 13.5f, 9.67f, 9.67f, 13.5f, 9.67f, 18, +H_LINE_TO, 8.33f, +CUBIC_TO, 8.33f, 13.5f, 4.5f, 9.67f, 0, 9.67f, +V_LINE_TO, 8.33f, +CLOSE, +NEW_PATH \ No newline at end of file diff --git a/chrome/app/vector_icons/more_horiz.icon b/chrome/app/vector_icons/more_horiz.icon new file mode 100644 index 00000000000000..67504624591873 --- /dev/null +++ b/chrome/app/vector_icons/more_horiz.icon @@ -0,0 +1,32 @@ +CANVAS_DIMENSIONS, 20, +FILL_RULE_NONZERO, +MOVE_TO, 5.5f, 11.5f, +CUBIC_TO, 5.08f, 11.5f, 4.73f, 11.35f, 4.44f, 11.06f, +CUBIC_TO, 4.15f, 10.76f, 4, 10.41f, 4, 10, +CUBIC_TO, 4, 9.58f, 4.15f, 9.23f, 4.44f, 8.94f, +CUBIC_TO, 4.74f, 8.65f, 5.09f, 8.5f, 5.5f, 8.5f, +CUBIC_TO, 5.92f, 8.5f, 6.27f, 8.65f, 6.56f, 8.94f, +CUBIC_TO, 6.85f, 9.24f, 7, 9.59f, 7, 10, +CUBIC_TO, 7, 10.42f, 6.85f, 10.77f, 6.56f, 11.06f, +CUBIC_TO, 6.26f, 11.35f, 5.91f, 11.5f, 5.5f, 11.5f, +CLOSE, +MOVE_TO, 10, 11.5f, +CUBIC_TO, 9.58f, 11.5f, 9.23f, 11.35f, 8.94f, 11.06f, +CUBIC_TO, 8.65f, 10.76f, 8.5f, 10.41f, 8.5f, 10, +CUBIC_TO, 8.5f, 9.58f, 8.65f, 9.23f, 8.94f, 8.94f, +CUBIC_TO, 9.24f, 8.65f, 9.59f, 8.5f, 10, 8.5f, +CUBIC_TO, 10.42f, 8.5f, 10.77f, 8.65f, 11.06f, 8.94f, +CUBIC_TO, 11.35f, 9.24f, 11.5f, 9.59f, 11.5f, 10, +CUBIC_TO, 11.5f, 10.42f, 11.35f, 10.77f, 11.06f, 11.06f, +CUBIC_TO, 10.76f, 11.35f, 10.41f, 11.5f, 10, 11.5f, +CLOSE, +MOVE_TO, 14.5f, 11.5f, +CUBIC_TO, 14.08f, 11.5f, 13.73f, 11.35f, 13.44f, 11.06f, +CUBIC_TO, 13.15f, 10.76f, 13, 10.41f, 13, 10, +CUBIC_TO, 13, 9.58f, 13.15f, 9.23f, 13.44f, 8.94f, +CUBIC_TO, 13.74f, 8.65f, 14.09f, 8.5f, 14.5f, 8.5f, +CUBIC_TO, 14.92f, 8.5f, 15.27f, 8.65f, 15.56f, 8.94f, +CUBIC_TO, 15.85f, 9.24f, 16, 9.59f, 16, 10, +CUBIC_TO, 16, 10.42f, 15.85f, 10.77f, 15.56f, 11.06f, +CUBIC_TO, 15.26f, 11.35f, 14.91f, 11.5f, 14.5f, 11.5f, +CLOSE \ No newline at end of file diff --git a/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc index ca4ca764c8d47d..656b37672ad089 100644 --- a/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc +++ b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc @@ -15,7 +15,7 @@ AIChatToolbarButton::AIChatToolbarButton(PressedCallback callback) // Todo: to add localized text and vector icon SetTooltipText(u"Chat"); SetHorizontalAlignment(gfx::ALIGN_CENTER); - SetVectorIcon(kSpeakerIcon); + SetVectorIcon(kChatIcon); SetVisible(true); ConfigureInkDropForToolbar(this); } diff --git a/chrome/browser/ui/views/toolbar/browser_app_menu_button.cc b/chrome/browser/ui/views/toolbar/browser_app_menu_button.cc index 5300d5a128c80d..0633f7c3a370b4 100644 --- a/chrome/browser/ui/views/toolbar/browser_app_menu_button.cc +++ b/chrome/browser/ui/views/toolbar/browser_app_menu_button.cc @@ -151,7 +151,7 @@ void BrowserAppMenuButton::UpdateThemeBasedState() { void BrowserAppMenuButton::UpdateIcon() { const gfx::VectorIcon& icon = ui::TouchUiController::Get()->touch_ui() ? kBrowserToolsTouchIcon - : kBrowserToolsChromeRefreshIcon; + : kMoreHorizIcon; for (auto state : kButtonStates) { SkColor icon_color = GetForegroundColor(state); SetImageModel(state, ui::ImageModel::FromVectorIcon(icon, icon_color)); From b74b89a219a651279eb8bd9aec1eeaf13e22300d Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 23 Oct 2024 17:17:42 +0800 Subject: [PATCH 06/34] make all toolbar button highlighted with rounded corner and modify their corer --- chrome/browser/ui/browser_actions.cc | 5 ++--- chrome/browser/ui/views/toolbar/toolbar_button.cc | 9 +++++---- chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/chrome/browser/ui/browser_actions.cc b/chrome/browser/ui/browser_actions.cc index 677abd6318ac29..e46ad6f1f891a3 100644 --- a/chrome/browser/ui/browser_actions.cc +++ b/chrome/browser/ui/browser_actions.cc @@ -143,9 +143,8 @@ void BrowserActions::InitializeBrowserActions() { IDS_READ_LATER_TITLE, IDS_READ_LATER_TITLE, kReadingListIcon, kActionSidePanelShowReadingList, browser, true), - SidePanelAction(SidePanelEntryId::kAIChat, - IDS_AI_CHAT_TITLE, IDS_AI_CHAT_TITLE, - kSpeakerIcon, kActionAIChat, + SidePanelAction(SidePanelEntryId::kAIChat, IDS_AI_CHAT_TITLE, + IDS_AI_CHAT_TITLE, kChatIcon, kActionAIChat, browser, false), SidePanelAction(SidePanelEntryId::kAboutThisSite, IDS_PAGE_INFO_ABOUT_THIS_PAGE_TITLE, diff --git a/chrome/browser/ui/views/toolbar/toolbar_button.cc b/chrome/browser/ui/views/toolbar/toolbar_button.cc index 1328a7f6277005..80b2fdd357c37c 100644 --- a/chrome/browser/ui/views/toolbar/toolbar_button.cc +++ b/chrome/browser/ui/views/toolbar/toolbar_button.cc @@ -162,7 +162,7 @@ void ToolbarButton::ClearHighlight() { int ToolbarButton::GetRoundedCornerRadius() const { return ChromeLayoutProvider::Get()->GetCornerRadiusMetric( - views::Emphasis::kMaximum, GetTargetSize()); + views::Emphasis::kHigh, GetTargetSize()); } float ToolbarButton::GetCornerRadiusFor(ToolbarButton::Edge edge) const { @@ -253,12 +253,13 @@ void ToolbarButton::UpdateIconsWithColors(const gfx::VectorIcon& icon, SkColor pressed_color, SkColor disabled_color) { const int icon_size = GetIconSize(); + const int alpha = 150; SetImageModel(ButtonState::STATE_NORMAL, - ui::ImageModel::FromVectorIcon(icon, normal_color, icon_size)); + ui::ImageModel::FromVectorIcon(icon, SkColorSetA(normal_color, alpha), icon_size)); SetImageModel(ButtonState::STATE_HOVERED, - ui::ImageModel::FromVectorIcon(icon, hovered_color, icon_size)); + ui::ImageModel::FromVectorIcon(icon, SkColorSetA( hovered_color, alpha), icon_size)); SetImageModel(ButtonState::STATE_PRESSED, - ui::ImageModel::FromVectorIcon(icon, pressed_color, icon_size)); + ui::ImageModel::FromVectorIcon(icon, SkColorSetA(pressed_color, alpha), icon_size)); SetImageModel(Button::STATE_DISABLED, ui::ImageModel::FromVectorIcon( icon, disabled_color, icon_size)); } diff --git a/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc b/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc index a6f79cf9a3973a..9c26c0dd88edee 100644 --- a/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc +++ b/chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.cc @@ -44,7 +44,6 @@ class ToolbarButtonHighlightPathGenerator ); SkPath path; - // path.addRoundRect(gfx::RectToSkRect(rect), radii, radii); path.addRoundRect(gfx::RectToSkRect( gfx::Rect( rect)), radii, radii); return path; } From bafa6f9060d719793aa4c5312c8bd5db94957dcd Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 25 Oct 2024 18:33:30 +0800 Subject: [PATCH 07/34] Update chat.mojom for conversation - WIP --- .../resources/side_panel/chat/chat_app.ts | 7 ++- .../ui/webui/side_panel/chat/chat.mojom | 56 +++++++++++++++++-- .../side_panel/chat/chat_page_handler.cc | 21 +++++++ .../webui/side_panel/chat/chat_page_handler.h | 6 +- 4 files changed, 84 insertions(+), 6 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index daf9df0ce15ccf..67fbc046ffc662 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -13,7 +13,12 @@ export class ChatAppElement extends CrLitElement { private listenerIds_: number[] = []; - protected siteInfo_ : SiteInfo = { url : "url", title : "title"}; + protected siteInfo_ : SiteInfo = { + url: "url", + title: "title", + isContentUsableInConversations: false, + isContentModified: false, + }; constructor() { super(); diff --git a/chrome/browser/ui/webui/side_panel/chat/chat.mojom b/chrome/browser/ui/webui/side_panel/chat/chat.mojom index ab6e0d9aee8597..3d5ba70ed354fc 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat.mojom +++ b/chrome/browser/ui/webui/side_panel/chat/chat.mojom @@ -2,18 +2,52 @@ module chat.mojom; import "url/mojom/url.mojom"; +enum APIError { + None, + ConnectionError, + RateLimitReached, + ContextLimitReached, +}; + struct SiteInfo { - // The title of the currently active tab if it has opening page + // The title of the currently active tab, if it has an open page string? title; - // The url of the currently active tab if it has opening page + // The url of the currently active tab if it has opening page or probaly file which we might support in future string? url; + + // Indicates whether the current URL contains content that can be used in conversations with Yep Chat + bool is_content_usable_in_conversations; + + // Indicates that the content has been refined by distilling the most relevant sections from lengthy page material. + bool is_content_modified; +}; + +enum ActionType { + SUMMARIZE_PAGE, + SUMMARIZE_VIDEO, + EXPLAIN, + TRANSLATE, + DRAFT_SOCIAL_MEDIA_POST, + FACT_CHECK, + QUERY, + NONE, +}; + +struct ActionItem { + ActionType action_type; + string label; +}; + +struct ActionResponse { + ActionType action_type; + string result; }; // Used by the WebUI page to bootstrap bidirectional communication. interface PageHandlerFactory { - CreatePageHandler(pending_remote page, - pending_receiver handler); + CreatePageHandler(pending_remote page, + pending_receiver handler); }; // Browser-side handler for requests from WebUI page. @@ -22,6 +56,18 @@ interface PageHandler { GetSiteInfo() => (SiteInfo site_info); + // Show as menu items in Chat UI + // This will be called only when SiteInfo.is_content_usable_in_conversations is true + GetActionList() => (array action_list); + + // The user selects a predefined action by clicking on menu items in the Chat UI + // The content of the current opening page will serve as context + // and a predefined prompt will be applied accordingly. + SubmitAction(ActionType action_type); + + // Usual conversation + SubmitQuery(ActionType action_type, string query); + // Notify the backend that the UI is ready to be shown. ShowUI(); @@ -33,4 +79,6 @@ interface PageHandler { // Browser -> Renderer interface Page { OnSiteInfoChanged(SiteInfo info); + + OnSubmitActionResponse(ActionResponse response); }; \ No newline at end of file diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index ed43fe6b5de8bc..d663181b3f5106 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -52,3 +52,24 @@ void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { site_info->url = url; std::move(callback).Run(site_info.Clone()); } + +void ChatPageHandler::GetActionList(GetActionListCallback callback) { +// chat::mojom::ActionItemPtr summarize_item = chat::mojom::ActionItem::New(); +// summarize_item->action_type = chat::mojom::ActionType::SUMMARIZE_PAGE; +// summarize_item->label = "Summarize this page"; +// chat::mojom::ActionItemPtr action_items[1] = {summarize_item.Clone()}; +// std::move(callback).Run(action_items); +} + +void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { + if (page_.is_bound()) { + chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); + response->action_type = action_type; + response->result = "Mock Result"; + page_->OnSubmitActionResponse(response.Clone()); + } +} + +void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { + +} diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 9d4ee0f2bc375f..9ff96f9af24f9e 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -26,9 +26,13 @@ class ChatPageHandler : public chat::mojom::PageHandler { ~ChatPageHandler() override; + void GetSiteInfo(GetSiteInfoCallback callback) override; + void GetActionList(GetActionListCallback cllback) override; + void SubmitAction(chat::mojom::ActionType action_type) override; + void SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) override; void ShowUI() override; void CloseUI() override; - void GetSiteInfo(GetSiteInfoCallback callback) override; + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); private: From 5fe6be27e306493dd284aefe652a58c5ecdb457e Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 28 Oct 2024 19:59:14 +0800 Subject: [PATCH 08/34] Change side_panel chat folder name --- chrome/browser/ui/views/side_panel/BUILD.gn | 8 +-- .../ai_chat/ai_chat_side_panel_coordinator.cc | 60 ------------------- .../ai_chat/ai_chat_side_panel_coordinator.h | 31 ---------- .../chat/chat_side_panel_coordinator.cc | 59 ++++++++++++++++++ .../chat/chat_side_panel_coordinator.h | 32 ++++++++++ ...eb_view.cc => chat_side_panel_web_view.cc} | 26 ++++---- ..._web_view.h => chat_side_panel_web_view.h} | 22 +++---- .../ui/views/side_panel/side_panel_util.cc | 14 ++--- 8 files changed, 126 insertions(+), 126 deletions(-) delete mode 100644 chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc delete mode 100644 chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h create mode 100644 chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.cc create mode 100644 chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h rename chrome/browser/ui/views/side_panel/{ai_chat_side_panel_web_view.cc => chat_side_panel_web_view.cc} (81%) rename chrome/browser/ui/views/side_panel/{ai_chat_side_panel_web_view.h => chat_side_panel_web_view.h} (61%) diff --git a/chrome/browser/ui/views/side_panel/BUILD.gn b/chrome/browser/ui/views/side_panel/BUILD.gn index 6b6b4e78723821..4cf7e092b8d359 100644 --- a/chrome/browser/ui/views/side_panel/BUILD.gn +++ b/chrome/browser/ui/views/side_panel/BUILD.gn @@ -13,12 +13,12 @@ source_set("side_panel_enums") { source_set("side_panel") { sources = [ - "ai_chat_side_panel_web_view.cc", - "ai_chat_side_panel_web_view.h", - "ai_chat/ai_chat_side_panel_coordinator.cc", - "ai_chat/ai_chat_side_panel_coordinator.h", "bookmarks/bookmarks_side_panel_coordinator.cc", "bookmarks/bookmarks_side_panel_coordinator.h", + "chat/chat_side_panel_coordinator.cc", + "chat/chat_side_panel_coordinator.h", + "chat_side_panel_web_view.cc", + "chat_side_panel_web_view.h", "companion/companion_side_panel_controller_utils.h", "companion/companion_tab_helper.cc", "companion/companion_tab_helper.h", diff --git a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc deleted file mode 100644 index 277e7474642c44..00000000000000 --- a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.cc +++ /dev/null @@ -1,60 +0,0 @@ -#include "ai_chat_side_panel_coordinator.h" - -#include - -#include "chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h" -#include "base/functional/callback.h" -#include "chrome/app/vector_icons/vector_icons.h" -#include "chrome/browser/ui/browser.h" -#include "chrome/browser/ui/views/frame/browser_view.h" -#include "chrome/browser/ui/views/side_panel/side_panel_entry.h" -#include "chrome/browser/ui/views/side_panel/side_panel_registry.h" -#include "chrome/grit/generated_resources.h" -#include "ui/base/l10n/l10n_util.h" -#include "ui/base/models/image_model.h" -#include "ui/base/ui_base_features.h" -#include "ui/views/vector_icons.h" -#include "chrome/browser/ui/views/toolbar/toolbar_view.h" - -AIChatSidePanelCoordinator::AIChatSidePanelCoordinator(Browser* browser) - : BrowserUserData(*browser) {} - -AIChatSidePanelCoordinator::~AIChatSidePanelCoordinator() = default; - -void AIChatSidePanelCoordinator::CreateAndRegisterEntry(SidePanelRegistry* global_registry) { - global_registry->Register(std::make_unique( - SidePanelEntry::Id::kAIChat, - base::BindRepeating( - &AIChatSidePanelCoordinator::CreateAIChatWebView, - base::Unretained(this)))); -} - -std::unique_ptr -AIChatSidePanelCoordinator::CreateAIChatWebView() { - return std::make_unique(&GetBrowser(), - base::RepeatingClosure()); -} - -void AIChatSidePanelCoordinator::UpdateOpeningPanelId(SidePanelEntryId panel_id) { - auto* browser = &GetBrowser() ; - auto* browser_view = BrowserView::GetBrowserViewForBrowser(browser); - if (browser_view) { - auto* toolbar = browser_view->toolbar(); - if (panel_id != SidePanelEntry::Id::kAIChat) { - toolbar->ResetHighlightForAIChatButton(); - } else { - toolbar->AddHighlightForAIChatButton(); - } - } -} - -void AIChatSidePanelCoordinator::UpdateClosingPanelId(SidePanelEntryId panel_id) { - auto* browser = &GetBrowser() ; - auto* browser_view = BrowserView::GetBrowserViewForBrowser(browser); - if (browser_view && panel_id == SidePanelEntry::Id::kAIChat) { - auto* toolbar = browser_view->toolbar(); - toolbar->ResetHighlightForAIChatButton(); - } -} - -BROWSER_USER_DATA_KEY_IMPL(AIChatSidePanelCoordinator); diff --git a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h b/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h deleted file mode 100644 index c4b349a54e7248..00000000000000 --- a/chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef CHROMIUM_AI_CHAT_SIDE_PANEL_COORDINATOR_H -#define CHROMIUM_AI_CHAT_SIDE_PANEL_COORDINATOR_H - -#include "chrome/browser/ui/browser_user_data.h" -#include "chrome/browser/ui/views/side_panel/side_panel_entry_id.h" - -class Browser; -class SidePanelRegistry; - -namespace views { - class View; -} // namespace views - -class AIChatSidePanelCoordinator : public BrowserUserData { - public: - explicit AIChatSidePanelCoordinator(Browser* browser); - ~AIChatSidePanelCoordinator() override; - - void CreateAndRegisterEntry(SidePanelRegistry* global_registry); - void UpdateOpeningPanelId(SidePanelEntryId panel_id); - void UpdateClosingPanelId(SidePanelEntryId panel_id); - - private: - friend class BrowserUserData; - - std::unique_ptr CreateAIChatWebView(); - - BROWSER_USER_DATA_KEY_DECL(); -}; - -#endif //CHROMIUM_AI_CHAT_SIDE_PANEL_COORDINATOR_H diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.cc new file mode 100644 index 00000000000000..8ce3c08c36d243 --- /dev/null +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.cc @@ -0,0 +1,59 @@ +#include "chat_side_panel_coordinator.h" + +#include + +#include "base/functional/callback.h" +#include "chrome/app/vector_icons/vector_icons.h" +#include "chrome/browser/ui/browser.h" +#include "chrome/browser/ui/views/frame/browser_view.h" +#include "chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h" +#include "chrome/browser/ui/views/side_panel/side_panel_entry.h" +#include "chrome/browser/ui/views/side_panel/side_panel_registry.h" +#include "chrome/browser/ui/views/toolbar/toolbar_view.h" +#include "chrome/grit/generated_resources.h" +#include "ui/base/l10n/l10n_util.h" +#include "ui/base/models/image_model.h" +#include "ui/base/ui_base_features.h" +#include "ui/views/vector_icons.h" + +ChatSidePanelCoordinator::ChatSidePanelCoordinator(Browser* browser) + : BrowserUserData(*browser) {} + +ChatSidePanelCoordinator::~ChatSidePanelCoordinator() = default; + +void ChatSidePanelCoordinator::CreateAndRegisterEntry( + SidePanelRegistry* global_registry) { + global_registry->Register(std::make_unique( + SidePanelEntry::Id::kAIChat, + base::BindRepeating(&ChatSidePanelCoordinator::CreateChatWebView, + base::Unretained(this)))); +} + +std::unique_ptr ChatSidePanelCoordinator::CreateChatWebView() { + return std::make_unique(&GetBrowser(), + base::RepeatingClosure()); +} + +void ChatSidePanelCoordinator::UpdateOpeningPanelId(SidePanelEntryId panel_id) { + auto* browser = &GetBrowser(); + auto* browser_view = BrowserView::GetBrowserViewForBrowser(browser); + if (browser_view) { + auto* toolbar = browser_view->toolbar(); + if (panel_id != SidePanelEntry::Id::kAIChat) { + toolbar->ResetHighlightForAIChatButton(); + } else { + toolbar->AddHighlightForAIChatButton(); + } + } +} + +void ChatSidePanelCoordinator::UpdateClosingPanelId(SidePanelEntryId panel_id) { + auto* browser = &GetBrowser(); + auto* browser_view = BrowserView::GetBrowserViewForBrowser(browser); + if (browser_view && panel_id == SidePanelEntry::Id::kAIChat) { + auto* toolbar = browser_view->toolbar(); + toolbar->ResetHighlightForAIChatButton(); + } +} + +BROWSER_USER_DATA_KEY_IMPL(ChatSidePanelCoordinator); diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h new file mode 100644 index 00000000000000..a068ee9d28fb48 --- /dev/null +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h @@ -0,0 +1,32 @@ +#ifndef CHROMIUM_CHAT_SIDE_PANEL_COORDINATOR_H +#define CHROMIUM_CHAT_SIDE_PANEL_COORDINATOR_H + +#include "chrome/browser/ui/browser_user_data.h" +#include "chrome/browser/ui/views/side_panel/side_panel_entry_id.h" + +class Browser; +class SidePanelRegistry; + +namespace views { +class View; +} // namespace views + +class ChatSidePanelCoordinator + : public BrowserUserData { + public: + explicit ChatSidePanelCoordinator(Browser* browser); + ~ChatSidePanelCoordinator() override; + + void CreateAndRegisterEntry(SidePanelRegistry* global_registry); + void UpdateOpeningPanelId(SidePanelEntryId panel_id); + void UpdateClosingPanelId(SidePanelEntryId panel_id); + + private: + friend class BrowserUserData; + + std::unique_ptr CreateChatWebView(); + + BROWSER_USER_DATA_KEY_DECL(); +}; + +#endif // CHROMIUM_CHAT_SIDE_PANEL_COORDINATOR_H diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc similarity index 81% rename from chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc rename to chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc index 8560018055e775..03d4d0c06c3cf6 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc @@ -1,4 +1,4 @@ -#include "ai_chat_side_panel_web_view.h" +#include "chat_side_panel_web_view.h" #include #include @@ -27,11 +27,11 @@ using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; BEGIN_TEMPLATE_METADATA(SidePanelWebUIViewT_ChatUI, SidePanelWebUIViewT) END_METADATA -AIChatSidePanelWebView::AIChatSidePanelWebView(Browser* browser, - base::RepeatingClosure close_cb) +ChatSidePanelWebView::ChatSidePanelWebView(Browser* browser, + base::RepeatingClosure close_cb) : SidePanelWebUIViewT( base::BindRepeating( - &AIChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab, + &ChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab, base::Unretained(this)), close_cb, std::make_unique>( @@ -44,7 +44,7 @@ AIChatSidePanelWebView::AIChatSidePanelWebView(Browser* browser, browser_->tab_strip_model()->AddObserver(this); } -void AIChatSidePanelWebView::OnTabStripModelChanged( +void ChatSidePanelWebView::OnTabStripModelChanged( TabStripModel* tab_strip_model, const TabStripModelChange& change, const TabStripSelectionChange& selection) { @@ -53,16 +53,16 @@ void AIChatSidePanelWebView::OnTabStripModelChanged( } } -void AIChatSidePanelWebView::TabChangedAt(content::WebContents* contents, - int index, - TabChangeType change_type) { +void ChatSidePanelWebView::TabChangedAt(content::WebContents* contents, + int index, + TabChangeType change_type) { if (GetVisible() && index == browser_->tab_strip_model()->active_index() && change_type == TabChangeType::kAll) { UpdateActiveSiteInfo(browser_->tab_strip_model()->GetWebContentsAt(index)); } } -void AIChatSidePanelWebView::UpdateActiveSiteInfo( +void ChatSidePanelWebView::UpdateActiveSiteInfo( content::WebContents* contents) { auto* controller = contents_wrapper()->GetWebUIController(); if (!controller || !contents) { @@ -101,11 +101,11 @@ void AIChatSidePanelWebView::UpdateActiveSiteInfo( controller->GetAs()->SetSiteInfo(site_info.Clone()); } -void AIChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab() { +void ChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab() { UpdateActiveSiteInfo(browser_->tab_strip_model()->GetActiveWebContents()); } -AIChatSidePanelWebView::~AIChatSidePanelWebView() = default; +ChatSidePanelWebView::~ChatSidePanelWebView() = default; -BEGIN_METADATA(AIChatSidePanelWebView) -END_METADATA \ No newline at end of file +BEGIN_METADATA(ChatSidePanelWebView) +END_METADATA diff --git a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h similarity index 61% rename from chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h rename to chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h index 90730f1f933695..7bda0b04b78d35 100644 --- a/chrome/browser/ui/views/side_panel/ai_chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h @@ -1,5 +1,5 @@ -#ifndef CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H -#define CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H +#ifndef CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H +#define CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H #include "base/functional/callback_forward.h" #include "base/memory/raw_ptr.h" @@ -13,15 +13,15 @@ class Browser; -class AIChatSidePanelWebView : public SidePanelWebUIViewT, - public TabStripModelObserver { +class ChatSidePanelWebView : public SidePanelWebUIViewT, + public TabStripModelObserver { using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; - METADATA_HEADER(AIChatSidePanelWebView, SidePanelWebUIViewT_ChatUI) + METADATA_HEADER(ChatSidePanelWebView, SidePanelWebUIViewT_ChatUI) public: - AIChatSidePanelWebView(Browser* browser, base::RepeatingClosure close_cb); - AIChatSidePanelWebView(const AIChatSidePanelWebView&) = delete; - AIChatSidePanelWebView& operator=(const AIChatSidePanelWebView&) = delete; - ~AIChatSidePanelWebView() override; + ChatSidePanelWebView(Browser* browser, base::RepeatingClosure close_cb); + ChatSidePanelWebView(const ChatSidePanelWebView&) = delete; + ChatSidePanelWebView& operator=(const ChatSidePanelWebView&) = delete; + ~ChatSidePanelWebView() override; // TabStripModelObserver: void OnTabStripModelChanged( @@ -38,6 +38,6 @@ class AIChatSidePanelWebView : public SidePanelWebUIViewT, private: const raw_ptr browser_; - base::WeakPtrFactory weak_factory_{this}; + base::WeakPtrFactory weak_factory_{this}; }; -#endif //CHROMIUM_AI_CHAT_SIDE_PANEL_WEB_VIEW_H \ No newline at end of file +#endif // CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H diff --git a/chrome/browser/ui/views/side_panel/side_panel_util.cc b/chrome/browser/ui/views/side_panel/side_panel_util.cc index 3514641e531a3f..a635094c1f2589 100644 --- a/chrome/browser/ui/views/side_panel/side_panel_util.cc +++ b/chrome/browser/ui/views/side_panel/side_panel_util.cc @@ -16,7 +16,9 @@ #include "chrome/browser/ui/browser_window/public/browser_window_features.h" #include "chrome/browser/ui/ui_features.h" #include "chrome/browser/ui/views/side_panel/bookmarks/bookmarks_side_panel_coordinator.h" +#include "chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h" #include "chrome/browser/ui/views/side_panel/companion/companion_utils.h" +#include "chrome/browser/ui/views/side_panel/extensions/extension_side_panel_manager.h" #include "chrome/browser/ui/views/side_panel/history_clusters/history_clusters_side_panel_coordinator.h" #include "chrome/browser/ui/views/side_panel/reading_list/reading_list_side_panel_coordinator.h" #include "chrome/browser/ui/views/side_panel/search_companion/search_companion_side_panel_coordinator.h" @@ -30,8 +32,6 @@ #include "components/user_notes/user_notes_features.h" #include "ui/accessibility/accessibility_features.h" #include "ui/actions/actions.h" -#include "chrome/browser/ui/views/side_panel/extensions/extension_side_panel_manager.h" -#include "chrome/browser/ui/views/side_panel/ai_chat/ai_chat_side_panel_coordinator.h" // static void SidePanelUtil::PopulateGlobalEntries(Browser* browser, @@ -41,7 +41,7 @@ void SidePanelUtil::PopulateGlobalEntries(Browser* browser, ->CreateAndRegisterEntry(window_registry); // Add ai chat - AIChatSidePanelCoordinator::GetOrCreateForBrowser(browser) + ChatSidePanelCoordinator::GetOrCreateForBrowser(browser) ->CreateAndRegisterEntry(window_registry); // Add bookmarks. @@ -100,8 +100,8 @@ void SidePanelUtil::RecordSidePanelClosed(Browser* browser, base::UmaHistogramLongTimes("SidePanel.OpenDuration", base::TimeTicks::Now() - opened_timestamp); - - auto* ai_chat_coordinator = AIChatSidePanelCoordinator::GetOrCreateForBrowser(browser); + auto* ai_chat_coordinator = + ChatSidePanelCoordinator::GetOrCreateForBrowser(browser); ai_chat_coordinator->UpdateClosingPanelId(id); } @@ -158,8 +158,8 @@ void SidePanelUtil::RecordEntryShowTriggeredMetrics( Browser* browser, SidePanelEntry::Id id, std::optional trigger) { - - auto* ai_chat_coordinator = AIChatSidePanelCoordinator::GetOrCreateForBrowser(browser); + auto* ai_chat_coordinator = + ChatSidePanelCoordinator::GetOrCreateForBrowser(browser); ai_chat_coordinator->UpdateOpeningPanelId(id); if (trigger.has_value()) { From 217571869bb81da1a39a4d35934be16ca263f87e Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 29 Oct 2024 01:55:05 +0800 Subject: [PATCH 09/34] Add string values used in the chat feature into the translation file so it can be localized into different languages. --- chrome/app/generated_resources.grd | 18 +++++++++ .../resources/side_panel/chat/chat.html | 2 +- .../side_panel/chat/chat_app.html.ts | 2 +- .../resources/side_panel/chat/chat_app.ts | 4 ++ .../views/toolbar/ai_chat_toolbar_button.cc | 18 ++++----- .../ui/views/toolbar/toolbar_button.cc | 3 ++ .../ui/webui/side_panel/chat/chat.mojom | 1 - .../side_panel/chat/chat_page_handler.cc | 38 ++++++++++++++++--- .../webui/side_panel/chat/chat_page_handler.h | 2 +- .../ui/webui/side_panel/chat/chat_ui.cc | 9 +++++ .../ui/webui/side_panel/chat/chat_ui.h | 1 - 11 files changed, 79 insertions(+), 19 deletions(-) diff --git a/chrome/app/generated_resources.grd b/chrome/app/generated_resources.grd index d479b3f86f2d02..3a630346eeb457 100644 --- a/chrome/app/generated_resources.grd +++ b/chrome/app/generated_resources.grd @@ -8976,6 +8976,24 @@ Keep your key file in a safe place. You will need it to create new versions of y Chat + + Summarize this page + + + Explain it in simple language + + + Translate... + + + Draft a social media post... + + + Fact check + + + Ask anything... + diff --git a/chrome/browser/resources/side_panel/chat/chat.html b/chrome/browser/resources/side_panel/chat/chat.html index 9aa0e342f292be..ba033d5c1f9d82 100644 --- a/chrome/browser/resources/side_panel/chat/chat.html +++ b/chrome/browser/resources/side_panel/chat/chat.html @@ -2,7 +2,7 @@ - Chat + $i18n{title} diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 72b5c2e759f00f..8720a8d97c9988 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -10,7 +10,7 @@ export function getHtml(this: ChatAppElement) {
- + send
`; diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 67fbc046ffc662..7e82d4e3013d29 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -5,6 +5,7 @@ import {getCss} from './chat_app.css.js'; import {getHtml} from './chat_app.html.js'; import type {ChatApiProxy} from "./chat_api_proxy.js"; import {ChatApiProxyImpl} from "./chat_api_proxy.js"; +import {loadTimeData} from 'chrome://resources/js/load_time_data.js'; import {SiteInfo} from "./chat.mojom-webui.js"; @@ -20,6 +21,8 @@ export class ChatAppElement extends CrLitElement { isContentModified: false, }; + protected askAnythingLabel_ = loadTimeData.getString('askAnything'); + constructor() { super(); } @@ -34,6 +37,7 @@ export class ChatAppElement extends CrLitElement { static override get properties() { return { siteInfo_: {type: Object}, + askAnythingLabel_: {type: String}, }; } diff --git a/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc index 656b37672ad089..778e899ee68908 100644 --- a/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc +++ b/chrome/browser/ui/views/toolbar/ai_chat_toolbar_button.cc @@ -1,23 +1,23 @@ #include "ai_chat_toolbar_button.h" -#include "ui/base/metadata/metadata_header_macros.h" + #include "chrome/app/vector_icons/vector_icons.h" -#include "components/vector_icons/vector_icons.h" #include "chrome/browser/ui/browser.h" #include "chrome/browser/ui/view_ids.h" +#include "chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.h" #include "chrome/grit/generated_resources.h" +#include "components/vector_icons/vector_icons.h" #include "ui/base/l10n/l10n_util.h" +#include "ui/base/metadata/metadata_header_macros.h" #include "ui/base/metadata/metadata_impl_macros.h" #include "ui/views/view.h" -#include "chrome/browser/ui/views/toolbar/toolbar_ink_drop_util.h" AIChatToolbarButton::AIChatToolbarButton(PressedCallback callback) : ToolbarButton(std::move(callback)) { - // Todo: to add localized text and vector icon - SetTooltipText(u"Chat"); - SetHorizontalAlignment(gfx::ALIGN_CENTER); - SetVectorIcon(kChatIcon); - SetVisible(true); - ConfigureInkDropForToolbar(this); + SetTooltipText(l10n_util::GetStringUTF16(IDS_AI_CHAT_TITLE)); + SetHorizontalAlignment(gfx::ALIGN_CENTER); + SetVectorIcon(kChatIcon); + SetVisible(true); + ConfigureInkDropForToolbar(this); } AIChatToolbarButton::~AIChatToolbarButton() = default; diff --git a/chrome/browser/ui/views/toolbar/toolbar_button.cc b/chrome/browser/ui/views/toolbar/toolbar_button.cc index 80b2fdd357c37c..75cbe02f6b7f1d 100644 --- a/chrome/browser/ui/views/toolbar/toolbar_button.cc +++ b/chrome/browser/ui/views/toolbar/toolbar_button.cc @@ -253,7 +253,10 @@ void ToolbarButton::UpdateIconsWithColors(const gfx::VectorIcon& icon, SkColor pressed_color, SkColor disabled_color) { const int icon_size = GetIconSize(); + + // todo: to check alpha value with Yuri when UI customization starts const int alpha = 150; + SetImageModel(ButtonState::STATE_NORMAL, ui::ImageModel::FromVectorIcon(icon, SkColorSetA(normal_color, alpha), icon_size)); SetImageModel(ButtonState::STATE_HOVERED, diff --git a/chrome/browser/ui/webui/side_panel/chat/chat.mojom b/chrome/browser/ui/webui/side_panel/chat/chat.mojom index 3d5ba70ed354fc..bf86be573b2287 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat.mojom +++ b/chrome/browser/ui/webui/side_panel/chat/chat.mojom @@ -25,7 +25,6 @@ struct SiteInfo { enum ActionType { SUMMARIZE_PAGE, - SUMMARIZE_VIDEO, EXPLAIN, TRANSLATE, DRAFT_SOCIAL_MEDIA_POST, diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index d663181b3f5106..fddcaf8750c140 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -1,11 +1,14 @@ #include "chat_page_handler.h" #include +#include #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" +#include "chrome/grit/generated_resources.h" #include "content/public/browser/web_contents.h" #include "content/public/browser/web_ui.h" +#include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" ChatPageHandler::ChatPageHandler( @@ -54,11 +57,36 @@ void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { } void ChatPageHandler::GetActionList(GetActionListCallback callback) { -// chat::mojom::ActionItemPtr summarize_item = chat::mojom::ActionItem::New(); -// summarize_item->action_type = chat::mojom::ActionType::SUMMARIZE_PAGE; -// summarize_item->label = "Summarize this page"; -// chat::mojom::ActionItemPtr action_items[1] = {summarize_item.Clone()}; -// std::move(callback).Run(action_items); + std::vector action_items; + + chat::mojom::ActionItemPtr summarize_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::SUMMARIZE_PAGE, + l10n_util::GetStringUTF8(IDS_CHAT_SUMMARIZE_THIS_PAGE)); + + chat::mojom::ActionItemPtr explain_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::EXPLAIN, + l10n_util::GetStringUTF8(IDS_CHAT_EXPLAIN_IT_IN_SIMPLE_LANGUAGE)); + + chat::mojom::ActionItemPtr translate_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::TRANSLATE, + l10n_util::GetStringUTF8(IDS_CHAT_TRANSLATE)); + + chat::mojom::ActionItemPtr draft_social_media_post_item = + chat::mojom::ActionItem::New( + chat::mojom::ActionType::DRAFT_SOCIAL_MEDIA_POST, + l10n_util::GetStringUTF8(IDS_CHAT_DRAFT_A_SOCIAL_MEDIA_POST)); + + chat::mojom::ActionItemPtr fact_check_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::FACT_CHECK, + l10n_util::GetStringUTF8(IDS_CHAT_DRAFT_FACT_CHECT)); + + action_items.push_back(summarize_item.Clone()); + action_items.push_back(explain_item.Clone()); + action_items.push_back(translate_item.Clone()); + action_items.push_back(draft_social_media_post_item.Clone()); + action_items.push_back(fact_check_item.Clone()); + + std::move(callback).Run(std::move(action_items)); } void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 9ff96f9af24f9e..b429a9636d934d 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -27,7 +27,7 @@ class ChatPageHandler : public chat::mojom::PageHandler { ~ChatPageHandler() override; void GetSiteInfo(GetSiteInfoCallback callback) override; - void GetActionList(GetActionListCallback cllback) override; + void GetActionList(GetActionListCallback callback) override; void SubmitAction(chat::mojom::ActionType action_type) override; void SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) override; void ShowUI() override; diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc index 5ebea1f16c8f64..26ef74af39fc91 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc @@ -13,6 +13,7 @@ #include "chrome/grit/side_panel_chat_resources_map.h" #include "ui/webui/mojo_web_ui_controller.h" #include "chrome/browser/profiles/profile.h" +#include "chrome/grit/generated_resources.h" #pragma allow_unsafe_buffers @@ -21,6 +22,14 @@ ChatUI::ChatUI(content::WebUI* web_ui) Profile* const profile = Profile::FromWebUI(web_ui); content::WebUIDataSource* source = content::WebUIDataSource::CreateAndAdd( profile, chrome::kChromeUIChatHost); + + static constexpr webui::LocalizedString kLocalizedStrings[] = { + {"title", IDS_AI_CHAT_TITLE}, + {"askAnything", IDS_CHAT_ASK_ANYTHING}, + }; + for (const auto& str : kLocalizedStrings) + webui::AddLocalizedString(source, str.name, str.id); + webui::SetupWebUIDataSource( source, base::make_span(kSidePanelChatResources, kSidePanelChatResourcesSize), diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h index 2a04af4369ae6f..6d1ac9086bed95 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -47,7 +47,6 @@ class ChatUI : public TopChromeWebUIController, void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); private: - // chat::mojom::PageHandlerFactory: void CreatePageHandler( mojo::PendingRemote page, mojo::PendingReceiver receiver) override; From 46f55f1c935cf51cc5b11df3544b081778eed4ae Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 29 Oct 2024 12:51:05 +0800 Subject: [PATCH 10/34] Display menu item list - WIP --- .../side_panel/chat/chat_api_proxy.ts | 31 +++++++++--- .../side_panel/chat/chat_app.html.ts | 17 +++++-- .../resources/side_panel/chat/chat_app.ts | 49 +++++++++++++------ .../side_panel/chat_side_panel_web_view.cc | 5 ++ .../side_panel/chat/chat_page_handler.cc | 23 ++++++++- 5 files changed, 96 insertions(+), 29 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts index c6558617a554d1..f8c9999a505510 100644 --- a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts +++ b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts @@ -1,8 +1,11 @@ -import {PageCallbackRouter, PageHandlerFactory, PageHandlerRemote, SiteInfo} - from "./chat.mojom-webui.js"; +import {PageCallbackRouter, PageHandlerFactory, PageHandlerRemote, SiteInfo, + ActionItem, ActionType} from "./chat.mojom-webui.js"; export interface ChatApiProxy { - getSiteInfo():Promise<{siteInfo: SiteInfo}> + getActionList(): Promise<{ actionList: ActionItem[] }>; + submitAction(actionType: ActionType): void; + submitQuery(actionType: ActionType, query: string): void; + getSiteInfo(): Promise<{siteInfo: SiteInfo}> showUI(): void; closeUI(): void; getCallbackRouter(): PageCallbackRouter; @@ -12,7 +15,7 @@ let instance: ChatApiProxy|null = null; export class ChatApiProxyImpl implements ChatApiProxy { private readonly callbackRouter: PageCallbackRouter = new PageCallbackRouter(); - handler: PageHandlerRemote = new PageHandlerRemote(); + private handler: PageHandlerRemote = new PageHandlerRemote(); constructor() { this.callbackRouter = new PageCallbackRouter(); @@ -31,8 +34,20 @@ export class ChatApiProxyImpl implements ChatApiProxy { instance = proxy; } - getCallbackRouter() { - return this.callbackRouter; + getActionList(): Promise<{ actionList: ActionItem[] }> { + return this.handler.getActionList() ; + } + + submitAction(actionType: ActionType) { + this.handler.submitAction(actionType); + } + + submitQuery(_: ActionType, __: string) { + + } + + getSiteInfo() { + return this.handler.getSiteInfo(); } showUI() { @@ -43,7 +58,7 @@ export class ChatApiProxyImpl implements ChatApiProxy { this.handler.closeUI(); } - getSiteInfo() { - return this.handler.getSiteInfo(); + getCallbackRouter() { + return this.callbackRouter; } } \ No newline at end of file diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 8720a8d97c9988..df40f63ece12ca 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -8,10 +8,19 @@ export function getHtml(this: ChatAppElement) {
${this.siteInfo_.url}
${this.siteInfo_.title}
-
-
- - send +
+ ${this.submitResponse_.result} +
+
+ ${this.actionList_.map((item,_) => html` + + ${item.label} + + `)} +
+ + send +
`; } \ No newline at end of file diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 7e82d4e3013d29..91edccab4482c8 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -1,31 +1,33 @@ import './strings.m.js'; +import {loadTimeData} from 'chrome://resources/js/load_time_data.js'; import {CrLitElement} from '//resources/lit/v3_0/lit.rollup.js'; import {getCss} from './chat_app.css.js'; import {getHtml} from './chat_app.html.js'; import type {ChatApiProxy} from "./chat_api_proxy.js"; import {ChatApiProxyImpl} from "./chat_api_proxy.js"; -import {loadTimeData} from 'chrome://resources/js/load_time_data.js'; +import {SiteInfo, ActionItem, ActionType, ActionResponse} from "./chat.mojom-webui.js"; -import {SiteInfo} - from "./chat.mojom-webui.js"; export class ChatAppElement extends CrLitElement { private chatApiProxy_: ChatApiProxy = ChatApiProxyImpl.getInstance(); - - private listenerIds_: number[] = []; + protected actionList_: ActionItem[] = []; + protected askAnythingLabel_ = loadTimeData.getString('askAnything'); protected siteInfo_ : SiteInfo = { - url: "url", - title: "title", + url: "", + title: "", isContentUsableInConversations: false, isContentModified: false, }; - - protected askAnythingLabel_ = loadTimeData.getString('askAnything'); + protected submitResponse_ : ActionResponse = { + actionType: ActionType.NONE, + result: "" + }; constructor() { super(); } + static get is() { return 'chat-app'; } @@ -38,29 +40,45 @@ export class ChatAppElement extends CrLitElement { return { siteInfo_: {type: Object}, askAnythingLabel_: {type: String}, + actionList_: {type: Array}, + submitResponse_: {type: Object}, }; } + private async updateSiteInfo(siteInfo: SiteInfo) { + this.siteInfo_ = siteInfo; + if (this.siteInfo_.isContentUsableInConversations) { + const {actionList} = await this.chatApiProxy_.getActionList(); + this.actionList_ = actionList; + } + this.updateComplete; + } + private async updateSubmitResponse(response: ActionResponse) { + this.submitResponse_ = response; + } -private async updateSiteInfo(siteInfo: SiteInfo) { - this.siteInfo_ = siteInfo; - await this.updateComplete; + protected onSubmitAction_(e: Event) { + e.stopPropagation(); + this.chatApiProxy_.submitAction(ActionType.SUMMARIZE_PAGE); } override connectedCallback() { super.connectedCallback(); + setTimeout(() => this.chatApiProxy_.showUI(), 0); this.listenerIds_.push( this.chatApiProxy_.getCallbackRouter().onSiteInfoChanged.addListener( - (siteInfo: SiteInfo) => - this.updateSiteInfo(siteInfo))); - + (siteInfo: SiteInfo) => this.updateSiteInfo(siteInfo)), + this.chatApiProxy_.getCallbackRouter().onSubmitActionResponse.addListener( + (response: ActionResponse) => this.updateSubmitResponse(response)) + ); } override disconnectedCallback() { super.disconnectedCallback(); + this.listenerIds_.forEach( id => this.chatApiProxy_.getCallbackRouter().removeListener(id)); } @@ -68,7 +86,6 @@ private async updateSiteInfo(siteInfo: SiteInfo) { override render() { return getHtml.bind(this)(); } - } declare global { diff --git a/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc index 03d4d0c06c3cf6..0f0746fc305203 100644 --- a/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.cc @@ -98,6 +98,11 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( site_info->title = base::UTF16ToUTF8(title); site_info->url = url; + // todo: to check the schema of the current tab + site_info->is_content_usable_in_conversations = true; + // todo: to check the content of the current tab + site_info->is_content_modified = false; + controller->GetAs()->SetSiteInfo(site_info.Clone()); } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index fddcaf8750c140..0df9a2adce5823 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -53,6 +53,13 @@ void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); site_info->title = title; site_info->url = url; + + // todo: to check the schema of the current tab + site_info->is_content_usable_in_conversations = true; + + // todo: to check the content of the current tab + site_info->is_content_modified = false; + std::move(callback).Run(site_info.Clone()); } @@ -90,14 +97,28 @@ void ChatPageHandler::GetActionList(GetActionListCallback callback) { } void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { + auto convertToString = [action_type](chat::mojom::ActionType action) { + switch (action_type) { + case chat::mojom::ActionType::SUMMARIZE_PAGE: return "SUMMARIZE_PAGE"; + case chat::mojom::ActionType::EXPLAIN: return "EXPLAIN"; + case chat::mojom::ActionType::TRANSLATE: return "TRANSLATE"; + case chat::mojom::ActionType::DRAFT_SOCIAL_MEDIA_POST: return "DRAFT_SOCIAL_MEDIA_POST"; + case chat::mojom::ActionType::FACT_CHECK: return "FACT_CHECK"; + case chat::mojom::ActionType::QUERY: return "QUERY"; + default: return "NONE"; + } + }; + if (page_.is_bound()) { + LOG(INFO) << action_type; chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; - response->result = "Mock Result"; + response->result =convertToString(action_type); page_->OnSubmitActionResponse(response.Clone()); } } + void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { } From 8ce872514f0dbe5978fd572b8835266813af3bdf Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 4 Nov 2024 01:50:59 +0800 Subject: [PATCH 11/34] Handling chat completion API - WIP --- chrome/browser/ui/BUILD.gn | 12 +- .../side_panel/chat/api/api_request_helper.cc | 536 ++++++++++++++++++ .../side_panel/chat/api/api_request_helper.h | 279 +++++++++ .../chat/api/completion_api_client.cc | 165 ++++++ .../chat/api/completion_api_client.h | 60 ++ .../side_panel/chat/chat_page_handler.cc | 14 +- 6 files changed, 1048 insertions(+), 18 deletions(-) create mode 100644 chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc create mode 100644 chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h create mode 100644 chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc create mode 100644 chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index 63580a48de0fa3..4fd268f6a3f0e2 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -1565,6 +1565,12 @@ static_library("ui") { "webui/side_panel/bookmarks/bookmarks_page_handler.h", "webui/side_panel/bookmarks/bookmarks_side_panel_ui.cc", "webui/side_panel/bookmarks/bookmarks_side_panel_ui.h", + "webui/side_panel/chat/api/api_request_helper.cc", + "webui/side_panel/chat/api/api_request_helper.h", + "webui/side_panel/chat/chat_page_handler.cc", + "webui/side_panel/chat/chat_page_handler.h", + "webui/side_panel/chat/chat_ui.cc", + "webui/side_panel/chat/chat_ui.h", "webui/side_panel/companion/companion_page_handler.cc", "webui/side_panel/companion/companion_page_handler.h", "webui/side_panel/companion/companion_side_panel_untrusted_ui.cc", @@ -1595,10 +1601,6 @@ static_library("ui") { "webui/side_panel/reading_list/reading_list_page_handler.h", "webui/side_panel/reading_list/reading_list_ui.cc", "webui/side_panel/reading_list/reading_list_ui.h", - "webui/side_panel/chat/chat_ui.cc", - "webui/side_panel/chat/chat_ui.h", - "webui/side_panel/chat/chat_page_handler.cc", - "webui/side_panel/chat/chat_page_handler.h", "webui/suggest_internals/suggest_internals_handler.cc", "webui/suggest_internals/suggest_internals_handler.h", "webui/suggest_internals/suggest_internals_ui.cc", @@ -1747,10 +1749,10 @@ static_library("ui") { "//chrome/browser/ui/views/side_panel:side_panel_enums", "//chrome/browser/ui/views/toolbar", "//chrome/browser/ui/webui/cr_components/theme_color_picker", - "//chrome/browser/ui/webui/side_panel/chat:mojo_bindings", "//chrome/browser/ui/webui/searchbox", "//chrome/browser/ui/webui/settings", "//chrome/browser/ui/webui/settings:impl", + "//chrome/browser/ui/webui/side_panel/chat:mojo_bindings", "//chrome/browser/ui/webui/signin", "//chrome/browser/ui/webui/signin:login", "//chrome/browser/ui/webui/signin:login_impl", diff --git a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc new file mode 100644 index 00000000000000..8c1cc7705ee888 --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc @@ -0,0 +1,536 @@ +#include "api_request_helper.h" + +#include +#include +#include + +#include "base/check.h" +#include "base/check_op.h" +#include "base/debug/alias.h" +#include "base/debug/dump_without_crashing.h" +#include "base/functional/callback_helpers.h" +#include "base/json/json_reader.h" +#include "base/json/json_writer.h" +#include "base/memory/raw_ptr.h" +#include "base/metrics/histogram_functions.h" +#include "base/ranges/algorithm.h" +#include "base/rust_buildflags.h" +#include "base/strings/string_split.h" +#include "base/task/thread_pool.h" +#include "base/time/time.h" +#include "base/timer/elapsed_timer.h" +#include "base/trace_event/trace_event.h" +#include "net/base/load_flags.h" +#include "net/http/http_status_code.h" +#include "services/data_decoder/public/cpp/data_decoder.h" +#include "services/network/public/cpp/resource_request.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/network/public/mojom/url_response_head.mojom.h" + +namespace api_request_helper { + +namespace { + +const unsigned int kRetriesCountOnNetworkChange = 1; + +scoped_refptr MakeDecoderTaskRunner() { + return base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::USER_VISIBLE, + base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN}); +} + +APIRequestResult ToAPIRequestResult( + std::unique_ptr loader) { + auto response_code = -1; + auto error_code = loader->NetError(); + auto final_url = loader->GetFinalURL(); + base::flat_map headers; + if (loader->ResponseInfo()) { + auto headers_list = loader->ResponseInfo()->headers; + if (headers_list) { + response_code = headers_list->response_code(); + DVLOG(1) << "Response code: " << response_code; + size_t header_iter = 0; + std::string key; + std::string value; + while (headers_list->EnumerateHeaderLines(&header_iter, &key, &value)) { + key = base::ToLowerASCII(key); + headers[key] = value; + DVLOG(2) << "< " << key << ": " << value; + } + } + } + + return APIRequestResult(response_code, base::Value(), std::move(headers), + error_code, final_url); +} + +// A helper class to measure performance of the callback. +class ScopedPerfTracker { + public: + explicit ScopedPerfTracker(const char* uma_name) : uma_name_(uma_name) {} + + ~ScopedPerfTracker() { + if (timer_.is_supported()) { + base::UmaHistogramMediumTimes(uma_name_, timer_.Elapsed()); + } + } + + private: + raw_ptr uma_name_; + base::ElapsedThreadTimer timer_; +}; + +} // namespace + +APIRequestResult::APIRequestResult() = default; + +APIRequestResult::APIRequestResult( + int response_code, + base::Value value_body, + base::flat_map headers, + int error_code, + GURL final_url) + : response_code_(response_code), + value_body_(std::move(value_body)), + headers_(std::move(headers)), + error_code_(error_code), + final_url_(std::move(final_url)) {} + +APIRequestResult::APIRequestResult(APIRequestResult&&) = default; + +APIRequestResult& APIRequestResult::operator=(APIRequestResult&&) = default; + +APIRequestResult::~APIRequestResult() = default; + +bool APIRequestResult::operator==(const APIRequestResult& other) const { + auto tied = [](auto& v) { + return std::tie(v.response_code_, v.value_body_, v.headers_, v.error_code_, + v.final_url_); + }; + return tied(*this) == tied(other); +} + +bool APIRequestResult::operator!=(const APIRequestResult& other) const { + return !(*this == other); +} + +bool APIRequestResult::Is2XXResponseCode() const { + return response_code_ >= 200 && response_code_ <= 299; +} + +bool APIRequestResult::IsResponseCodeValid() const { + return response_code_ >= 100 && response_code_ <= 599; +} + +base::Value APIRequestResult::TakeBody() { + CHECK(!body_consumed_); + body_consumed_ = true; + return std::move(value_body_); +} + +std::string APIRequestResult::SerializeBodyToString() const { + if (value_body_.is_none()) { + return std::string(); + } + std::string safe_json; + if (!base::JSONWriter::Write(value_body_, &safe_json)) { + VLOG(1) << "Response validation error: Encoding error"; + } + + return safe_json; +} + +APIRequestHelper::APIRequestHelper( + net::NetworkTrafficAnnotationTag annotation_tag, + scoped_refptr url_loader_factory) + : annotation_tag_(annotation_tag), + url_loader_factory_(url_loader_factory), + task_runner_(MakeDecoderTaskRunner()) {} + +APIRequestHelper::~APIRequestHelper() = default; + +APIRequestHelper::Ticket APIRequestHelper::Request( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + ResultCallback callback, + const base::flat_map& headers, + const APIRequestOptions& request_options, + ResponseConversionCallback conversion_callback) { + auto iter = CreateRequestURLLoaderHandler( + method, url, payload, payload_content_type, request_options, headers, + std::move(callback)); + auto* handler = iter->get(); + + if (request_options.max_body_size == -1u) { + handler->url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( + url_loader_factory_.get(), + base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnResponse, + handler->GetWeakPtr(), std::move(conversion_callback))); + } else { + handler->url_loader_->DownloadToString( + url_loader_factory_.get(), + base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnResponse, + handler->GetWeakPtr(), std::move(conversion_callback)), + request_options.max_body_size); + } + + return iter; +} + +APIRequestHelper::Ticket APIRequestHelper::RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options) { + return RequestSSE(method, url, payload, payload_content_type, + std::move(data_received_callback), + std::move(result_callback), headers, request_options, + base::NullCallback()); +} + +APIRequestHelper::Ticket APIRequestHelper::RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options, + ResponseStartedCallback response_started_callback) { + auto iter = CreateRequestURLLoaderHandler( + method, url, payload, payload_content_type, request_options, headers, + std::move(result_callback)); + auto* handler = iter->get(); + + // Set streaming data callback + handler->data_received_callback_ = std::move(data_received_callback); + + handler->response_started_callback_ = std::move(response_started_callback); + + handler->url_loader_->DownloadAsStream(url_loader_factory_.get(), handler); + return iter; +} + +void APIRequestHelper::DeleteAndSendResult(Ticket iter, + ResultCallback callback, + APIRequestResult result) { + Cancel(iter); + std::move(callback).Run(std::move(result)); +} + +void APIRequestHelper::Cancel(const Ticket& ticket) { + url_loaders_.erase(ticket); +} + +void APIRequestHelper::CancelAll() { + url_loaders_.clear(); +} + +APIRequestHelper::Ticket APIRequestHelper::CreateURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + bool auto_retry_on_network_change, + bool enable_cache, + bool allow_http_error_result, + const base::flat_map& headers) { + auto request = std::make_unique(); + request->url = url; + request->load_flags = net::LOAD_DO_NOT_SAVE_COOKIES; + if (!enable_cache) { + request->load_flags = + request->load_flags | net::LOAD_BYPASS_CACHE | net::LOAD_DISABLE_CACHE; + } + + request->credentials_mode = network::mojom::CredentialsMode::kOmit; + if (!method.empty()) { + request->method = method; + } + + DVLOG(4) << method << " " << url.spec(); + + if (!headers.empty()) { + for (auto entry : headers) { + DVLOG(4) << "> " << entry.first << ": " << entry.second; + request->headers.SetHeader(entry.first, entry.second); + } + } + + if (!payload.empty()) { + DVLOG(4) << "Payload type " << payload_content_type << ":"; + DVLOG(4) << payload; + } + + auto url_loader = + network::SimpleURLLoader::Create(std::move(request), annotation_tag_); + if (!payload.empty()) { + url_loader->AttachStringForUpload(payload, payload_content_type); + } + url_loader->SetRetryOptions( + kRetriesCountOnNetworkChange, + auto_retry_on_network_change + ? network::SimpleURLLoader::RetryMode::RETRY_ON_NETWORK_CHANGE + : network::SimpleURLLoader::RetryMode::RETRY_NEVER); + url_loader->SetAllowHttpErrorResults(allow_http_error_result); + + auto loader_wrapper_handler = + std::make_unique(this, task_runner_); + loader_wrapper_handler->RegisterURLLoader(std::move(url_loader)); + + auto iter = url_loaders_.insert(url_loaders_.begin(), + std::move(loader_wrapper_handler)); + + return iter; +} + +APIRequestHelper::Ticket APIRequestHelper::CreateRequestURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + const APIRequestOptions& request_options, + const base::flat_map& headers, + ResultCallback result_callback) { + auto iter = CreateURLLoaderHandler( + method, url, payload, payload_content_type, + request_options.auto_retry_on_network_change, + request_options.enable_cache, true /* allow_http_error_result*/, headers); + auto* handler = iter->get(); + + handler->result_callback_ = base::BindOnce( + &APIRequestHelper::DeleteAndSendResult, weak_ptr_factory_.GetWeakPtr(), + iter, std::move(result_callback)); + if (request_options.timeout) { + handler->url_loader_->SetTimeoutDuration(request_options.timeout.value()); + } + return iter; +} + +APIRequestHelper::URLLoaderHandler::URLLoaderHandler( + APIRequestHelper* api_request_helper, + scoped_refptr task_runner) + : api_request_helper_(api_request_helper), + task_runner_(std::move(task_runner)) {} + +APIRequestHelper::URLLoaderHandler::~URLLoaderHandler() = default; + +void APIRequestHelper::URLLoaderHandler::RegisterURLLoader( + std::unique_ptr loader) { + url_loader_ = std::move(loader); + + auto on_response_start = + [](base::WeakPtr handler, + const GURL& final_url, + const network::mojom::URLResponseHead& response_head) { + if (handler) { + if (response_head.mime_type == "text/event-stream") { + handler->is_sse_ = true; + } + if (handler->response_started_callback_) { + std::move(handler->response_started_callback_) + .Run(final_url.spec(), response_head.content_length); + } + } + }; + + url_loader_->SetOnResponseStartedCallback(base::BindOnce( + std::move(on_response_start), weak_ptr_factory_.GetWeakPtr())); +} + +base::WeakPtr +APIRequestHelper::URLLoaderHandler::GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); +} + +void APIRequestHelper::URLLoaderHandler::send_sse_data_for_testing( + std::string_view string_piece, + bool is_sse, + DataReceivedCallback callback) { + is_sse_ = is_sse; + data_received_callback_ = std::move(callback); + OnDataReceived(string_piece, base::BindOnce([]() {})); +} + +void APIRequestHelper::URLLoaderHandler::ParseJsonImpl( + std::string json, + base::OnceCallback callback) { + if (!data_decoder_) { + VLOG(1) << "Creating DataDecoder for APIRequestHelper"; + data_decoder_ = std::make_unique(); + } + + data_decoder_->ParseJson(json, std::move(callback)); +} + +void APIRequestHelper::URLLoaderHandler::OnDataReceived( + std::string_view string_piece, + base::OnceClosure resume) { + DVLOG(2) << "[[" << __func__ << "]]" << " Chunk received"; + if (is_sse_) { + ParseSSE(string_piece); + } else { + DVLOG(4) << "Chunk content: \n" << string_piece; + ScopedPerfTracker tracker("APIRequestHelper.OnDataReceivedNoSSE"); + data_received_callback_.Run(base::Value(string_piece)); + } + std::move(resume).Run(); +} + +void APIRequestHelper::URLLoaderHandler::OnComplete(bool success) { + DCHECK(result_callback_); + VLOG(1) << "[[" << __func__ << "]]" << " Response completed\n"; + + request_is_finished_ = true; + + // Delete now or when decoding operations are complete + MaybeSendResult(); +} + +void APIRequestHelper::URLLoaderHandler::OnRetry( + base::OnceClosure start_retry) { + std::move(start_retry).Run(); +} + +void APIRequestHelper::URLLoaderHandler::OnResponse( + ResponseConversionCallback conversion_callback, + const std::unique_ptr response_body) { + VLOG(1) << "[[" << __func__ << "]]" << " Response received\n"; + DCHECK(result_callback_); + + DCHECK_EQ(current_decoding_operation_count_, 0); + APIRequestResult result = ToAPIRequestResult(std::move(url_loader_)); + + if (!response_body) { + std::move(result_callback_).Run(std::move(result)); + return; + } + auto& raw_body = *response_body; + if (conversion_callback) { + auto converted_body = std::move(conversion_callback).Run(raw_body); + if (!converted_body) { + result.response_code_ = 422; + std::move(result_callback_).Run(std::move(result)); + return; + } + raw_body = converted_body.value(); + } + + ParseJsonImpl( + std::move(raw_body), + base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnParseJsonResponse, + GetWeakPtr(), std::move(result))); +} + +void APIRequestHelper::URLLoaderHandler::OnParseJsonResponse( + APIRequestResult result, + ValueOrError result_value) { + ScopedPerfTracker tracker("APIRequestHelper.ProcessResultOnUI"); + if (!result_value.has_value()) { + VLOG(1) << "Response validation error:" << result_value.error(); + if (result_value.error().starts_with("trailing comma")) { + DEBUG_ALIAS_FOR_GURL(url_alias, result.final_url()); + DEBUG_ALIAS_FOR_CSTR(result_str, result_value.error().c_str(), 1024); + base::debug::DumpWithoutCrashing(); + } + std::move(result_callback_).Run(std::move(result)); + return; + } + if (!result_value.value().is_dict() && !result_value.value().is_list()) { + VLOG(1) << "Response validation error: Invalid top-level type"; + std::move(result_callback_).Run(std::move(result)); + return; + } + + VLOG(2) << "Response validation successful"; + result.value_body_ = std::move(result_value.value()); + std::move(result_callback_).Run(std::move(result)); +} + +void APIRequestHelper::URLLoaderHandler::MaybeSendResult() { + DCHECK_LE(0, current_decoding_operation_count_); + const bool decoding_is_complete = (current_decoding_operation_count_ == 0); + + if (request_is_finished_ && decoding_is_complete) { + std::move(result_callback_).Run(ToAPIRequestResult(std::move(url_loader_))); + } else if (decoding_is_complete) { + VLOG(3) << "Did not run URLLoaderHandler completion handler, still have " + << current_decoding_operation_count_ + << " decoding operations in progress, waiting for them to" + << " complete..."; + } +} + +void APIRequestHelper::URLLoaderHandler::ParseSSE( + std::string_view string_piece) { + // New chunks should only be received before the request is completed + DCHECK(!request_is_finished_); + // We split the string into multiple chunks because there are cases where + // multiple chunks are received in a single call. + std::vector stream_data = base::SplitStringPiece( + string_piece, "\r\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY); + + // Remove SSE events that don't look like JSON - could be string or [DONE] + // message. + // TODO(@nullhook): Parse both JSON and string values. The below currently + // only identifies JSON values. + static constexpr char kDataPrefix[] = "data: {"; + std::erase_if(stream_data, [](std::string_view item) { + DVLOG(3) << "Received chunk: " << item; + if (!base::StartsWith(item, kDataPrefix)) { + // This is useful to log in case an API starts + // coming back with unknown data type in some + // scenarios. + VLOG(1) << "Data did not start with SSE prefix"; + return true; + } + return false; + }); + + // Keep track of number of in-progress data decoding operations + // so that we can know if any are still in-progress when the request + // completes. + current_decoding_operation_count_ += stream_data.size(); + + for (const auto& data : stream_data) { + auto json = data.substr(strlen(kDataPrefix) - 1); + auto on_json_parsed = + [](base::WeakPtr handler, + ValueOrError result) { + DVLOG(2) << "Chunk parsed"; + if (!handler) { + return; + } + ScopedPerfTracker tracker("APIRequestHelper.ParseSSECallback"); + handler->current_decoding_operation_count_--; + DCHECK(handler->data_received_callback_); + handler->data_received_callback_.Run(std::move(result)); + // Parsing is potentially the last operation for |URLLoaderHandler|. + handler->MaybeSendResult(); + }; + + DVLOG(2) << "Going to call ParseJsonImpl"; + ParseJsonImpl(std::string(json), + base::BindOnce(std::move(on_json_parsed), + weak_ptr_factory_.GetWeakPtr())); + } +} + +void APIRequestHelper::SetUrlLoaderFactoryForTesting( + scoped_refptr url_loader_factory) { + url_loader_factory_ = std::move(url_loader_factory); +} + +void SanitizeAndParseJson(std::string json, + base::OnceCallback callback) { + data_decoder::DataDecoder::ParseJsonIsolated(json, std::move(callback)); +} +} // namespace api_request_helper diff --git a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h new file mode 100644 index 00000000000000..534050c1522664 --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h @@ -0,0 +1,279 @@ +#ifndef CHROMIUM_API_REQUEST_HELPER_H +#define CHROMIUM_API_REQUEST_HELPER_H + +#include +#include +#include +#include +#include + +#include "base/containers/flat_map.h" +#include "base/files/file_path.h" +#include "base/functional/callback.h" +#include "base/functional/callback_forward.h" +#include "base/functional/callback_helpers.h" +#include "base/time/time.h" +#include "base/types/expected.h" +#include "base/values.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "services/network/public/cpp/simple_url_loader.h" +#include "services/network/public/cpp/simple_url_loader_stream_consumer.h" +#include "url/gurl.h" + +namespace network { +class SharedURLLoaderFactory; +} // namespace network + +namespace data_decoder { +class DataDecoder; +} + +namespace api_request_helper { + +class APIRequestResult { + public: + APIRequestResult(); + APIRequestResult(int response_code, + base::Value value_body, + base::flat_map headers, + int error_code, + GURL final_url); + APIRequestResult(const APIRequestResult&) = delete; + APIRequestResult& operator=(const APIRequestResult&) = delete; + APIRequestResult(APIRequestResult&&); + APIRequestResult& operator=(APIRequestResult&&); + ~APIRequestResult(); + + bool operator==(const APIRequestResult& other) const; + bool operator!=(const APIRequestResult& other) const; + + bool Is2XXResponseCode() const; + bool IsResponseCodeValid() const; + + // HTTP response code. + int response_code() const { return response_code_; } + + // Extract the sanitized response as base::Value. + base::Value TakeBody(); + + // Returns the sanitized response as base::Value. + // Note: don't clone large responses, use TakeBody() instead. + const base::Value& value_body() const { return value_body_; } + + // Serialize the sanitized response and returns it as string. + // Note: use TakeBody()/value_body() instead where possible. + std::string SerializeBodyToString() const; + + // HTTP response headers. + const base::flat_map& headers() const { + return headers_; + } + // `net::Error` code + int error_code() const { return error_code_; } + // Actual url requested. May differ from original request url in case of + // redirects happened. + GURL final_url() const { return final_url_; } + + private: + friend class APIRequestHelper; + + int response_code_ = -1; + base::Value value_body_; + base::flat_map headers_; + int error_code_ = -1; + GURL final_url_; + bool body_consumed_ = false; +}; + +struct APIRequestOptions { + bool auto_retry_on_network_change = false; + bool enable_cache = false; + size_t max_body_size = -1u; + std::optional timeout; +}; + +using ValueOrError = base::expected; + +class APIRequestHelper { + public: + using DataReceivedCallback = + base::RepeatingCallback; + using ResultCallback = base::OnceCallback; + using ResponseStartedCallback = + base::OnceCallback; + using ResponseConversionCallback = + base::OnceCallback( + const std::string& raw_response)>; + + class URLLoaderHandler : public network::SimpleURLLoaderStreamConsumer { + public: + URLLoaderHandler(APIRequestHelper* api_request_helper, + scoped_refptr task_runner); + + ~URLLoaderHandler() override; + + URLLoaderHandler(const URLLoaderHandler&) = delete; + + URLLoaderHandler& operator=(const URLLoaderHandler&) = delete; + + void RegisterURLLoader(std::unique_ptr loader); + + void SetResultCallback(ResultCallback result_callback); + + base::WeakPtr GetWeakPtr(); + + void send_sse_data_for_testing(std::string_view string_piece, + bool is_sse, + DataReceivedCallback callback); + + private: + friend class APIRequestHelper; + + void ParseJsonImpl(std::string json, + base::OnceCallback callback); + + // Run completion callback if there are no operations in progress. + // If Cancel is needed even if url or data operations are in progress, + // then call |APIRequestHelper::Cancel|. + void MaybeSendResult(); + + void ParseSSE(std::string_view string_piece); + + // network::SimpleURLLoaderStreamConsumer implementation: + void OnDataReceived(std::string_view string_piece, + base::OnceClosure resume) override; + + void OnComplete(bool success) override; + + void OnRetry(base::OnceClosure start_retry) override; + + // This is used for one shot responses + void OnResponse(ResponseConversionCallback conversion_callback, + const std::unique_ptr response_body); + + // Decode one shot responses + void OnParseJsonResponse(APIRequestResult result, + ValueOrError result_value); + + std::unique_ptr url_loader_; + raw_ptr api_request_helper_; + + DataReceivedCallback data_received_callback_; + ResponseStartedCallback response_started_callback_; + ResultCallback result_callback_; + ResponseConversionCallback conversion_callback_; + + bool is_sse_ = false; + + // To ensure ordered processing of stream chunks, we create our own + // instance of DataDecoder per request. This avoids the issue + // of unordered chunks that can occur when calling the static function, + // which creates a new instance of the process for each call. By using a + // single instance of the parser, we can reuse it for consecutive calls. + std::unique_ptr data_decoder_; + // Keep track of number of in-progress data decoding operations + // so that we can know if any are still in-progress when the request + // completes. + int current_decoding_operation_count_ = 0; + bool request_is_finished_ = false; + + const scoped_refptr task_runner_; + + base::WeakPtrFactory weak_ptr_factory_{this}; + }; + + using URLLoaderHandlerList = std::list>; + using Ticket = std::list>::iterator; + + APIRequestHelper( + net::NetworkTrafficAnnotationTag annotation_tag, + scoped_refptr url_loader_factory); + + virtual ~APIRequestHelper(); + + // Each response is expected in json format and will be validated through + // JsonSanitizer. In cases where json contains values that are not supported + // by the standard base/json parser it is necessary to convert such values + // into string before validating the response. For these purposes + // conversion_callback is added which receives raw response and can perform + // necessary conversions. + Ticket Request( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + ResultCallback callback, + const base::flat_map& headers = {}, + const APIRequestOptions& request_options = {}, + ResponseConversionCallback conversion_callback = base::NullCallback()); + + virtual Ticket RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options); + + virtual Ticket RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options, + ResponseStartedCallback response_started_callback); + + void Cancel(const Ticket& ticket); + + void CancelAll(); + + void SetUrlLoaderFactoryForTesting( + scoped_refptr url_loader_factory); + + private: + APIRequestHelper(const APIRequestHelper&) = delete; + + APIRequestHelper& operator=(const APIRequestHelper&) = delete; + + APIRequestHelper::Ticket CreateURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + bool auto_retry_on_network_change, + bool enable_cache, + bool allow_http_error_result, + const base::flat_map& headers); + + APIRequestHelper::Ticket CreateRequestURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + const APIRequestOptions& request_options, + const base::flat_map& headers, + ResultCallback result_callback); + + void DeleteAndSendResult(Ticket iter, + ResultCallback callback, + APIRequestResult result); + + net::NetworkTrafficAnnotationTag annotation_tag_; + URLLoaderHandlerList url_loaders_; + scoped_refptr url_loader_factory_; + const scoped_refptr task_runner_; + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + +void SanitizeAndParseJson(std::string json, + base::OnceCallback callback); + +} // namespace api_request_helper + +#endif // CHROMIUM_API_REQUEST_HELPER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc new file mode 100644 index 00000000000000..707c5b819d0064 --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc @@ -0,0 +1,165 @@ +#include "completion_api_client.h" + +#include + +#include +#include +#include + +#include "base/containers/flat_set.h" +#include "base/functional/bind.h" +#include "base/functional/callback_helpers.h" +#include "base/json/json_writer.h" +#include "base/no_destructor.h" +#include "base/strings/strcat.h" +#include "base/values.h" +#include "net/http/http_status_code.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/network/public/cpp/simple_url_loader.h" +#include "url/gurl.h" + +namespace ai_chat { +namespace { + +constexpr char kHttpMethod[] = "POST"; + +net::NetworkTrafficAnnotationTag GetNetworkTrafficAnnotationTag() { + return net::DefineNetworkTrafficAnnotation("ai_chat", R"( + semantics { + sender: "AI Chat" + description: + "This is used to communicate with Yep Chat api." + trigger: + "Triggered by user sending a prompt." + data: + "Will generate a text that attempts to match the user gave it" + destination: WEBSITE + } + policy { + cookies_allowed: NO + policy_exception_justification: + "Not implemented." + } + )"); +} + +std::string CreateJSONRequestBody(const std::vector& prompt) { + base::Value::Dict dict; + + base::Value::List all_stop_sequences; + std::vector stop_sequences = {"\nUser:", "\nAssistant:"}; + for (const auto& item : stop_sequences) { + all_stop_sequences.Append(item); + } + + dict.Set("stream", true); + dict.Set("max_tokens", 1280); + dict.Set("top_p", 0.7); + dict.Set("temperature", 0.6); + dict.Set("model", "Mixtral-8x7B-Instruct-v0.1"); + dict.Set("stop_sequences", std::move(stop_sequences)); + + base::Value::List prompt_messages; + for (const auto& item : prompt) { + base::Value::Dict message; + message.Set("content", prompt); + message.Set("role", "user"); + dict.Set("messages", std::move(message)); + } + + std::string json; + base::JSONWriter::Write(&dict, &json); + return json; +} + +} // namespace + +CompletionApiClient::CompletionApiClient( + scoped_refptr url_loader_factory) + : api_request_helper_(GetNetworkTrafficAnnotationTag(), + std::move(url_loader_factory)) {} + +CompletionApiClient::~CompletionApiClient() = default; + +void CompletionApiClient::QueryPrompt( + const std::string& prompt, + GenerationCompletedCallback data_completed_callback, + GenerationDataCallback + data_received_callback /* = base::NullCallback() */) { + + GURL api_url{base::StrCat({url::kHttpsScheme, url::kStandardSchemeSeparator, + "api.yep.com", "/", "v1/chat/completions"})}; + DCHECK(api_url.is_valid()) << "Invalid API Url: " << api_url.spec(); + + base::flat_map headers; + headers.emplace("Accept", "text/event-stream"); + + auto on_received = base::BindRepeating( + &CompletionApiClient::OnQueryDataReceived, weak_ptr_factory_.GetWeakPtr(), + std::move(data_received_callback)); + auto on_complete = base::BindOnce(&CompletionApiClient::OnQueryCompleted, + weak_ptr_factory_.GetWeakPtr(), credential, + std::move(data_completed_callback)); + + api_request_helper_.RequestSSE(kHttpMethod, api_url, request_body, + "application/json", std::move(on_received), + std::move(on_complete), headers, {}); +} + +void CompletionApiClient::ClearAllQueries() { + api_request_helper_.CancelAll(); +} + +void CompletionApiClient::OnQueryDataReceived( + GenerationDataCallback callback, + base::expected result) { + if (!result.has_value() || !result->is_dict()) { + return; + } + + // This client only supports completion events + const std::string* completion = result->GetDict().FindString("completion"); + if (completion) { + callback.Run(std::move(*completion)); + } +} + +void CompletionApiClient::OnQueryCompleted( + std::optional credential, + GenerationCompletedCallback callback, + APIRequestResult result) { + const bool success = result.Is2XXResponseCode(); + // Handle successful request + if (success) { + std::string completion = ""; + // We're checking for a value body in case for non-streaming API results. + if (result.value_body().is_dict()) { + const std::string* value = + result.value_body().GetDict().FindString("completion"); + if (value) { + // Trimming necessary for Llama 2 which prepends responses with a " ". + completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); + } + } + + std::move(callback).Run(base::ok(std::move(completion))); + return; + } + + + // Handle error + mojom::APIError error; + + if (net::HTTP_TOO_MANY_REQUESTS == result.response_code()) { + error = mojom::APIError::RateLimitReached; + } else if (net::HTTP_REQUEST_ENTITY_TOO_LARGE == result.response_code()) { + error = mojom::APIError::ContextLimitReached; + } else { + error = mojom::APIError::ConnectionIssue; + } + + std::move(callback).Run(base::unexpected(std::move(error))); +} + +} // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h new file mode 100644 index 00000000000000..dd7ac5d82b61ba --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h @@ -0,0 +1,60 @@ +#ifndef CHROMIUM_COMPLETION_API_CLIENT_H +#define CHROMIUM_COMPLETION_API_CLIENT_H + +#include +#include +#include +#include +#include + +#include "base/containers/flat_set.h" +#include "base/functional/callback_forward.h" +#include "base/memory/weak_ptr.h" +#include "base/types/expected.h" + +namespace network { +class SharedURLLoaderFactory; +} // namespace network + +namespace ai_chat { + +using api_request_helper::APIRequestResult; + +class CompletionApiClient { + public: + using GenerationResult = base::expected; + using GenerationDataCallback = base::RepeatingCallback; + using GenerationCompletedCallback = + base::OnceCallback; + + CompletionApiClient( + scoped_refptr url_loader_factory); + + CompletionApiClient(const CompletionApiClient&) = delete; + CompletionApiClient& operator=(const CompletionApiClient&) = delete; + virtual ~CompletionApiClient(); + + // This function queries both types of APIs: SSE and non-SSE. + // In non-SSE cases, only the data_completed_callback will be triggered. + virtual void QueryPrompt( + const std::string& prompt, + GenerationCompletedCallback data_completed_callback, + GenerationDataCallback data_received_callback = base::NullCallback()); + // Clears all in-progress requests + void ClearAllQueries(); + + private: + void OnQueryDataReceived(GenerationDataCallback callback, + base::expected result); + void OnQueryCompleted(std::optional credential, + GenerationCompletedCallback callback, + APIRequestResult result); + + const base::flat_set stop_sequences_; + api_request_helper::APIRequestHelper api_request_helper_; + + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + +} // namespace ai_chat +#endif // CHROMIUM_COMPLETION_API_CLIENT_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 0df9a2adce5823..2c7d18bd36a900 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -97,23 +97,11 @@ void ChatPageHandler::GetActionList(GetActionListCallback callback) { } void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { - auto convertToString = [action_type](chat::mojom::ActionType action) { - switch (action_type) { - case chat::mojom::ActionType::SUMMARIZE_PAGE: return "SUMMARIZE_PAGE"; - case chat::mojom::ActionType::EXPLAIN: return "EXPLAIN"; - case chat::mojom::ActionType::TRANSLATE: return "TRANSLATE"; - case chat::mojom::ActionType::DRAFT_SOCIAL_MEDIA_POST: return "DRAFT_SOCIAL_MEDIA_POST"; - case chat::mojom::ActionType::FACT_CHECK: return "FACT_CHECK"; - case chat::mojom::ActionType::QUERY: return "QUERY"; - default: return "NONE"; - } - }; - if (page_.is_bound()) { LOG(INFO) << action_type; chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; - response->result =convertToString(action_type); + response->result = "MOCK Result"; page_->OnSubmitActionResponse(response.Clone()); } } From c5baa007caa11eedf43d051f3a0925c6bde4902e Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 4 Nov 2024 08:06:12 +0800 Subject: [PATCH 12/34] Implement completion api request (parsing stream data and pumping them to WebUI layer in progress) --- chrome/browser/ui/BUILD.gn | 2 + .../side_panel/chat/api/api_request_helper.cc | 965 +++++++++--------- .../side_panel/chat/api/api_request_helper.h | 482 ++++----- .../chat/api/completion_api_client.cc | 205 ++-- .../chat/api/completion_api_client.h | 73 +- .../side_panel/chat/chat_page_handler.cc | 131 ++- .../webui/side_panel/chat/chat_page_handler.h | 46 +- 7 files changed, 971 insertions(+), 933 deletions(-) diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index 4fd268f6a3f0e2..bca3d2514f32fb 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -1567,6 +1567,8 @@ static_library("ui") { "webui/side_panel/bookmarks/bookmarks_side_panel_ui.h", "webui/side_panel/chat/api/api_request_helper.cc", "webui/side_panel/chat/api/api_request_helper.h", + "webui/side_panel/chat/api/completion_api_client.cc", + "webui/side_panel/chat/api/completion_api_client.h", "webui/side_panel/chat/chat_page_handler.cc", "webui/side_panel/chat/chat_page_handler.h", "webui/side_panel/chat/chat_ui.cc", diff --git a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc index 8c1cc7705ee888..f33b1d562f044f 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc @@ -29,508 +29,489 @@ namespace api_request_helper { -namespace { - -const unsigned int kRetriesCountOnNetworkChange = 1; - -scoped_refptr MakeDecoderTaskRunner() { - return base::ThreadPool::CreateSequencedTaskRunner( - {base::TaskPriority::USER_VISIBLE, - base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN}); -} - -APIRequestResult ToAPIRequestResult( - std::unique_ptr loader) { - auto response_code = -1; - auto error_code = loader->NetError(); - auto final_url = loader->GetFinalURL(); - base::flat_map headers; - if (loader->ResponseInfo()) { - auto headers_list = loader->ResponseInfo()->headers; - if (headers_list) { - response_code = headers_list->response_code(); - DVLOG(1) << "Response code: " << response_code; - size_t header_iter = 0; - std::string key; - std::string value; - while (headers_list->EnumerateHeaderLines(&header_iter, &key, &value)) { - key = base::ToLowerASCII(key); - headers[key] = value; - DVLOG(2) << "< " << key << ": " << value; - } + namespace { + + const unsigned int kRetriesCountOnNetworkChange = 1; + + scoped_refptr MakeDecoderTaskRunner() { + return base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::USER_VISIBLE, + base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN}); + } + + APIRequestResult ToAPIRequestResult( + std::unique_ptr loader) { + auto response_code = -1; + auto error_code = loader->NetError(); + auto final_url = loader->GetFinalURL(); + base::flat_map headers; + if (loader->ResponseInfo()) { + auto headers_list = loader->ResponseInfo()->headers; + if (headers_list) { + response_code = headers_list->response_code(); + DVLOG(1) << "Response code: " << response_code; + size_t header_iter = 0; + std::string key; + std::string value; + while (headers_list->EnumerateHeaderLines(&header_iter, &key, &value)) { + key = base::ToLowerASCII(key); + headers[key] = value; + DVLOG(2) << "< " << key << ": " << value; + } + } + } + + return APIRequestResult(response_code, base::Value(), std::move(headers), + error_code, final_url); + } + + } // namespace + + APIRequestResult::APIRequestResult() = default; + + APIRequestResult::APIRequestResult( + int response_code, + base::Value value_body, + base::flat_map headers, + int error_code, + GURL final_url) + : response_code_(response_code), + value_body_(std::move(value_body)), + headers_(std::move(headers)), + error_code_(error_code), + final_url_(std::move(final_url)) {} + + APIRequestResult::APIRequestResult(APIRequestResult&&) = default; + + APIRequestResult& APIRequestResult::operator=(APIRequestResult&&) = default; + + APIRequestResult::~APIRequestResult() = default; + + bool APIRequestResult::operator==(const APIRequestResult& other) const { + auto tied = [](auto& v) { + return std::tie(v.response_code_, v.value_body_, v.headers_, v.error_code_, + v.final_url_); + }; + return tied(*this) == tied(other); + } + + bool APIRequestResult::operator!=(const APIRequestResult& other) const { + return !(*this == other); + } + + bool APIRequestResult::Is2XXResponseCode() const { + return response_code_ >= 200 && response_code_ <= 299; + } + + bool APIRequestResult::IsResponseCodeValid() const { + return response_code_ >= 100 && response_code_ <= 599; + } + + base::Value APIRequestResult::TakeBody() { + CHECK(!body_consumed_); + body_consumed_ = true; + return std::move(value_body_); + } + + std::string APIRequestResult::SerializeBodyToString() const { + if (value_body_.is_none()) { + return std::string(); + } + std::string safe_json; + if (!base::JSONWriter::Write(value_body_, &safe_json)) { + VLOG(1) << "Response validation error: Encoding error"; + } + + return safe_json; + } + + APIRequestHelper::APIRequestHelper( + net::NetworkTrafficAnnotationTag annotation_tag, + scoped_refptr url_loader_factory) + : annotation_tag_(annotation_tag), + url_loader_factory_(url_loader_factory), + task_runner_(MakeDecoderTaskRunner()) {} + + APIRequestHelper::~APIRequestHelper() = default; + + APIRequestHelper::Ticket APIRequestHelper::Request( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + ResultCallback callback, + const base::flat_map& headers, + const APIRequestOptions& request_options, + ResponseConversionCallback conversion_callback) { + auto iter = CreateRequestURLLoaderHandler( + method, url, payload, payload_content_type, request_options, headers, + std::move(callback)); + auto* handler = iter->get(); + + if (request_options.max_body_size == -1u) { + handler->url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( + url_loader_factory_.get(), + base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnResponse, + handler->GetWeakPtr(), std::move(conversion_callback))); + } else { + handler->url_loader_->DownloadToString( + url_loader_factory_.get(), + base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnResponse, + handler->GetWeakPtr(), std::move(conversion_callback)), + request_options.max_body_size); + } + + return iter; + } + + APIRequestHelper::Ticket APIRequestHelper::RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options) { + return RequestSSE(method, url, payload, payload_content_type, + std::move(data_received_callback), + std::move(result_callback), headers, request_options, + base::NullCallback()); + } + + APIRequestHelper::Ticket APIRequestHelper::RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options, + ResponseStartedCallback response_started_callback) { + auto iter = CreateRequestURLLoaderHandler( + method, url, payload, payload_content_type, request_options, headers, + std::move(result_callback)); + auto* handler = iter->get(); + + // Set streaming data callback + handler->data_received_callback_ = std::move(data_received_callback); + + handler->response_started_callback_ = std::move(response_started_callback); + + handler->url_loader_->DownloadAsStream(url_loader_factory_.get(), handler); + return iter; + } + + void APIRequestHelper::DeleteAndSendResult(Ticket iter, + ResultCallback callback, + APIRequestResult result) { + Cancel(iter); + std::move(callback).Run(std::move(result)); + } + + void APIRequestHelper::Cancel(const Ticket& ticket) { + url_loaders_.erase(ticket); + } + + void APIRequestHelper::CancelAll() { + url_loaders_.clear(); + } + + APIRequestHelper::Ticket APIRequestHelper::CreateURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + bool auto_retry_on_network_change, + bool enable_cache, + bool allow_http_error_result, + const base::flat_map& headers) { + auto request = std::make_unique(); + request->url = url; + request->load_flags = net::LOAD_DO_NOT_SAVE_COOKIES; + if (!enable_cache) { + request->load_flags = + request->load_flags | net::LOAD_BYPASS_CACHE | net::LOAD_DISABLE_CACHE; + } + + request->credentials_mode = network::mojom::CredentialsMode::kOmit; + if (!method.empty()) { + request->method = method; + } + + DVLOG(4) << method << " " << url.spec(); + + if (!headers.empty()) { + for (auto entry : headers) { + DVLOG(4) << "> " << entry.first << ": " << entry.second; + request->headers.SetHeader(entry.first, entry.second); + } + } + + if (!payload.empty()) { + DVLOG(4) << "Payload type " << payload_content_type << ":"; + DVLOG(4) << payload; + } + + auto url_loader = + network::SimpleURLLoader::Create(std::move(request), annotation_tag_); + if (!payload.empty()) { + url_loader->AttachStringForUpload(payload, payload_content_type); + } + url_loader->SetRetryOptions( + kRetriesCountOnNetworkChange, + auto_retry_on_network_change + ? network::SimpleURLLoader::RetryMode::RETRY_ON_NETWORK_CHANGE + : network::SimpleURLLoader::RetryMode::RETRY_NEVER); + url_loader->SetAllowHttpErrorResults(allow_http_error_result); + + auto loader_wrapper_handler = + std::make_unique(this, task_runner_); + loader_wrapper_handler->RegisterURLLoader(std::move(url_loader)); + + auto iter = url_loaders_.insert(url_loaders_.begin(), + std::move(loader_wrapper_handler)); + + return iter; } - } - return APIRequestResult(response_code, base::Value(), std::move(headers), - error_code, final_url); -} + APIRequestHelper::Ticket APIRequestHelper::CreateRequestURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + const APIRequestOptions& request_options, + const base::flat_map& headers, + ResultCallback result_callback) { + auto iter = CreateURLLoaderHandler( + method, url, payload, payload_content_type, + request_options.auto_retry_on_network_change, + request_options.enable_cache, true /* allow_http_error_result*/, headers); + auto* handler = iter->get(); + + handler->result_callback_ = base::BindOnce( + &APIRequestHelper::DeleteAndSendResult, weak_ptr_factory_.GetWeakPtr(), + iter, std::move(result_callback)); + if (request_options.timeout) { + handler->url_loader_->SetTimeoutDuration(request_options.timeout.value()); + } + return iter; + } -// A helper class to measure performance of the callback. -class ScopedPerfTracker { - public: - explicit ScopedPerfTracker(const char* uma_name) : uma_name_(uma_name) {} + APIRequestHelper::URLLoaderHandler::URLLoaderHandler( + APIRequestHelper* api_request_helper, + scoped_refptr task_runner) + : api_request_helper_(api_request_helper), + task_runner_(std::move(task_runner)) {} + + APIRequestHelper::URLLoaderHandler::~URLLoaderHandler() = default; + + void APIRequestHelper::URLLoaderHandler::RegisterURLLoader( + std::unique_ptr loader) { + url_loader_ = std::move(loader); + + auto on_response_start = + [](base::WeakPtr handler, + const GURL& final_url, + const network::mojom::URLResponseHead& response_head) { + if (handler) { + if (response_head.mime_type == "text/event-stream") { + handler->is_sse_ = true; + } + if (handler->response_started_callback_) { + std::move(handler->response_started_callback_) + .Run(final_url.spec(), response_head.content_length); + } + } + }; + + url_loader_->SetOnResponseStartedCallback(base::BindOnce( + std::move(on_response_start), weak_ptr_factory_.GetWeakPtr())); + } - ~ScopedPerfTracker() { - if (timer_.is_supported()) { - base::UmaHistogramMediumTimes(uma_name_, timer_.Elapsed()); + base::WeakPtr + APIRequestHelper::URLLoaderHandler::GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); } - } - - private: - raw_ptr uma_name_; - base::ElapsedThreadTimer timer_; -}; - -} // namespace - -APIRequestResult::APIRequestResult() = default; - -APIRequestResult::APIRequestResult( - int response_code, - base::Value value_body, - base::flat_map headers, - int error_code, - GURL final_url) - : response_code_(response_code), - value_body_(std::move(value_body)), - headers_(std::move(headers)), - error_code_(error_code), - final_url_(std::move(final_url)) {} - -APIRequestResult::APIRequestResult(APIRequestResult&&) = default; - -APIRequestResult& APIRequestResult::operator=(APIRequestResult&&) = default; - -APIRequestResult::~APIRequestResult() = default; - -bool APIRequestResult::operator==(const APIRequestResult& other) const { - auto tied = [](auto& v) { - return std::tie(v.response_code_, v.value_body_, v.headers_, v.error_code_, - v.final_url_); - }; - return tied(*this) == tied(other); -} - -bool APIRequestResult::operator!=(const APIRequestResult& other) const { - return !(*this == other); -} - -bool APIRequestResult::Is2XXResponseCode() const { - return response_code_ >= 200 && response_code_ <= 299; -} - -bool APIRequestResult::IsResponseCodeValid() const { - return response_code_ >= 100 && response_code_ <= 599; -} - -base::Value APIRequestResult::TakeBody() { - CHECK(!body_consumed_); - body_consumed_ = true; - return std::move(value_body_); -} - -std::string APIRequestResult::SerializeBodyToString() const { - if (value_body_.is_none()) { - return std::string(); - } - std::string safe_json; - if (!base::JSONWriter::Write(value_body_, &safe_json)) { - VLOG(1) << "Response validation error: Encoding error"; - } - - return safe_json; -} - -APIRequestHelper::APIRequestHelper( - net::NetworkTrafficAnnotationTag annotation_tag, - scoped_refptr url_loader_factory) - : annotation_tag_(annotation_tag), - url_loader_factory_(url_loader_factory), - task_runner_(MakeDecoderTaskRunner()) {} - -APIRequestHelper::~APIRequestHelper() = default; - -APIRequestHelper::Ticket APIRequestHelper::Request( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - ResultCallback callback, - const base::flat_map& headers, - const APIRequestOptions& request_options, - ResponseConversionCallback conversion_callback) { - auto iter = CreateRequestURLLoaderHandler( - method, url, payload, payload_content_type, request_options, headers, - std::move(callback)); - auto* handler = iter->get(); - - if (request_options.max_body_size == -1u) { - handler->url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( - url_loader_factory_.get(), - base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnResponse, - handler->GetWeakPtr(), std::move(conversion_callback))); - } else { - handler->url_loader_->DownloadToString( - url_loader_factory_.get(), - base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnResponse, - handler->GetWeakPtr(), std::move(conversion_callback)), - request_options.max_body_size); - } - - return iter; -} - -APIRequestHelper::Ticket APIRequestHelper::RequestSSE( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - DataReceivedCallback data_received_callback, - ResultCallback result_callback, - const base::flat_map& headers, - const APIRequestOptions& request_options) { - return RequestSSE(method, url, payload, payload_content_type, - std::move(data_received_callback), - std::move(result_callback), headers, request_options, - base::NullCallback()); -} - -APIRequestHelper::Ticket APIRequestHelper::RequestSSE( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - DataReceivedCallback data_received_callback, - ResultCallback result_callback, - const base::flat_map& headers, - const APIRequestOptions& request_options, - ResponseStartedCallback response_started_callback) { - auto iter = CreateRequestURLLoaderHandler( - method, url, payload, payload_content_type, request_options, headers, - std::move(result_callback)); - auto* handler = iter->get(); - - // Set streaming data callback - handler->data_received_callback_ = std::move(data_received_callback); - - handler->response_started_callback_ = std::move(response_started_callback); - - handler->url_loader_->DownloadAsStream(url_loader_factory_.get(), handler); - return iter; -} - -void APIRequestHelper::DeleteAndSendResult(Ticket iter, - ResultCallback callback, - APIRequestResult result) { - Cancel(iter); - std::move(callback).Run(std::move(result)); -} - -void APIRequestHelper::Cancel(const Ticket& ticket) { - url_loaders_.erase(ticket); -} - -void APIRequestHelper::CancelAll() { - url_loaders_.clear(); -} - -APIRequestHelper::Ticket APIRequestHelper::CreateURLLoaderHandler( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - bool auto_retry_on_network_change, - bool enable_cache, - bool allow_http_error_result, - const base::flat_map& headers) { - auto request = std::make_unique(); - request->url = url; - request->load_flags = net::LOAD_DO_NOT_SAVE_COOKIES; - if (!enable_cache) { - request->load_flags = - request->load_flags | net::LOAD_BYPASS_CACHE | net::LOAD_DISABLE_CACHE; - } - - request->credentials_mode = network::mojom::CredentialsMode::kOmit; - if (!method.empty()) { - request->method = method; - } - - DVLOG(4) << method << " " << url.spec(); - - if (!headers.empty()) { - for (auto entry : headers) { - DVLOG(4) << "> " << entry.first << ": " << entry.second; - request->headers.SetHeader(entry.first, entry.second); + + void APIRequestHelper::URLLoaderHandler::send_sse_data_for_testing( + std::string_view string_piece, + bool is_sse, + DataReceivedCallback callback) { + is_sse_ = is_sse; + data_received_callback_ = std::move(callback); + OnDataReceived(string_piece, base::BindOnce([]() {})); + } + + void APIRequestHelper::URLLoaderHandler::ParseJsonImpl( + std::string json, + base::OnceCallback callback) { + if (!data_decoder_) { + VLOG(1) << "Creating DataDecoder for APIRequestHelper"; + data_decoder_ = std::make_unique(); + } + + data_decoder_->ParseJson(json, std::move(callback)); } - } - - if (!payload.empty()) { - DVLOG(4) << "Payload type " << payload_content_type << ":"; - DVLOG(4) << payload; - } - - auto url_loader = - network::SimpleURLLoader::Create(std::move(request), annotation_tag_); - if (!payload.empty()) { - url_loader->AttachStringForUpload(payload, payload_content_type); - } - url_loader->SetRetryOptions( - kRetriesCountOnNetworkChange, - auto_retry_on_network_change - ? network::SimpleURLLoader::RetryMode::RETRY_ON_NETWORK_CHANGE - : network::SimpleURLLoader::RetryMode::RETRY_NEVER); - url_loader->SetAllowHttpErrorResults(allow_http_error_result); - - auto loader_wrapper_handler = - std::make_unique(this, task_runner_); - loader_wrapper_handler->RegisterURLLoader(std::move(url_loader)); - - auto iter = url_loaders_.insert(url_loaders_.begin(), - std::move(loader_wrapper_handler)); - - return iter; -} - -APIRequestHelper::Ticket APIRequestHelper::CreateRequestURLLoaderHandler( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - const APIRequestOptions& request_options, - const base::flat_map& headers, - ResultCallback result_callback) { - auto iter = CreateURLLoaderHandler( - method, url, payload, payload_content_type, - request_options.auto_retry_on_network_change, - request_options.enable_cache, true /* allow_http_error_result*/, headers); - auto* handler = iter->get(); - - handler->result_callback_ = base::BindOnce( - &APIRequestHelper::DeleteAndSendResult, weak_ptr_factory_.GetWeakPtr(), - iter, std::move(result_callback)); - if (request_options.timeout) { - handler->url_loader_->SetTimeoutDuration(request_options.timeout.value()); - } - return iter; -} - -APIRequestHelper::URLLoaderHandler::URLLoaderHandler( - APIRequestHelper* api_request_helper, - scoped_refptr task_runner) - : api_request_helper_(api_request_helper), - task_runner_(std::move(task_runner)) {} - -APIRequestHelper::URLLoaderHandler::~URLLoaderHandler() = default; - -void APIRequestHelper::URLLoaderHandler::RegisterURLLoader( - std::unique_ptr loader) { - url_loader_ = std::move(loader); - - auto on_response_start = - [](base::WeakPtr handler, - const GURL& final_url, - const network::mojom::URLResponseHead& response_head) { - if (handler) { - if (response_head.mime_type == "text/event-stream") { - handler->is_sse_ = true; - } - if (handler->response_started_callback_) { - std::move(handler->response_started_callback_) - .Run(final_url.spec(), response_head.content_length); - } + + void APIRequestHelper::URLLoaderHandler::OnDataReceived( + std::string_view string_piece, + base::OnceClosure resume) { + DVLOG(2) << "[[" << __func__ << "]]" << " Chunk received"; + if (is_sse_) { + ParseSSE(string_piece); + } else { + DVLOG(4) << "Chunk content: \n" << string_piece; + data_received_callback_.Run(base::Value(string_piece)); } - }; - - url_loader_->SetOnResponseStartedCallback(base::BindOnce( - std::move(on_response_start), weak_ptr_factory_.GetWeakPtr())); -} - -base::WeakPtr -APIRequestHelper::URLLoaderHandler::GetWeakPtr() { - return weak_ptr_factory_.GetWeakPtr(); -} - -void APIRequestHelper::URLLoaderHandler::send_sse_data_for_testing( - std::string_view string_piece, - bool is_sse, - DataReceivedCallback callback) { - is_sse_ = is_sse; - data_received_callback_ = std::move(callback); - OnDataReceived(string_piece, base::BindOnce([]() {})); -} - -void APIRequestHelper::URLLoaderHandler::ParseJsonImpl( - std::string json, - base::OnceCallback callback) { - if (!data_decoder_) { - VLOG(1) << "Creating DataDecoder for APIRequestHelper"; - data_decoder_ = std::make_unique(); - } - - data_decoder_->ParseJson(json, std::move(callback)); -} - -void APIRequestHelper::URLLoaderHandler::OnDataReceived( - std::string_view string_piece, - base::OnceClosure resume) { - DVLOG(2) << "[[" << __func__ << "]]" << " Chunk received"; - if (is_sse_) { - ParseSSE(string_piece); - } else { - DVLOG(4) << "Chunk content: \n" << string_piece; - ScopedPerfTracker tracker("APIRequestHelper.OnDataReceivedNoSSE"); - data_received_callback_.Run(base::Value(string_piece)); - } - std::move(resume).Run(); -} - -void APIRequestHelper::URLLoaderHandler::OnComplete(bool success) { - DCHECK(result_callback_); - VLOG(1) << "[[" << __func__ << "]]" << " Response completed\n"; - - request_is_finished_ = true; - - // Delete now or when decoding operations are complete - MaybeSendResult(); -} - -void APIRequestHelper::URLLoaderHandler::OnRetry( - base::OnceClosure start_retry) { - std::move(start_retry).Run(); -} - -void APIRequestHelper::URLLoaderHandler::OnResponse( - ResponseConversionCallback conversion_callback, - const std::unique_ptr response_body) { - VLOG(1) << "[[" << __func__ << "]]" << " Response received\n"; - DCHECK(result_callback_); - - DCHECK_EQ(current_decoding_operation_count_, 0); - APIRequestResult result = ToAPIRequestResult(std::move(url_loader_)); - - if (!response_body) { - std::move(result_callback_).Run(std::move(result)); - return; - } - auto& raw_body = *response_body; - if (conversion_callback) { - auto converted_body = std::move(conversion_callback).Run(raw_body); - if (!converted_body) { - result.response_code_ = 422; - std::move(result_callback_).Run(std::move(result)); - return; + std::move(resume).Run(); } - raw_body = converted_body.value(); - } - - ParseJsonImpl( - std::move(raw_body), - base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnParseJsonResponse, - GetWeakPtr(), std::move(result))); -} - -void APIRequestHelper::URLLoaderHandler::OnParseJsonResponse( - APIRequestResult result, - ValueOrError result_value) { - ScopedPerfTracker tracker("APIRequestHelper.ProcessResultOnUI"); - if (!result_value.has_value()) { - VLOG(1) << "Response validation error:" << result_value.error(); - if (result_value.error().starts_with("trailing comma")) { - DEBUG_ALIAS_FOR_GURL(url_alias, result.final_url()); - DEBUG_ALIAS_FOR_CSTR(result_str, result_value.error().c_str(), 1024); - base::debug::DumpWithoutCrashing(); + + void APIRequestHelper::URLLoaderHandler::OnComplete(bool success) { + DCHECK(result_callback_); + VLOG(1) << "[[" << __func__ << "]]" << " Response completed\n"; + + request_is_finished_ = true; + + // Delete now or when decoding operations are complete + MaybeSendResult(); + } + + void APIRequestHelper::URLLoaderHandler::OnRetry( + base::OnceClosure start_retry) { + std::move(start_retry).Run(); } - std::move(result_callback_).Run(std::move(result)); - return; - } - if (!result_value.value().is_dict() && !result_value.value().is_list()) { - VLOG(1) << "Response validation error: Invalid top-level type"; - std::move(result_callback_).Run(std::move(result)); - return; - } - - VLOG(2) << "Response validation successful"; - result.value_body_ = std::move(result_value.value()); - std::move(result_callback_).Run(std::move(result)); -} - -void APIRequestHelper::URLLoaderHandler::MaybeSendResult() { - DCHECK_LE(0, current_decoding_operation_count_); - const bool decoding_is_complete = (current_decoding_operation_count_ == 0); - - if (request_is_finished_ && decoding_is_complete) { - std::move(result_callback_).Run(ToAPIRequestResult(std::move(url_loader_))); - } else if (decoding_is_complete) { - VLOG(3) << "Did not run URLLoaderHandler completion handler, still have " - << current_decoding_operation_count_ - << " decoding operations in progress, waiting for them to" - << " complete..."; - } -} - -void APIRequestHelper::URLLoaderHandler::ParseSSE( - std::string_view string_piece) { - // New chunks should only be received before the request is completed - DCHECK(!request_is_finished_); - // We split the string into multiple chunks because there are cases where - // multiple chunks are received in a single call. - std::vector stream_data = base::SplitStringPiece( - string_piece, "\r\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY); - - // Remove SSE events that don't look like JSON - could be string or [DONE] - // message. - // TODO(@nullhook): Parse both JSON and string values. The below currently - // only identifies JSON values. - static constexpr char kDataPrefix[] = "data: {"; - std::erase_if(stream_data, [](std::string_view item) { - DVLOG(3) << "Received chunk: " << item; - if (!base::StartsWith(item, kDataPrefix)) { - // This is useful to log in case an API starts - // coming back with unknown data type in some - // scenarios. - VLOG(1) << "Data did not start with SSE prefix"; - return true; + + void APIRequestHelper::URLLoaderHandler::OnResponse( + ResponseConversionCallback conversion_callback, + const std::unique_ptr response_body) { + VLOG(1) << "[[" << __func__ << "]]" << " Response received\n"; + DCHECK(result_callback_); + + DCHECK_EQ(current_decoding_operation_count_, 0); + APIRequestResult result = ToAPIRequestResult(std::move(url_loader_)); + + if (!response_body) { + std::move(result_callback_).Run(std::move(result)); + return; + } + auto& raw_body = *response_body; + if (conversion_callback) { + auto converted_body = std::move(conversion_callback).Run(raw_body); + if (!converted_body) { + result.response_code_ = 422; + std::move(result_callback_).Run(std::move(result)); + return; + } + raw_body = converted_body.value(); + } + + ParseJsonImpl( + std::move(raw_body), + base::BindOnce(&APIRequestHelper::URLLoaderHandler::OnParseJsonResponse, + GetWeakPtr(), std::move(result))); } - return false; - }); - - // Keep track of number of in-progress data decoding operations - // so that we can know if any are still in-progress when the request - // completes. - current_decoding_operation_count_ += stream_data.size(); - - for (const auto& data : stream_data) { - auto json = data.substr(strlen(kDataPrefix) - 1); - auto on_json_parsed = - [](base::WeakPtr handler, - ValueOrError result) { - DVLOG(2) << "Chunk parsed"; - if (!handler) { + + void APIRequestHelper::URLLoaderHandler::OnParseJsonResponse( + APIRequestResult result, + ValueOrError result_value) { + if (!result_value.has_value()) { + VLOG(1) << "Response validation error:" << result_value.error(); + if (result_value.error().starts_with("trailing comma")) { + DEBUG_ALIAS_FOR_GURL(url_alias, result.final_url()); + DEBUG_ALIAS_FOR_CSTR(result_str, result_value.error().c_str(), 1024); + base::debug::DumpWithoutCrashing(); + } + std::move(result_callback_).Run(std::move(result)); return; - } - ScopedPerfTracker tracker("APIRequestHelper.ParseSSECallback"); - handler->current_decoding_operation_count_--; - DCHECK(handler->data_received_callback_); - handler->data_received_callback_.Run(std::move(result)); - // Parsing is potentially the last operation for |URLLoaderHandler|. - handler->MaybeSendResult(); - }; + } + if (!result_value.value().is_dict() && !result_value.value().is_list()) { + VLOG(1) << "Response validation error: Invalid top-level type"; + std::move(result_callback_).Run(std::move(result)); + return; + } - DVLOG(2) << "Going to call ParseJsonImpl"; - ParseJsonImpl(std::string(json), - base::BindOnce(std::move(on_json_parsed), - weak_ptr_factory_.GetWeakPtr())); - } -} - -void APIRequestHelper::SetUrlLoaderFactoryForTesting( - scoped_refptr url_loader_factory) { - url_loader_factory_ = std::move(url_loader_factory); -} - -void SanitizeAndParseJson(std::string json, - base::OnceCallback callback) { - data_decoder::DataDecoder::ParseJsonIsolated(json, std::move(callback)); -} + VLOG(2) << "Response validation successful"; + result.value_body_ = std::move(result_value.value()); + std::move(result_callback_).Run(std::move(result)); + } + + void APIRequestHelper::URLLoaderHandler::MaybeSendResult() { + DCHECK_LE(0, current_decoding_operation_count_); + const bool decoding_is_complete = (current_decoding_operation_count_ == 0); + + if (request_is_finished_ && decoding_is_complete) { + std::move(result_callback_).Run(ToAPIRequestResult(std::move(url_loader_))); + } else if (decoding_is_complete) { + VLOG(3) << "Did not run URLLoaderHandler completion handler, still have " + << current_decoding_operation_count_ + << " decoding operations in progress, waiting for them to" + << " complete..."; + } + } + + void APIRequestHelper::URLLoaderHandler::ParseSSE( + std::string_view string_piece) { + // New chunks should only be received before the request is completed + DCHECK(!request_is_finished_); + // We split the string into multiple chunks because there are cases where + // multiple chunks are received in a single call. + std::vector stream_data = base::SplitStringPiece( + string_piece, "\r\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY); + + // Remove SSE events that don't look like JSON - could be string or [DONE] + // message. + // TODO(@nullhook): Parse both JSON and string values. The below currently + // only identifies JSON values. + static constexpr char kDataPrefix[] = "data: {"; + std::erase_if(stream_data, [](std::string_view item) { + DVLOG(3) << "Received chunk: " << item; + if (!base::StartsWith(item, kDataPrefix)) { + // This is useful to log in case an API starts + // coming back with unknown data type in some + // scenarios. + VLOG(1) << "Data did not start with SSE prefix"; + return true; + } + return false; + }); + + // Keep track of number of in-progress data decoding operations + // so that we can know if any are still in-progress when the request + // completes. + current_decoding_operation_count_ += stream_data.size(); + + for (const auto& data : stream_data) { + auto json = data.substr(strlen(kDataPrefix) - 1); + auto on_json_parsed = + [](base::WeakPtr handler, + ValueOrError result) { + DVLOG(2) << "Chunk parsed"; + if (!handler) { + return; + } + handler->current_decoding_operation_count_--; + DCHECK(handler->data_received_callback_); + handler->data_received_callback_.Run(std::move(result)); + // Parsing is potentially the last operation for |URLLoaderHandler|. + handler->MaybeSendResult(); + }; + + DVLOG(2) << "Going to call ParseJsonImpl"; + ParseJsonImpl(std::string(json), + base::BindOnce(std::move(on_json_parsed), + weak_ptr_factory_.GetWeakPtr())); + } + } + + void APIRequestHelper::SetUrlLoaderFactoryForTesting( + scoped_refptr url_loader_factory) { + url_loader_factory_ = std::move(url_loader_factory); + } + + void SanitizeAndParseJson(std::string json, + base::OnceCallback callback) { + data_decoder::DataDecoder::ParseJsonIsolated(json, std::move(callback)); + } } // namespace api_request_helper diff --git a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h index 534050c1522664..b35f2442773b96 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h +++ b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h @@ -21,258 +21,258 @@ #include "url/gurl.h" namespace network { -class SharedURLLoaderFactory; + class SharedURLLoaderFactory; } // namespace network namespace data_decoder { -class DataDecoder; + class DataDecoder; } namespace api_request_helper { -class APIRequestResult { - public: - APIRequestResult(); - APIRequestResult(int response_code, - base::Value value_body, - base::flat_map headers, - int error_code, - GURL final_url); - APIRequestResult(const APIRequestResult&) = delete; - APIRequestResult& operator=(const APIRequestResult&) = delete; - APIRequestResult(APIRequestResult&&); - APIRequestResult& operator=(APIRequestResult&&); - ~APIRequestResult(); - - bool operator==(const APIRequestResult& other) const; - bool operator!=(const APIRequestResult& other) const; - - bool Is2XXResponseCode() const; - bool IsResponseCodeValid() const; - - // HTTP response code. - int response_code() const { return response_code_; } - - // Extract the sanitized response as base::Value. - base::Value TakeBody(); + class APIRequestResult { + public: + APIRequestResult(); + APIRequestResult(int response_code, + base::Value value_body, + base::flat_map headers, + int error_code, + GURL final_url); + APIRequestResult(const APIRequestResult&) = delete; + APIRequestResult& operator=(const APIRequestResult&) = delete; + APIRequestResult(APIRequestResult&&); + APIRequestResult& operator=(APIRequestResult&&); + ~APIRequestResult(); + + bool operator==(const APIRequestResult& other) const; + bool operator!=(const APIRequestResult& other) const; + + bool Is2XXResponseCode() const; + bool IsResponseCodeValid() const; + + // HTTP response code. + int response_code() const { return response_code_; } + + // Extract the sanitized response as base::Value. + base::Value TakeBody(); - // Returns the sanitized response as base::Value. - // Note: don't clone large responses, use TakeBody() instead. - const base::Value& value_body() const { return value_body_; } + // Returns the sanitized response as base::Value. + // Note: don't clone large responses, use TakeBody() instead. + const base::Value& value_body() const { return value_body_; } - // Serialize the sanitized response and returns it as string. - // Note: use TakeBody()/value_body() instead where possible. - std::string SerializeBodyToString() const; + // Serialize the sanitized response and returns it as string. + // Note: use TakeBody()/value_body() instead where possible. + std::string SerializeBodyToString() const; - // HTTP response headers. - const base::flat_map& headers() const { - return headers_; - } - // `net::Error` code - int error_code() const { return error_code_; } - // Actual url requested. May differ from original request url in case of - // redirects happened. - GURL final_url() const { return final_url_; } - - private: - friend class APIRequestHelper; - - int response_code_ = -1; - base::Value value_body_; - base::flat_map headers_; - int error_code_ = -1; - GURL final_url_; - bool body_consumed_ = false; -}; - -struct APIRequestOptions { - bool auto_retry_on_network_change = false; - bool enable_cache = false; - size_t max_body_size = -1u; - std::optional timeout; -}; + // HTTP response headers. + const base::flat_map& headers() const { + return headers_; + } + // `net::Error` code + int error_code() const { return error_code_; } + // Actual url requested. May differ from original request url in case of + // redirects happened. + GURL final_url() const { return final_url_; } + + private: + friend class APIRequestHelper; + + int response_code_ = -1; + base::Value value_body_; + base::flat_map headers_; + int error_code_ = -1; + GURL final_url_; + bool body_consumed_ = false; + }; + + struct APIRequestOptions { + bool auto_retry_on_network_change = false; + bool enable_cache = false; + size_t max_body_size = -1u; + std::optional timeout; + }; -using ValueOrError = base::expected; - -class APIRequestHelper { - public: - using DataReceivedCallback = - base::RepeatingCallback; - using ResultCallback = base::OnceCallback; - using ResponseStartedCallback = - base::OnceCallback; - using ResponseConversionCallback = - base::OnceCallback( - const std::string& raw_response)>; - - class URLLoaderHandler : public network::SimpleURLLoaderStreamConsumer { - public: - URLLoaderHandler(APIRequestHelper* api_request_helper, - scoped_refptr task_runner); - - ~URLLoaderHandler() override; - - URLLoaderHandler(const URLLoaderHandler&) = delete; - - URLLoaderHandler& operator=(const URLLoaderHandler&) = delete; - - void RegisterURLLoader(std::unique_ptr loader); - - void SetResultCallback(ResultCallback result_callback); - - base::WeakPtr GetWeakPtr(); - - void send_sse_data_for_testing(std::string_view string_piece, - bool is_sse, - DataReceivedCallback callback); - - private: - friend class APIRequestHelper; - - void ParseJsonImpl(std::string json, - base::OnceCallback callback); - - // Run completion callback if there are no operations in progress. - // If Cancel is needed even if url or data operations are in progress, - // then call |APIRequestHelper::Cancel|. - void MaybeSendResult(); - - void ParseSSE(std::string_view string_piece); - - // network::SimpleURLLoaderStreamConsumer implementation: - void OnDataReceived(std::string_view string_piece, - base::OnceClosure resume) override; - - void OnComplete(bool success) override; - - void OnRetry(base::OnceClosure start_retry) override; - - // This is used for one shot responses - void OnResponse(ResponseConversionCallback conversion_callback, - const std::unique_ptr response_body); - - // Decode one shot responses - void OnParseJsonResponse(APIRequestResult result, - ValueOrError result_value); - - std::unique_ptr url_loader_; - raw_ptr api_request_helper_; - - DataReceivedCallback data_received_callback_; - ResponseStartedCallback response_started_callback_; - ResultCallback result_callback_; - ResponseConversionCallback conversion_callback_; - - bool is_sse_ = false; - - // To ensure ordered processing of stream chunks, we create our own - // instance of DataDecoder per request. This avoids the issue - // of unordered chunks that can occur when calling the static function, - // which creates a new instance of the process for each call. By using a - // single instance of the parser, we can reuse it for consecutive calls. - std::unique_ptr data_decoder_; - // Keep track of number of in-progress data decoding operations - // so that we can know if any are still in-progress when the request - // completes. - int current_decoding_operation_count_ = 0; - bool request_is_finished_ = false; - - const scoped_refptr task_runner_; - - base::WeakPtrFactory weak_ptr_factory_{this}; - }; - - using URLLoaderHandlerList = std::list>; - using Ticket = std::list>::iterator; - - APIRequestHelper( - net::NetworkTrafficAnnotationTag annotation_tag, - scoped_refptr url_loader_factory); - - virtual ~APIRequestHelper(); - - // Each response is expected in json format and will be validated through - // JsonSanitizer. In cases where json contains values that are not supported - // by the standard base/json parser it is necessary to convert such values - // into string before validating the response. For these purposes - // conversion_callback is added which receives raw response and can perform - // necessary conversions. - Ticket Request( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - ResultCallback callback, - const base::flat_map& headers = {}, - const APIRequestOptions& request_options = {}, - ResponseConversionCallback conversion_callback = base::NullCallback()); - - virtual Ticket RequestSSE( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - DataReceivedCallback data_received_callback, - ResultCallback result_callback, - const base::flat_map& headers, - const APIRequestOptions& request_options); - - virtual Ticket RequestSSE( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - DataReceivedCallback data_received_callback, - ResultCallback result_callback, - const base::flat_map& headers, - const APIRequestOptions& request_options, - ResponseStartedCallback response_started_callback); - - void Cancel(const Ticket& ticket); - - void CancelAll(); - - void SetUrlLoaderFactoryForTesting( - scoped_refptr url_loader_factory); - - private: - APIRequestHelper(const APIRequestHelper&) = delete; - - APIRequestHelper& operator=(const APIRequestHelper&) = delete; - - APIRequestHelper::Ticket CreateURLLoaderHandler( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - bool auto_retry_on_network_change, - bool enable_cache, - bool allow_http_error_result, - const base::flat_map& headers); - - APIRequestHelper::Ticket CreateRequestURLLoaderHandler( - const std::string& method, - const GURL& url, - const std::string& payload, - const std::string& payload_content_type, - const APIRequestOptions& request_options, - const base::flat_map& headers, - ResultCallback result_callback); - - void DeleteAndSendResult(Ticket iter, - ResultCallback callback, - APIRequestResult result); - - net::NetworkTrafficAnnotationTag annotation_tag_; - URLLoaderHandlerList url_loaders_; - scoped_refptr url_loader_factory_; - const scoped_refptr task_runner_; - base::WeakPtrFactory weak_ptr_factory_{this}; -}; - -void SanitizeAndParseJson(std::string json, - base::OnceCallback callback); + using ValueOrError = base::expected; + + class APIRequestHelper { + public: + using DataReceivedCallback = + base::RepeatingCallback; + using ResultCallback = base::OnceCallback; + using ResponseStartedCallback = + base::OnceCallback; + using ResponseConversionCallback = + base::OnceCallback( + const std::string& raw_response)>; + + class URLLoaderHandler : public network::SimpleURLLoaderStreamConsumer { + public: + URLLoaderHandler(APIRequestHelper* api_request_helper, + scoped_refptr task_runner); + + ~URLLoaderHandler() override; + + URLLoaderHandler(const URLLoaderHandler&) = delete; + + URLLoaderHandler& operator=(const URLLoaderHandler&) = delete; + + void RegisterURLLoader(std::unique_ptr loader); + + void SetResultCallback(ResultCallback result_callback); + + base::WeakPtr GetWeakPtr(); + + void send_sse_data_for_testing(std::string_view string_piece, + bool is_sse, + DataReceivedCallback callback); + + private: + friend class APIRequestHelper; + + void ParseJsonImpl(std::string json, + base::OnceCallback callback); + + // Run completion callback if there are no operations in progress. + // If Cancel is needed even if url or data operations are in progress, + // then call |APIRequestHelper::Cancel|. + void MaybeSendResult(); + + void ParseSSE(std::string_view string_piece); + + // network::SimpleURLLoaderStreamConsumer implementation: + void OnDataReceived(std::string_view string_piece, + base::OnceClosure resume) override; + + void OnComplete(bool success) override; + + void OnRetry(base::OnceClosure start_retry) override; + + // This is used for one shot responses + void OnResponse(ResponseConversionCallback conversion_callback, + const std::unique_ptr response_body); + + // Decode one shot responses + void OnParseJsonResponse(APIRequestResult result, + ValueOrError result_value); + + std::unique_ptr url_loader_; + raw_ptr api_request_helper_; + + DataReceivedCallback data_received_callback_; + ResponseStartedCallback response_started_callback_; + ResultCallback result_callback_; + ResponseConversionCallback conversion_callback_; + + bool is_sse_ = false; + + // To ensure ordered processing of stream chunks, we create our own + // instance of DataDecoder per request. This avoids the issue + // of unordered chunks that can occur when calling the static function, + // which creates a new instance of the process for each call. By using a + // single instance of the parser, we can reuse it for consecutive calls. + std::unique_ptr data_decoder_; + // Keep track of number of in-progress data decoding operations + // so that we can know if any are still in-progress when the request + // completes. + int current_decoding_operation_count_ = 0; + bool request_is_finished_ = false; + + const scoped_refptr task_runner_; + + base::WeakPtrFactory weak_ptr_factory_{this}; + }; + + using URLLoaderHandlerList = std::list>; + using Ticket = std::list>::iterator; + + APIRequestHelper( + net::NetworkTrafficAnnotationTag annotation_tag, + scoped_refptr url_loader_factory); + + virtual ~APIRequestHelper(); + + // Each response is expected in json format and will be validated through + // JsonSanitizer. In cases where json contains values that are not supported + // by the standard base/json parser it is necessary to convert such values + // into string before validating the response. For these purposes + // conversion_callback is added which receives raw response and can perform + // necessary conversions. + Ticket Request( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + ResultCallback callback, + const base::flat_map& headers = {}, + const APIRequestOptions& request_options = {}, + ResponseConversionCallback conversion_callback = base::NullCallback()); + + virtual Ticket RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options); + + virtual Ticket RequestSSE( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + DataReceivedCallback data_received_callback, + ResultCallback result_callback, + const base::flat_map& headers, + const APIRequestOptions& request_options, + ResponseStartedCallback response_started_callback); + + void Cancel(const Ticket& ticket); + + void CancelAll(); + + void SetUrlLoaderFactoryForTesting( + scoped_refptr url_loader_factory); + + private: + APIRequestHelper(const APIRequestHelper&) = delete; + + APIRequestHelper& operator=(const APIRequestHelper&) = delete; + + APIRequestHelper::Ticket CreateURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + bool auto_retry_on_network_change, + bool enable_cache, + bool allow_http_error_result, + const base::flat_map& headers); + + APIRequestHelper::Ticket CreateRequestURLLoaderHandler( + const std::string& method, + const GURL& url, + const std::string& payload, + const std::string& payload_content_type, + const APIRequestOptions& request_options, + const base::flat_map& headers, + ResultCallback result_callback); + + void DeleteAndSendResult(Ticket iter, + ResultCallback callback, + APIRequestResult result); + + net::NetworkTrafficAnnotationTag annotation_tag_; + URLLoaderHandlerList url_loaders_; + scoped_refptr url_loader_factory_; + const scoped_refptr task_runner_; + base::WeakPtrFactory weak_ptr_factory_{this}; + }; + + void SanitizeAndParseJson(std::string json, + base::OnceCallback callback); } // namespace api_request_helper diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc index 707c5b819d0064..4fa438376f4151 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc @@ -1,5 +1,6 @@ #include "completion_api_client.h" + #include #include @@ -15,17 +16,15 @@ #include "base/values.h" #include "net/http/http_status_code.h" #include "net/traffic_annotation/network_traffic_annotation.h" -#include "services/network/public/cpp/shared_url_loader_factory.h" #include "services/network/public/cpp/simple_url_loader.h" #include "url/gurl.h" -namespace ai_chat { namespace { -constexpr char kHttpMethod[] = "POST"; + constexpr char kHttpMethod[] = "POST"; -net::NetworkTrafficAnnotationTag GetNetworkTrafficAnnotationTag() { - return net::DefineNetworkTrafficAnnotation("ai_chat", R"( + net::NetworkTrafficAnnotationTag GetNetworkTrafficAnnotationTag() { + return net::DefineNetworkTrafficAnnotation("ai_chat", R"( semantics { sender: "AI Chat" description: @@ -42,124 +41,140 @@ net::NetworkTrafficAnnotationTag GetNetworkTrafficAnnotationTag() { "Not implemented." } )"); -} + } -std::string CreateJSONRequestBody(const std::vector& prompt) { - base::Value::Dict dict; - - base::Value::List all_stop_sequences; - std::vector stop_sequences = {"\nUser:", "\nAssistant:"}; - for (const auto& item : stop_sequences) { - all_stop_sequences.Append(item); - } - - dict.Set("stream", true); - dict.Set("max_tokens", 1280); - dict.Set("top_p", 0.7); - dict.Set("temperature", 0.6); - dict.Set("model", "Mixtral-8x7B-Instruct-v0.1"); - dict.Set("stop_sequences", std::move(stop_sequences)); - - base::Value::List prompt_messages; - for (const auto& item : prompt) { - base::Value::Dict message; - message.Set("content", prompt); - message.Set("role", "user"); - dict.Set("messages", std::move(message)); - } - - std::string json; - base::JSONWriter::Write(&dict, &json); - return json; -} + std::string CreateJSONRequestBody(const std::vector& prompt) { + base::Value::Dict dict; + + // base::Value::List stop_sequences; + // stop_sequences.Append("\nUser:"); + // stop_sequences.Append("\nAssistant:"); + + dict.Set("stream", true); + dict.Set("max_tokens", 1280); + dict.Set("top_p", 0.7); + dict.Set("temperature", 0.6); + dict.Set("model", "Mixtral-8x7B-Instruct-v0.1"); + // dict.Set("stop_sequences", std::move(stop_sequences)); + + base::Value::List prompt_messages; + for (const auto& item : prompt) { + base::Value::Dict message; + message.Set("content", std::move(item)); + message.Set("role", "user"); + prompt_messages.Append(std::move(message)); + } + dict.Set("messages", std::move(prompt_messages)); + + std::string json; + base::JSONWriter::Write(dict, &json); + return json; + } } // namespace CompletionApiClient::CompletionApiClient( - scoped_refptr url_loader_factory) - : api_request_helper_(GetNetworkTrafficAnnotationTag(), - std::move(url_loader_factory)) {} + scoped_refptr url_loader_factory) + : api_request_helper_(GetNetworkTrafficAnnotationTag(), + std::move(url_loader_factory)) {} CompletionApiClient::~CompletionApiClient() = default; void CompletionApiClient::QueryPrompt( - const std::string& prompt, - GenerationCompletedCallback data_completed_callback, - GenerationDataCallback + const std::string& prompt, + GenerationCompletedCallback data_completed_callback, + GenerationDataCallback data_received_callback /* = base::NullCallback() */) { - GURL api_url{base::StrCat({url::kHttpsScheme, url::kStandardSchemeSeparator, - "api.yep.com", "/", "v1/chat/completions"})}; - DCHECK(api_url.is_valid()) << "Invalid API Url: " << api_url.spec(); + GURL api_url{base::StrCat({url::kHttpsScheme, url::kStandardSchemeSeparator, + "api.yep.com", "/", "v1/chat/completions"})}; + DCHECK(api_url.is_valid()) << "Invalid API Url: " << api_url.spec(); + + base::flat_map headers; + headers.emplace("Accept", "text/event-stream"); - base::flat_map headers; - headers.emplace("Accept", "text/event-stream"); + auto on_received = base::BindRepeating( + &CompletionApiClient::OnQueryDataReceived, weak_ptr_factory_.GetWeakPtr(), + std::move(data_received_callback)); + auto on_complete = base::BindOnce(&CompletionApiClient::OnQueryCompleted, + weak_ptr_factory_.GetWeakPtr(), + std::move(data_completed_callback)); - auto on_received = base::BindRepeating( - &CompletionApiClient::OnQueryDataReceived, weak_ptr_factory_.GetWeakPtr(), - std::move(data_received_callback)); - auto on_complete = base::BindOnce(&CompletionApiClient::OnQueryCompleted, - weak_ptr_factory_.GetWeakPtr(), credential, - std::move(data_completed_callback)); + std::vector prompts = {prompt}; + const std::string request_body = CreateJSONRequestBody(prompts); - api_request_helper_.RequestSSE(kHttpMethod, api_url, request_body, - "application/json", std::move(on_received), - std::move(on_complete), headers, {}); + api_request_helper_.RequestSSE(kHttpMethod, api_url, request_body, + "application/json", std::move(on_received), + std::move(on_complete), headers, {}); } void CompletionApiClient::ClearAllQueries() { - api_request_helper_.CancelAll(); + api_request_helper_.CancelAll(); } void CompletionApiClient::OnQueryDataReceived( - GenerationDataCallback callback, - base::expected result) { - if (!result.has_value() || !result->is_dict()) { - return; - } - - // This client only supports completion events - const std::string* completion = result->GetDict().FindString("completion"); - if (completion) { - callback.Run(std::move(*completion)); - } + GenerationDataCallback callback, + base::expected result) { + if (!result.has_value() || !result->is_dict()) { + return; + } + const base::Value::List* list = result->GetDict().FindList("choices"); + if (list) { + for (const auto& item : *list) { + if (item.is_dict()) { + const base::Value::Dict* delta = item.GetDict().FindDict("delta"); + if (delta) { + const std::string* content = delta->FindString("content"); + if (content) { + LOG(INFO) << "delta content: " << *content; + } + } + } + } + } + + // This client only supports completion events + const std::string* completion = result->GetDict().FindString("choices"); + if (completion) { + callback.Run(std::move(*completion)); + } } void CompletionApiClient::OnQueryCompleted( - std::optional credential, - GenerationCompletedCallback callback, - APIRequestResult result) { - const bool success = result.Is2XXResponseCode(); - // Handle successful request - if (success) { - std::string completion = ""; - // We're checking for a value body in case for non-streaming API results. - if (result.value_body().is_dict()) { - const std::string* value = - result.value_body().GetDict().FindString("completion"); - if (value) { - // Trimming necessary for Llama 2 which prepends responses with a " ". - completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); - } + GenerationCompletedCallback callback, + APIRequestResult result) { + const bool success = result.Is2XXResponseCode(); + // Handle successful request + if (success) { + std::string completion = ""; + // We're checking for a value body in case for non-streaming API results. + if (result.value_body().is_dict()) { + const std::string* value = + result.value_body().GetDict().FindString("choices"); + if (value) { + // Trimming necessary for Llama 2 which prepends responses with a " ". + completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); + } + } + + // std::move(callback).Run(base::ok(std::move(completion))); + return; } - std::move(callback).Run(base::ok(std::move(completion))); - return; - } + // Handle error + chat::mojom::APIError error; - // Handle error - mojom::APIError error; + if (net::HTTP_TOO_MANY_REQUESTS == result.response_code()) { + error = chat::mojom::APIError::RateLimitReached; + } else if (net::HTTP_REQUEST_ENTITY_TOO_LARGE == result.response_code()) { + error = chat::mojom::APIError::ContextLimitReached; + } else { + error = chat::mojom::APIError::ConnectionError; + } - if (net::HTTP_TOO_MANY_REQUESTS == result.response_code()) { - error = mojom::APIError::RateLimitReached; - } else if (net::HTTP_REQUEST_ENTITY_TOO_LARGE == result.response_code()) { - error = mojom::APIError::ContextLimitReached; - } else { - error = mojom::APIError::ConnectionIssue; - } + auto _ = error; - std::move(callback).Run(base::unexpected(std::move(error))); + // std::move(callback).Run(base::unexpected(std::move(error))); } -} // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h index dd7ac5d82b61ba..c53f3a93808f7b 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h @@ -11,50 +11,49 @@ #include "base/functional/callback_forward.h" #include "base/memory/weak_ptr.h" #include "base/types/expected.h" +#include "chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" namespace network { -class SharedURLLoaderFactory; + class SharedURLLoaderFactory; } // namespace network -namespace ai_chat { using api_request_helper::APIRequestResult; class CompletionApiClient { - public: - using GenerationResult = base::expected; - using GenerationDataCallback = base::RepeatingCallback; - using GenerationCompletedCallback = - base::OnceCallback; - - CompletionApiClient( - scoped_refptr url_loader_factory); - - CompletionApiClient(const CompletionApiClient&) = delete; - CompletionApiClient& operator=(const CompletionApiClient&) = delete; - virtual ~CompletionApiClient(); - - // This function queries both types of APIs: SSE and non-SSE. - // In non-SSE cases, only the data_completed_callback will be triggered. - virtual void QueryPrompt( - const std::string& prompt, - GenerationCompletedCallback data_completed_callback, - GenerationDataCallback data_received_callback = base::NullCallback()); - // Clears all in-progress requests - void ClearAllQueries(); - - private: - void OnQueryDataReceived(GenerationDataCallback callback, - base::expected result); - void OnQueryCompleted(std::optional credential, - GenerationCompletedCallback callback, - APIRequestResult result); - - const base::flat_set stop_sequences_; - api_request_helper::APIRequestHelper api_request_helper_; - - base::WeakPtrFactory weak_ptr_factory_{this}; +public: + using GenerationResult = base::expected; + using GenerationDataCallback = base::RepeatingCallback; + using GenerationCompletedCallback = + base::OnceCallback; + + CompletionApiClient( + scoped_refptr url_loader_factory); + + CompletionApiClient(const CompletionApiClient&) = delete; + CompletionApiClient& operator=(const CompletionApiClient&) = delete; + virtual ~CompletionApiClient(); + + // This function queries both types of APIs: SSE and non-SSE. + // In non-SSE cases, only the data_completed_callback will be triggered. + virtual void QueryPrompt( + const std::string& prompt, + GenerationCompletedCallback data_completed_callback, + GenerationDataCallback data_received_callback = base::NullCallback()); + // Clears all in-progress requests + void ClearAllQueries(); + +private: + void OnQueryDataReceived(GenerationDataCallback callback, + base::expected result); + void OnQueryCompleted(GenerationCompletedCallback callback, + APIRequestResult result); + + api_request_helper::APIRequestHelper api_request_helper_; + + base::WeakPtrFactory weak_ptr_factory_{this}; }; -} // namespace ai_chat -#endif // CHROMIUM_COMPLETION_API_CLIENT_H +#endif // CHROMIUM_COMPLETION_API_CLIENT_H \ No newline at end of file diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 2c7d18bd36a900..0acb466629530b 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -1,26 +1,33 @@ #include "chat_page_handler.h" +#include #include #include #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" +#include "chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h" #include "chrome/grit/generated_resources.h" +#include "content/public/browser/browser_context.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/storage_partition.h" #include "content/public/browser/web_contents.h" #include "content/public/browser/web_ui.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" ChatPageHandler::ChatPageHandler( - mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui, - content::WebUI* web_ui) - : receiver_(this, std::move(receiver)), - page_(std::move(page)), - chat_ui_(chat_ui), - web_ui_(web_ui), - web_contents_(web_ui->GetWebContents()) {} + mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui) + : receiver_(this, std::move(receiver)), + page_(std::move(page)), + chat_ui_(chat_ui), + web_ui_(web_ui), + web_contents_(web_ui->GetWebContents()), + profile_(Profile::FromWebUI(web_ui)) {} ChatPageHandler::~ChatPageHandler() = default; @@ -38,62 +45,71 @@ void ChatPageHandler::CloseUI() { } void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { - if (page_.is_bound()) { - page_->OnSiteInfoChanged(std::move(site_info)); - } + if (page_.is_bound()) { + page_->OnSiteInfoChanged(std::move(site_info)); + } } void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { - auto title = base::UTF16ToUTF8(web_contents_->GetTitle()); - std::string url; - const GURL gurl = web_contents_->GetLastCommittedURL(); - if (gurl.SchemeIsHTTPOrHTTPS()) { - url = gurl.spec(); - } - chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); - site_info->title = title; - site_info->url = url; - - // todo: to check the schema of the current tab - site_info->is_content_usable_in_conversations = true; - - // todo: to check the content of the current tab - site_info->is_content_modified = false; - - std::move(callback).Run(site_info.Clone()); + auto title = base::UTF16ToUTF8(web_contents_->GetTitle()); + std::string url; + const GURL gurl = web_contents_->GetLastCommittedURL(); + if (gurl.SchemeIsHTTPOrHTTPS()) { + url = gurl.spec(); + } + chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + site_info->title = title; + site_info->url = url; + + // todo: to check the schema of the current tab + site_info->is_content_usable_in_conversations = true; + + // todo: to check the content of the current tab + site_info->is_content_modified = false; + + std::move(callback).Run(site_info.Clone()); } void ChatPageHandler::GetActionList(GetActionListCallback callback) { - std::vector action_items; + std::vector action_items; - chat::mojom::ActionItemPtr summarize_item = chat::mojom::ActionItem::New( - chat::mojom::ActionType::SUMMARIZE_PAGE, - l10n_util::GetStringUTF8(IDS_CHAT_SUMMARIZE_THIS_PAGE)); + chat::mojom::ActionItemPtr summarize_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::SUMMARIZE_PAGE, + l10n_util::GetStringUTF8(IDS_CHAT_SUMMARIZE_THIS_PAGE)); - chat::mojom::ActionItemPtr explain_item = chat::mojom::ActionItem::New( - chat::mojom::ActionType::EXPLAIN, - l10n_util::GetStringUTF8(IDS_CHAT_EXPLAIN_IT_IN_SIMPLE_LANGUAGE)); + chat::mojom::ActionItemPtr explain_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::EXPLAIN, + l10n_util::GetStringUTF8(IDS_CHAT_EXPLAIN_IT_IN_SIMPLE_LANGUAGE)); - chat::mojom::ActionItemPtr translate_item = chat::mojom::ActionItem::New( - chat::mojom::ActionType::TRANSLATE, - l10n_util::GetStringUTF8(IDS_CHAT_TRANSLATE)); + chat::mojom::ActionItemPtr translate_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::TRANSLATE, + l10n_util::GetStringUTF8(IDS_CHAT_TRANSLATE)); - chat::mojom::ActionItemPtr draft_social_media_post_item = - chat::mojom::ActionItem::New( - chat::mojom::ActionType::DRAFT_SOCIAL_MEDIA_POST, - l10n_util::GetStringUTF8(IDS_CHAT_DRAFT_A_SOCIAL_MEDIA_POST)); + chat::mojom::ActionItemPtr draft_social_media_post_item = + chat::mojom::ActionItem::New( + chat::mojom::ActionType::DRAFT_SOCIAL_MEDIA_POST, + l10n_util::GetStringUTF8(IDS_CHAT_DRAFT_A_SOCIAL_MEDIA_POST)); - chat::mojom::ActionItemPtr fact_check_item = chat::mojom::ActionItem::New( - chat::mojom::ActionType::FACT_CHECK, - l10n_util::GetStringUTF8(IDS_CHAT_DRAFT_FACT_CHECT)); + chat::mojom::ActionItemPtr fact_check_item = chat::mojom::ActionItem::New( + chat::mojom::ActionType::FACT_CHECK, + l10n_util::GetStringUTF8(IDS_CHAT_DRAFT_FACT_CHECT)); - action_items.push_back(summarize_item.Clone()); - action_items.push_back(explain_item.Clone()); - action_items.push_back(translate_item.Clone()); - action_items.push_back(draft_social_media_post_item.Clone()); - action_items.push_back(fact_check_item.Clone()); + action_items.push_back(summarize_item.Clone()); + action_items.push_back(explain_item.Clone()); + action_items.push_back(translate_item.Clone()); + action_items.push_back(draft_social_media_post_item.Clone()); + action_items.push_back(fact_check_item.Clone()); - std::move(callback).Run(std::move(action_items)); + std::move(callback).Run(std::move(action_items)); +} + +void ChatPageHandler::SubmitQueryCallback(std::string completion) { + LOG(INFO) << completion; +} + +void ChatPageHandler::SubmitQueryCompletedCallback( + base::expected result) { + LOG(INFO) << result.has_value(); } void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { @@ -101,7 +117,18 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { LOG(INFO) << action_type; chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; + + scoped_refptr url_loader_factory = + profile_->GetDefaultStoragePartition() + ->GetURLLoaderFactoryForBrowserProcess(); + api_client_ = std::make_unique( + std::move(url_loader_factory)); + + // Fix: use proper callback + api_client_->QueryPrompt("test", base::NullCallback(), + base::NullCallback()); response->result = "MOCK Result"; + page_->OnSubmitActionResponse(response.Clone()); } } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index b429a9636d934d..bcc499365306fb 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -1,8 +1,16 @@ #ifndef CHROMIUM_CHAT_PAGE_HANDLER_H #define CHROMIUM_CHAT_PAGE_HANDLER_H +#include +#include +#include + +#include "base/functional/callback_forward.h" #include "base/memory/raw_ptr.h" +#include "base/types/expected.h" #include "chat_ui.h" +#include "chrome/browser/profiles/profile.h" +#include "chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h" #include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" @@ -10,30 +18,34 @@ #include "mojo/public/cpp/bindings/remote.h" namespace content { -class WebContents; -class WebUI; + class WebContents; + class WebUI; } // namespace content class ChatPageHandler : public chat::mojom::PageHandler { public: - ChatPageHandler(mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui, - content::WebUI* web_ui); + ChatPageHandler(mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui); + + ChatPageHandler(const ChatPageHandler&) = delete; + ChatPageHandler& operator=(const ChatPageHandler&) = delete; - ChatPageHandler(const ChatPageHandler&) = delete; - ChatPageHandler& operator=(const ChatPageHandler&) = delete; + ~ChatPageHandler() override; - ~ChatPageHandler() override; + void GetSiteInfo(GetSiteInfoCallback callback) override; + void GetActionList(GetActionListCallback callback) override; + void SubmitAction(chat::mojom::ActionType action_type) override; + void SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) override; + void ShowUI() override; + void CloseUI() override; - void GetSiteInfo(GetSiteInfoCallback callback) override; - void GetActionList(GetActionListCallback callback) override; - void SubmitAction(chat::mojom::ActionType action_type) override; - void SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) override; - void ShowUI() override; - void CloseUI() override; + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); - void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); + void SubmitQueryCallback(std::string completion); + void SubmitQueryCompletedCallback( + base::expected result); private: mojo::Receiver receiver_; @@ -41,5 +53,7 @@ class ChatPageHandler : public chat::mojom::PageHandler { const raw_ptr chat_ui_; const raw_ptr web_ui_; raw_ptr web_contents_; + const raw_ptr profile_; + std::unique_ptr api_client_ = nullptr; }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H From c11d5e22659b0ac0c218181d2b3024150fded74d Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 4 Nov 2024 09:57:44 +0800 Subject: [PATCH 13/34] Implement demo chat UI --- .../side_panel/chat/chat_api_proxy.ts | 4 +- .../resources/side_panel/chat/chat_app.css | 52 +++++++++++++++---- .../side_panel/chat/chat_app.html.ts | 23 +++++--- .../resources/side_panel/chat/chat_app.ts | 16 ++++++ .../side_panel/chat/chat_page_handler.cc | 20 +++---- 5 files changed, 84 insertions(+), 31 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts index f8c9999a505510..decf77ad1c60ac 100644 --- a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts +++ b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts @@ -42,8 +42,8 @@ export class ChatApiProxyImpl implements ChatApiProxy { this.handler.submitAction(actionType); } - submitQuery(_: ActionType, __: string) { - + submitQuery(actionType: ActionType, query: string) { + this.handler.submitQuery(actionType, query); } getSiteInfo() { diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index db15933072aaf3..53099167026e89 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -1,9 +1,10 @@ + /* #css_wrapper_metadata_start * #type=style-lit + * #import=../chat_app.css.js * #scheme=relative * #css_wrapper_metadata_end */ - @import url('https://fonts.googleapis.com/css2?family=Poppins:wght@400;500;600&display=swap'); * { margin: 0; @@ -23,6 +24,24 @@ --incoming-chat-border: #D9D9E3; } +cr-input, cr-textarea { + width: 100%; +} + +cr-button { + + padding: 10px 10px; + border: 1px solid #000000; + border-radius: 12px; + cursor: pointer; + font-size: 12px; + text-decoration: none; + color: #474444; +} + +.no-error { + --cr-input-error-display: none; +} #container { display: flex; flex-direction: column; @@ -52,40 +71,55 @@ .bottom { flex: 0 0 auto; display: flex; - /* Takes only the height of its content */ + flex-direction: column; + gap: 12px; +} + +.button-container { + display: flex; + gap: 6px; + flex-direction: column; + align-items: flex-start; +} + +.action-button { + display: inline-block; + padding: 10px 10px; + border: 1px solid #000000; + border-radius: 12px; + cursor: pointer; + font-size: 12px; + text-decoration: none; } .typing-container { - position: fixed; bottom: 0; width: 100%; display: flex; padding: 20px 10px; - justify-content: center; + gap: 12px; background: var(--outgoing-chat-bg); border-top: 1px solid var(--incoming-chat-border); + align-items: flex-start; } .typing-container .typing-content { display: flex; - max-width: 950px; width: 100%; - align-items: flex-end; } .typing-container .typing-textarea { width: 100%; display: flex; - position: relative; } -.typing-textarea textarea { +.typing-textarea{ resize: none; height: 55px; width: 100%; padding: 15px 45px 15px 20px; color: var(--text-color); - font-size: 1rem; + font-size: 14px; border-radius: 12px; max-height: 250px; overflow-y: auto; diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index df40f63ece12ca..4d0432bdaa4fd6 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -4,22 +4,29 @@ import {html} from '//resources/lit/v3_0/lit.rollup.js'; export function getHtml(this: ChatAppElement) { return html`
-
-
${this.siteInfo_.url}
-
${this.siteInfo_.title}
-
${this.submitResponse_.result}
+
${this.actionList_.map((item,_) => html` - + ${item.label} `)} -
- - send +
+
+ + + + Send +
`; diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 91edccab4482c8..710553fcc70715 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -1,5 +1,10 @@ import './strings.m.js'; +import '//resources/cr_elements/cr_button/cr_button.js'; +import '//resources/cr_elements/cr_icon_button/cr_icon_button.js'; +import '//resources/cr_elements/cr_input/cr_input.js'; +import '//resources/cr_elements/cr_textarea/cr_textarea.js'; + import {loadTimeData} from 'chrome://resources/js/load_time_data.js'; import {CrLitElement} from '//resources/lit/v3_0/lit.rollup.js'; import {getCss} from './chat_app.css.js'; @@ -23,6 +28,7 @@ export class ChatAppElement extends CrLitElement { actionType: ActionType.NONE, result: "" }; + protected textareaValue_?: string; constructor() { super(); @@ -42,6 +48,7 @@ export class ChatAppElement extends CrLitElement { askAnythingLabel_: {type: String}, actionList_: {type: Array}, submitResponse_: {type: Object}, + textareaValue_: {type: String}, }; } @@ -63,6 +70,15 @@ export class ChatAppElement extends CrLitElement { this.chatApiProxy_.submitAction(ActionType.SUMMARIZE_PAGE); } + protected onTextareaValueChanged_(e: CustomEvent<{value: string}>) { + this.textareaValue_ = e.detail.value; + } + + protected async onSubmitQuery_() { + console.log(this.textareaValue_); + this.chatApiProxy_.submitQuery(ActionType.QUERY, this.textareaValue_ ?? "" ); + } + override connectedCallback() { super.connectedCallback(); diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 0acb466629530b..1dfd2d68b52b54 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -117,23 +117,19 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { LOG(INFO) << action_type; chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; - - scoped_refptr url_loader_factory = - profile_->GetDefaultStoragePartition() - ->GetURLLoaderFactoryForBrowserProcess(); - api_client_ = std::make_unique( - std::move(url_loader_factory)); - - // Fix: use proper callback - api_client_->QueryPrompt("test", base::NullCallback(), - base::NullCallback()); response->result = "MOCK Result"; - page_->OnSubmitActionResponse(response.Clone()); } } void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { - + scoped_refptr url_loader_factory = + profile_->GetDefaultStoragePartition() + ->GetURLLoaderFactoryForBrowserProcess(); + api_client_ = + std::make_unique(std::move(url_loader_factory)); + + // Fix: use proper callback + api_client_->QueryPrompt(query, base::NullCallback(), base::NullCallback()); } From 205c96164d8f56aced9ee616ebdfd868f763dd2f Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Thu, 7 Nov 2024 09:33:29 +0800 Subject: [PATCH 14/34] Show action menu in chat panel when the current page includes content suitable for conversations --- .../side_panel/chat/chat_app.html.ts | 9 ++++- .../resources/side_panel/chat/chat_app.ts | 1 - chrome/browser/ui/views/side_panel/BUILD.gn | 4 +- .../chat/chat_side_panel_coordinator.cc | 2 +- .../{ => chat}/chat_side_panel_web_view.cc | 39 ++++--------------- .../{ => chat}/chat_side_panel_web_view.h | 2 +- .../ui/webui/side_panel/chat/chat.mojom | 3 -- .../side_panel/chat/chat_page_handler.cc | 19 ++++----- 8 files changed, 26 insertions(+), 53 deletions(-) rename chrome/browser/ui/views/side_panel/{ => chat}/chat_side_panel_web_view.cc (72%) rename chrome/browser/ui/views/side_panel/{ => chat}/chat_side_panel_web_view.h (97%) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 4d0432bdaa4fd6..799a7bcc454484 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -5,15 +5,20 @@ export function getHtml(this: ChatAppElement) { return html`
+
+
${this.siteInfo_.url}
+
${this.siteInfo_.title}
+
${this.submitResponse_.result}
- ${this.actionList_.map((item,_) => html` + ${ this.siteInfo_.isContentUsableInConversations ? + this.actionList_.map((item,_) => html` ${item.label} - `)} + `) : html``}
>( GURL(chrome::kChromeUIChatURL), @@ -69,44 +68,22 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( return; } - // auto title = base::UTF16ToUTF8( web_contents_->GetTitle()); - // std::string url; - // const GURL gurl = web_contents_->GetLastCommittedURL(); - // if (gurl.SchemeIsHTTPOrHTTPS()) { - // url = gurl.spec(); - // } - // chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); - // site_info->title = title; - // site_info->url = url; - // - auto title = contents->GetTitle(); std::string url; + chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); const GURL gurl = contents->GetLastCommittedURL(); if (gurl.SchemeIsHTTPOrHTTPS()) { url = gurl.spec(); + site_info->is_content_usable_in_conversations = true; + } else { + site_info->is_content_usable_in_conversations = false; } - // bool ConversationDriver::IsContentAssociationPossible() { - // const GURL url = GetPageURL(); - // - // if (!base::Contains(kAllowedSchemes, url.scheme())) { - // return false; - // } - // - // return true; - // } - chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); - site_info->title = base::UTF16ToUTF8(title); + site_info->title = base::UTF16ToUTF8(contents->GetTitle()); site_info->url = url; - // todo: to check the schema of the current tab - site_info->is_content_usable_in_conversations = true; - // todo: to check the content of the current tab - site_info->is_content_modified = false; - controller->GetAs()->SetSiteInfo(site_info.Clone()); } -void ChatSidePanelWebView::UpdateActiveSiteInfoToActiveTab() { +void ChatSidePanelWebView::UpdateActiveWebContents() { UpdateActiveSiteInfo(browser_->tab_strip_model()->GetActiveWebContents()); } diff --git a/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h similarity index 97% rename from chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h rename to chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h index 7bda0b04b78d35..d68673ff0c3685 100644 --- a/chrome/browser/ui/views/side_panel/chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h @@ -34,7 +34,7 @@ class ChatSidePanelWebView : public SidePanelWebUIViewT, TabChangeType change_type) override; void UpdateActiveSiteInfo(content::WebContents* contents); - void UpdateActiveSiteInfoToActiveTab(); + void UpdateActiveWebContents(); private: const raw_ptr browser_; diff --git a/chrome/browser/ui/webui/side_panel/chat/chat.mojom b/chrome/browser/ui/webui/side_panel/chat/chat.mojom index bf86be573b2287..140d69fe39c986 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat.mojom +++ b/chrome/browser/ui/webui/side_panel/chat/chat.mojom @@ -18,9 +18,6 @@ struct SiteInfo { // Indicates whether the current URL contains content that can be used in conversations with Yep Chat bool is_content_usable_in_conversations; - - // Indicates that the content has been refined by distilling the most relevant sections from lengthy page material. - bool is_content_modified; }; enum ActionType { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 1dfd2d68b52b54..e213ecac350d1e 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -51,21 +51,16 @@ void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { } void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { - auto title = base::UTF16ToUTF8(web_contents_->GetTitle()); - std::string url; + chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + site_info->title = base::UTF16ToUTF8(web_contents_->GetTitle()); const GURL gurl = web_contents_->GetLastCommittedURL(); if (gurl.SchemeIsHTTPOrHTTPS()) { - url = gurl.spec(); + site_info->url = gurl.spec(); + site_info->is_content_usable_in_conversations = true; + } else { + site_info->url = ""; + site_info->is_content_usable_in_conversations = false; } - chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); - site_info->title = title; - site_info->url = url; - - // todo: to check the schema of the current tab - site_info->is_content_usable_in_conversations = true; - - // todo: to check the content of the current tab - site_info->is_content_modified = false; std::move(callback).Run(site_info.Clone()); } From 04e6cf8d29f21598cbd198286ce0cc386dd972ea Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 8 Nov 2024 17:12:55 +0800 Subject: [PATCH 15/34] Add content extractor IDL - WIP --- chrome/browser/BUILD.gn | 1 + .../chat/chat_side_panel_web_view.cc | 9 +- .../side_panel/chat/chat_page_handler.cc | 33 ++- chrome/common/chat/BUILD.gn | 6 + .../common/chat/page_content_extractor.mojom | 24 +++ chrome/common/chrome_isolated_world_ids.h | 2 + chrome/renderer/BUILD.gn | 3 + .../renderer/chat/page_content_extractor.cc | 203 ++++++++++++++++++ chrome/renderer/chat/page_content_extractor.h | 58 +++++ 9 files changed, 329 insertions(+), 10 deletions(-) create mode 100644 chrome/common/chat/BUILD.gn create mode 100644 chrome/common/chat/page_content_extractor.mojom create mode 100644 chrome/renderer/chat/page_content_extractor.cc create mode 100644 chrome/renderer/chat/page_content_extractor.h diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn index d3564c952db123..b0facbeaba9e1c 100644 --- a/chrome/browser/BUILD.gn +++ b/chrome/browser/BUILD.gn @@ -1996,6 +1996,7 @@ static_library("browser") { "//chrome/common:supervised_user_commands_mojom", "//chrome/common:version_header", "//chrome/common/cart:mojo_bindings", + "//chrome/common/chat:mojo_bindings", "//chrome/common/net", "//chrome/common/notifications", "//chrome/installer/util:with_no_strings", diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc index 3c89d266aa7991..d2ac7fd830bf7e 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc @@ -68,18 +68,17 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( return; } - std::string url; chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + site_info->title = base::UTF16ToUTF8(contents->GetTitle()); + const GURL gurl = contents->GetLastCommittedURL(); if (gurl.SchemeIsHTTPOrHTTPS()) { - url = gurl.spec(); + site_info->url = gurl.spec(); site_info->is_content_usable_in_conversations = true; } else { + site_info->url = ""; site_info->is_content_usable_in_conversations = false; } - site_info->title = base::UTF16ToUTF8(contents->GetTitle()); - site_info->url = url; - controller->GetAs()->SetSiteInfo(site_info.Clone()); } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index e213ecac350d1e..3e62ae9005d7f8 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -13,6 +13,7 @@ #include "content/public/browser/storage_partition.h" #include "content/public/browser/web_contents.h" #include "content/public/browser/web_ui.h" +#include "page_content_extractor.h" #include "services/network/public/cpp/shared_url_loader_factory.h" #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" @@ -110,14 +111,36 @@ void ChatPageHandler::SubmitQueryCompletedCallback( void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { if (page_.is_bound()) { LOG(INFO) << action_type; - chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); - response->action_type = action_type; - response->result = "MOCK Result"; - page_->OnSubmitActionResponse(response.Clone()); + + if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { + auto on_page_text_extracted = + [this, &action_type](const std::optional& text) { + chat::mojom::ActionResponsePtr response = + chat::mojom::ActionResponse::New(); + response->action_type = action_type; + response->result = text.value_or(""); + this->page_->OnSubmitActionResponse(response.Clone()); + }; + + const GURL gurl = web_contents_->GetLastCommittedURL(); + if (gurl.SchemeIsHTTPOrHTTPS()) { + content::RenderFrameHost* main_frame = + web_contents_->GetPrimaryMainFrame(); + ai_chat::ExtractPageText(main_frame, + content::ISOLATED_WORLD_ID_GLOBAL, + on_page_text_extracted); + } + } else { + // todo: to implement for other action types later + chat::mojom::ActionResponsePtr response = + chat::mojom::ActionResponse::New(); + response->action_type = action_type; + response->result = "MOCK Result"; + page_->OnSubmitActionResponse(response.Clone()); + } } } - void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { scoped_refptr url_loader_factory = profile_->GetDefaultStoragePartition() diff --git a/chrome/common/chat/BUILD.gn b/chrome/common/chat/BUILD.gn new file mode 100644 index 00000000000000..272941ec432de9 --- /dev/null +++ b/chrome/common/chat/BUILD.gn @@ -0,0 +1,6 @@ +import("//mojo/public/tools/bindings/mojom.gni") + +mojom("mojo_bindings") { + sources = [ "page_content_extractor.mojom" ] + public_deps = [ "//url/mojom:url_mojom_gurl" ] +} diff --git a/chrome/common/chat/page_content_extractor.mojom b/chrome/common/chat/page_content_extractor.mojom new file mode 100644 index 00000000000000..a5e31c80c4542f --- /dev/null +++ b/chrome/common/chat/page_content_extractor.mojom @@ -0,0 +1,24 @@ +module chat.mojom; + +import "url/mojom/url.mojom"; + +enum PageContentType { + Text, + VideoTranscriptYouTube, + VideoTranscriptVTT +}; + +// It has either text or url +union PageContentData { + string content; + url.mojom.Url content_url; +}; + +struct PageContent { + PageContentData content; + PageContentType type; +}; + +interface PageContentExtractor { + ExtractPageContent() => (PageContent? page_content); +}; diff --git a/chrome/common/chrome_isolated_world_ids.h b/chrome/common/chrome_isolated_world_ids.h index 20c1984064b2e7..5dce77d0908c80 100644 --- a/chrome/common/chrome_isolated_world_ids.h +++ b/chrome/common/chrome_isolated_world_ids.h @@ -16,6 +16,8 @@ enum ChromeIsolatedWorldIDs { // Isolated world ID for internal Chrome features. ISOLATED_WORLD_ID_CHROME_INTERNAL, + ISOLATED_WORLD_ID_TAKTAK_INTERNAL, + #if BUILDFLAG(IS_MAC) // Isolated world ID for AppleScript. ISOLATED_WORLD_ID_APPLESCRIPT, diff --git a/chrome/renderer/BUILD.gn b/chrome/renderer/BUILD.gn index 7184fd42569ec8..f3191198cd004b 100644 --- a/chrome/renderer/BUILD.gn +++ b/chrome/renderer/BUILD.gn @@ -73,6 +73,8 @@ static_library("renderer") { "browser_exposed_renderer_interfaces.h", "cart/commerce_hint_agent.cc", "cart/commerce_hint_agent.h", + "chat/page_content_extractor.cc", + "chat/page_content_extractor.h", "chrome_content_renderer_client.cc", "chrome_content_renderer_client.h", "chrome_content_settings_agent_delegate.cc", @@ -143,6 +145,7 @@ static_library("renderer") { "//chrome/common", "//chrome/common:mojo_bindings", "//chrome/common/cart:mojo_bindings", + "//chrome/common/chat:mojo_bindings", "//chrome/common/net", "//chrome/common/search:mojo_bindings", "//chrome/services/speech/buildflags", diff --git a/chrome/renderer/chat/page_content_extractor.cc b/chrome/renderer/chat/page_content_extractor.cc new file mode 100644 index 00000000000000..16e083b211d8b4 --- /dev/null +++ b/chrome/renderer/chat/page_content_extractor.cc @@ -0,0 +1,203 @@ +#include "page_content_extractor.h" + +#include +#include +#include +#include +#include +#include + +#include "base/containers/contains.h" +#include "base/ranges/algorithm.h" +#include "base/strings/string_util.h" +#include "base/strings/utf_string_conversions.h" +#include "base/time/time.h" +#include "content/public/renderer/render_frame.h" +#include "third_party/blink/public/web/web_local_frame.h" +#include "third_party/blink/public/web/web_script_source.h" +#include "ui/accessibility/ax_node.h" +#include "ui/accessibility/ax_tree.h" + +namespace ai_chat { +namespace { +static const ax::mojom::Role kContentParentRoles[]{ + ax::mojom::Role::kMain, + ax::mojom::Role::kArticle, +}; + +static const ax::mojom::Role kContentRoles[]{ + ax::mojom::Role::kHeading, + ax::mojom::Role::kParagraph, + ax::mojom::Role::kNote, +}; + +static const ax::mojom::Role kRolesToSkip[]{ + ax::mojom::Role::kAudio, ax::mojom::Role::kBanner, + ax::mojom::Role::kButton, ax::mojom::Role::kComplementary, + ax::mojom::Role::kContentInfo, ax::mojom::Role::kFooter, + ax::mojom::Role::kImage, ax::mojom::Role::kLabelText, + ax::mojom::Role::kNavigation, ax::mojom::Role::kSectionFooter, + ax::mojom::Role::kTextField, ax::mojom::Role::kTextFieldWithComboBox, + ax::mojom::Role::kComboBoxSelect, ax::mojom::Role::kListBox, + ax::mojom::Role::kListBoxOption, ax::mojom::Role::kCheckBox, + ax::mojom::Role::kRadioButton, ax::mojom::Role::kSlider, + ax::mojom::Role::kSpinButton, ax::mojom::Role::kSearchBox, +}; + +void GetContentRootNodes(const ui::AXNode* root, + std::vector* content_root_nodes) { + std::queue queue; + queue.push(root); + while (!queue.empty()) { + const ui::AXNode* node = queue.front(); + queue.pop(); + // If a main or article node is found, add it to the list of content root + // nodes and continue. Do not explore children for nested article nodes. + if (base::Contains(kContentParentRoles, node->GetRole())) { + content_root_nodes->push_back(node); + continue; + } + for (auto iter = node->UnignoredChildrenBegin(); + iter != node->UnignoredChildrenEnd(); ++iter) { + queue.push(iter.get()); + } + } +} + +void AddContentNodesToVector(const ui::AXNode* node, + std::vector* content_nodes) { + if (base::Contains(kContentRoles, node->GetRole())) { + content_nodes->emplace_back(node); + return; + } + if (base::Contains(kRolesToSkip, node->GetRole())) { + return; + } + for (auto iter = node->UnignoredChildrenBegin(); + iter != node->UnignoredChildrenEnd(); ++iter) { + AddContentNodesToVector(iter.get(), content_nodes); + } +} + +void AddTextNodesToVector(const ui::AXNode* node, + std::vector* strings) { + const ui::AXNodeData& node_data = node->data(); + + if (base::Contains(kRolesToSkip, node_data.role)) { + return; + } + + if (node_data.role == ax::mojom::Role::kStaticText) { + if (node_data.HasStringAttribute(ax::mojom::StringAttribute::kName)) { + strings->push_back( + node_data.GetString16Attribute(ax::mojom::StringAttribute::kName)); + } + return; + } + + for (const ui::AXNode* child : node->children()) { + AddTextNodesToVector(child, strings); + } +} +} // namespace + +PageContentExtractor::PageContentExtractor( + content::RenderFrame* render_frame, + service_manager::BinderRegistry* registry) + : content::RenderFrameObserver(render_frame), + RenderFrameObserverTracker(render_frame), + weak_ptr_factory_(this) { + if (!render_frame->IsMainFrame()) { + return; + } + registry->AddInterface(base::BindRepeating( + &PageContentExtractor::BindReceiver, base::Unretained(this))); +} + +PageContentExtractor::~PageContentExtractor() = default; + +void PageContentExtractor::OnDestruct() { + delete this; +} + +base::WeakPtr PageContentExtractor::GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); +} + +void PageContentExtractor::ExtractPageContent( + mojom::PageContentExtractor::ExtractPageContentCallback callback) { + LOG(INFO) << "AI Chat renderer has been asked for page content."; + + blink::WebLocalFrame* main_frame = render_frame()->GetWebFrame(); + + ExtractPageText(render_frame(), ISOLATED_WORLD_ID_TAKTAK_INTERNAL, + std::move(callback)); +} + +void PageContentExtractor::ExtractPageText( + content::RenderFrameHost* render_frame, + int32_t isolated_world_id, + base::OnceCallback&)> callback) { + auto snapshotter = render_frame->CreateAXTreeSnapshotter( + ui::AXMode::kWebContents | ui::AXMode::kHTML | ui::AXMode::kScreenReader); + ui::AXTreeUpdate snapshot; + snapshotter->Snapshot( + /* max_nodes= */ 9000, /* timeout= */ base::Seconds(4), &snapshot); + ui::AXTree tree(snapshot); + + std::vector content_root_nodes; + std::vector content_nodes; + GetContentRootNodes(tree.root(), &content_root_nodes); + + for (const ui::AXNode* content_root_node : content_root_nodes) { + std::vector content_nodes_this_root; + AddContentNodesToVector(content_root_node, &content_nodes_this_root); + // If no content was retrieved for this root node, fall back to using + // the text directly from the root node. This ensures we capture content + // from the node identified as containing important information. + if (content_nodes_this_root.empty()) { + content_nodes.emplace_back(content_root_node); + } else { + base::ranges::move(content_nodes_this_root, + std::back_inserter(content_nodes)); + } + } + + std::vector text_node_contents; + for (const ui::AXNode* content_node : content_nodes) { + AddTextNodesToVector(content_node, &text_node_contents); + } + + std::string contents_text = + base::UTF16ToUTF8(base::JoinString(text_node_contents, u" ")); + + if (contents_text.empty()) { + blink::WebScriptSource source = blink::WebScriptSource( + blink::WebString::FromASCII("document.body.innerText")); + + auto on_script_executed = + [](base::OnceCallback&)> callback, + std::optional value, base::TimeTicks start_time) { + if (value->is_string()) { + std::move(callback).Run(value->GetString()); + return; + } + + std::move(callback).Run({}); + }; + + render_frame->GetWebFrame()->RequestExecuteScript( + isolated_world_id, UNSAFE_TODO(base::make_span(&source, 1u)), + blink::mojom::UserActivationOption::kDoNotActivate, + blink::mojom::EvaluationTiming::kAsynchronous, + blink::mojom::LoadEventBlockingOption::kDoNotBlock, + base::BindOnce(on_script_executed, std::move(callback)), + blink::BackForwardCacheAware::kAllow, + blink::mojom::WantResultOption::kWantResult, + blink::mojom::PromiseResultOption::kAwait); + return; + } + + std::move(callback).Run(contents_text); +} +} // namespace ai_chat diff --git a/chrome/renderer/chat/page_content_extractor.h b/chrome/renderer/chat/page_content_extractor.h new file mode 100644 index 00000000000000..6f57436a6cd990 --- /dev/null +++ b/chrome/renderer/chat/page_content_extractor.h @@ -0,0 +1,58 @@ +#ifndef CHROMIUM_PAGE_CONTENT_EXTRACTOR_H +#define CHROMIUM_PAGE_CONTENT_EXTRACTOR_H + +#include +#include +#include +#include + +#include "base/functional/callback_forward.h" +#include "base/values.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" +#include "content/public/renderer/render_frame.h" +#include "content/public/renderer/render_frame_observer.h" +#include "content/public/renderer/render_frame_observer_tracker.h" +#include "mojo/public/cpp/bindings/receiver.h" +#include "mojo/public/cpp/bindings/remote.h" + +namespace content { +class RenderFrame; +} + +namespace ai_chat { + +class PageContentExtractor + : public chat::mojom::PageContentExtractor, + public content::RenderFrameObserver, + public content::RenderFrameObserverTracker { + public: + PageContentExtractor(content::RenderFrame* render_frame, + service_manager::BinderRegistry* registry); + + PageContentExtractor(const PageContentExtractor&) = delete; + PageContentExtractor& operator=(const PageContentExtractor&) = delete; + ~PageContentExtractor() override; + + base::WeakPtr GetWeakPtr(); + + private: + // PageContentExtractor implementation: + void ExtractPageContent( + chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) + override; + + void BindReceiver( + mojo::PendingReceiver receiver); + + chat::mojo::Receiver receiver_{this}; + + base::WeakPtrFactory weak_ptr_factory_{this}; + + void ExtractPageText( + content::RenderFrame* render_frame, + int32_t isolated_world_id, + base::OnceCallback& text)>); +}; + +} // namespace ai_chat +#endif // CHROMIUM_PAGE_CONTENT_EXTRACTOR_H From 6e0a86e617c50985089408a80bafe1bbff4fa666 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 13 Nov 2024 03:19:50 +0800 Subject: [PATCH 16/34] Add content extractor - works for most of the sites (WIP - refactoring & error handling) --- .../chat/chat_side_panel_web_view.cc | 47 ++++++++++++ .../chat/chat_side_panel_web_view.h | 11 +++ .../side_panel/chat/chat_page_handler.cc | 64 +++++++++++----- .../webui/side_panel/chat/chat_page_handler.h | 3 + chrome/common/chrome_isolated_world_ids.h | 1 - .../renderer/chat/page_content_extractor.cc | 75 ++++++++++++++++--- chrome/renderer/chat/page_content_extractor.h | 18 ++++- .../chrome_content_renderer_client.cc | 9 +++ 8 files changed, 195 insertions(+), 33 deletions(-) diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc index d2ac7fd830bf7e..ca0d490df58ae9 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc @@ -22,6 +22,16 @@ #include "ui/views/view.h" #include "ui/views/view_class_properties.h" #include "url/gurl.h" +#include "services/service_manager/public/cpp/interface_provider.h" +#include "base/containers/contains.h" +#include "base/containers/fixed_flat_set.h" +#include "base/strings/string_util.h" +#include "base/strings/utf_string_conversions.h" + +constexpr auto kVideoPageContentTypes = + base::MakeFixedFlatSet( + {chat::mojom::PageContentType::VideoTranscriptYouTube, + chat::mojom::PageContentType::VideoTranscriptVTT}); using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; BEGIN_TEMPLATE_METADATA(SidePanelWebUIViewT_ChatUI, SidePanelWebUIViewT) @@ -79,9 +89,46 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( site_info->url = ""; site_info->is_content_usable_in_conversations = false; } + + //todo : to capture content here and hold it in SiteInfo + + + auto* primary_rfh = contents->GetPrimaryMainFrame(); + DCHECK(primary_rfh->IsRenderFrameLive()); + + mojo::Remote extractor; + primary_rfh->GetRemoteInterfaces()->GetInterface( + extractor.BindNewPipeAndPassReceiver()); + + extractor->ExtractPageContent( + base::BindOnce(&ChatSidePanelWebView::OnPageContentExtracted, + base::Unretained(this))); + controller->GetAs()->SetSiteInfo(site_info.Clone()); } + +void ChatSidePanelWebView::OnPageContentExtracted(chat::mojom::PageContentPtr data) { + if (!data) { + VLOG(1) << __func__ << " no data."; + return; + } + DVLOG(1) << "OnPageContentExtracted: " << data.get(); + const bool is_video = base::Contains(kVideoPageContentTypes, data->type); + DVLOG(1) << "Is video? " << is_video; + // Handle text mode response + if (!is_video) { + DCHECK(data->content->is_content()); + auto content = data->content->get_content(); + DVLOG(1) << __func__ << ": Got content with char length of " + << content.length(); + LOG(INFO) << content; + return; + } + + LOG(INFO) << "content is url"; +} + void ChatSidePanelWebView::UpdateActiveWebContents() { UpdateActiveSiteInfo(browser_->tab_strip_model()->GetActiveWebContents()); } diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h index d68673ff0c3685..d2c2a6c7b32328 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h @@ -1,6 +1,10 @@ #ifndef CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H #define CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H +#include +#include +#include + #include "base/functional/callback_forward.h" #include "base/memory/raw_ptr.h" #include "chrome/browser/ui/tabs/tab_strip_model_observer.h" @@ -10,6 +14,11 @@ #include "ui/base/metadata/metadata_header_macros.h" #include "ui/views/controls/webview/webview.h" #include "ui/views/view.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" +#include "mojo/public/cpp/bindings/pending_receiver.h" +#include "mojo/public/cpp/bindings/pending_remote.h" +#include "mojo/public/cpp/bindings/receiver.h" +#include "mojo/public/cpp/bindings/remote.h" class Browser; @@ -39,5 +48,7 @@ class ChatSidePanelWebView : public SidePanelWebUIViewT, private: const raw_ptr browser_; base::WeakPtrFactory weak_factory_{this}; + + void OnPageContentExtracted(chat::mojom::PageContentPtr data); }; #endif // CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 3e62ae9005d7f8..8348ec04bca5ac 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -4,20 +4,28 @@ #include #include +#include "base/containers/contains.h" +#include "base/containers/fixed_flat_set.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #include "chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h" #include "chrome/grit/generated_resources.h" +#include "chrome/renderer/chat/page_content_extractor.h" #include "content/public/browser/browser_context.h" #include "content/public/browser/browser_thread.h" #include "content/public/browser/storage_partition.h" #include "content/public/browser/web_contents.h" #include "content/public/browser/web_ui.h" -#include "page_content_extractor.h" #include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/service_manager/public/cpp/interface_provider.h" #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" +constexpr auto kVideoPageContentTypes = + base::MakeFixedFlatSet( + {chat::mojom::PageContentType::VideoTranscriptYouTube, + chat::mojom::PageContentType::VideoTranscriptVTT}); + ChatPageHandler::ChatPageHandler( mojo::PendingReceiver receiver, mojo::PendingRemote page, @@ -51,6 +59,7 @@ void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { } } +// todo: to remove probably, not correct url and title here void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); site_info->title = base::UTF16ToUTF8(web_contents_->GetTitle()); @@ -113,23 +122,21 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { LOG(INFO) << action_type; if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { - auto on_page_text_extracted = - [this, &action_type](const std::optional& text) { - chat::mojom::ActionResponsePtr response = - chat::mojom::ActionResponse::New(); - response->action_type = action_type; - response->result = text.value_or(""); - this->page_->OnSubmitActionResponse(response.Clone()); - }; - - const GURL gurl = web_contents_->GetLastCommittedURL(); - if (gurl.SchemeIsHTTPOrHTTPS()) { - content::RenderFrameHost* main_frame = - web_contents_->GetPrimaryMainFrame(); - ai_chat::ExtractPageText(main_frame, - content::ISOLATED_WORLD_ID_GLOBAL, - on_page_text_extracted); - } + LOG(INFO) << "ActionType: Summarize_page"; + LOG(INFO) << "****" << base::UTF16ToUTF8(web_contents_->GetTitle()); + const GURL gurl = web_contents_->GetLastCommittedURL(); + LOG(INFO) << "****" << gurl.spec(); + auto* primary_rfh = web_contents_->GetPrimaryMainFrame(); + DCHECK(primary_rfh->IsRenderFrameLive()); + + mojo::Remote extractor; + primary_rfh->GetRemoteInterfaces()->GetInterface( + extractor.BindNewPipeAndPassReceiver()); + + extractor->ExtractPageContent( + base::BindOnce(&ChatPageHandler::OnPageContentExtracted, + base::Unretained(this))); + } else { // todo: to implement for other action types later chat::mojom::ActionResponsePtr response = @@ -141,6 +148,27 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { } } +void ChatPageHandler::OnPageContentExtracted(chat::mojom::PageContentPtr data) { + if (!data) { + VLOG(1) << __func__ << " no data."; + return; + } + DVLOG(1) << "OnPageContentExtracted: " << data.get(); + const bool is_video = base::Contains(kVideoPageContentTypes, data->type); + DVLOG(1) << "Is video? " << is_video; + // Handle text mode response + if (!is_video) { + DCHECK(data->content->is_content()); + auto content = data->content->get_content(); + DVLOG(1) << __func__ << ": Got content with char length of " + << content.length(); + LOG(INFO) << content; + return; + } + + LOG(INFO) << "content is url"; +} + void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { scoped_refptr url_loader_factory = profile_->GetDefaultStoragePartition() diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index bcc499365306fb..01d5faaa670271 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -12,6 +12,7 @@ #include "chrome/browser/profiles/profile.h" #include "chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h" #include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" @@ -55,5 +56,7 @@ class ChatPageHandler : public chat::mojom::PageHandler { raw_ptr web_contents_; const raw_ptr profile_; std::unique_ptr api_client_ = nullptr; + + void OnPageContentExtracted(chat::mojom::PageContentPtr data); }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/common/chrome_isolated_world_ids.h b/chrome/common/chrome_isolated_world_ids.h index 5dce77d0908c80..e789ed8b2dff1f 100644 --- a/chrome/common/chrome_isolated_world_ids.h +++ b/chrome/common/chrome_isolated_world_ids.h @@ -16,7 +16,6 @@ enum ChromeIsolatedWorldIDs { // Isolated world ID for internal Chrome features. ISOLATED_WORLD_ID_CHROME_INTERNAL, - ISOLATED_WORLD_ID_TAKTAK_INTERNAL, #if BUILDFLAG(IS_MAC) // Isolated world ID for AppleScript. diff --git a/chrome/renderer/chat/page_content_extractor.cc b/chrome/renderer/chat/page_content_extractor.cc index 16e083b211d8b4..52f5d1c18de575 100644 --- a/chrome/renderer/chat/page_content_extractor.cc +++ b/chrome/renderer/chat/page_content_extractor.cc @@ -8,15 +8,23 @@ #include #include "base/containers/contains.h" +#include "base/metrics/histogram_macros.h" #include "base/ranges/algorithm.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #include "base/time/time.h" +#include "chrome/common/chrome_isolated_world_ids.h" #include "content/public/renderer/render_frame.h" +#include "content/public/renderer/render_thread.h" +#include "services/metrics/public/cpp/ukm_builders.h" +#include "third_party/blink/public/platform/browser_interface_broker_proxy.h" +#include "third_party/blink/public/platform/scheduler/web_agent_group_scheduler.h" +#include "third_party/blink/public/platform/web_string.h" #include "third_party/blink/public/web/web_local_frame.h" #include "third_party/blink/public/web/web_script_source.h" #include "ui/accessibility/ax_node.h" #include "ui/accessibility/ax_tree.h" +#include "v8/include/v8-isolate.h" namespace ai_chat { namespace { @@ -103,9 +111,11 @@ void AddTextNodesToVector(const ui::AXNode* node, PageContentExtractor::PageContentExtractor( content::RenderFrame* render_frame, - service_manager::BinderRegistry* registry) + service_manager::BinderRegistry* registry, + int32_t isolated_world_id) : content::RenderFrameObserver(render_frame), RenderFrameObserverTracker(render_frame), + isolated_world_id_(isolated_world_id), weak_ptr_factory_(this) { if (!render_frame->IsMainFrame()) { return; @@ -125,19 +135,20 @@ base::WeakPtr PageContentExtractor::GetWeakPtr() { } void PageContentExtractor::ExtractPageContent( - mojom::PageContentExtractor::ExtractPageContentCallback callback) { - LOG(INFO) << "AI Chat renderer has been asked for page content."; + chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) { + LOG(INFO) << "AI Chat renderer has been asked for page content.xxx"; - blink::WebLocalFrame* main_frame = render_frame()->GetWebFrame(); - - ExtractPageText(render_frame(), ISOLATED_WORLD_ID_TAKTAK_INTERNAL, - std::move(callback)); + ExtractPageText( + render_frame(), isolated_world_id_, + base::BindOnce(&PageContentExtractor::OnPageTextExtracted, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); } void PageContentExtractor::ExtractPageText( - content::RenderFrameHost* render_frame, + content::RenderFrame* render_frame, int32_t isolated_world_id, base::OnceCallback&)> callback) { + LOG(INFO) << "ExtractPageText()"; auto snapshotter = render_frame->CreateAXTreeSnapshotter( ui::AXMode::kWebContents | ui::AXMode::kHTML | ui::AXMode::kScreenReader); ui::AXTreeUpdate snapshot; @@ -171,21 +182,35 @@ void PageContentExtractor::ExtractPageText( std::string contents_text = base::UTF16ToUTF8(base::JoinString(text_node_contents, u" ")); + LOG(INFO) << contents_text; if (contents_text.empty()) { + blink::WebLocalFrame* main_frame = render_frame->GetWebFrame(); + v8::HandleScope handle_scope( + main_frame->GetAgentGroupScheduler()->Isolate()); +// blink::WebScriptSource source = blink::WebScriptSource( +// blink::WebString::FromASCII("document.addEventListener('DOMContentLoaded', function() { document.body.innerText; });")); blink::WebScriptSource source = blink::WebScriptSource( blink::WebString::FromASCII("document.body.innerText")); auto on_script_executed = [](base::OnceCallback&)> callback, std::optional value, base::TimeTicks start_time) { + LOG(INFO) << "on_script_executed"; + LOG(INFO) << value->DebugString(); if (value->is_string()) { + LOG(INFO) << "value is string ...."; std::move(callback).Run(value->GetString()); return; } + LOG(INFO) << "Value is not string"; std::move(callback).Run({}); }; + if (render_frame->GetWebFrame()) { + LOG(INFO) << "has local web frame"; + } + render_frame->GetWebFrame()->RequestExecuteScript( isolated_world_id, UNSAFE_TODO(base::make_span(&source, 1u)), blink::mojom::UserActivationOption::kDoNotActivate, @@ -195,9 +220,39 @@ void PageContentExtractor::ExtractPageText( blink::BackForwardCacheAware::kAllow, blink::mojom::WantResultOption::kWantResult, blink::mojom::PromiseResultOption::kAwait); - return; + LOG(INFO) << "called RequestExecuteScript"; + } else { + std::move(callback).Run(contents_text); } +} - std::move(callback).Run(contents_text); +void PageContentExtractor::BindReceiver( + mojo::PendingReceiver receiver) { + VLOG(1) << "AIChat PageContentExtractor handler bound."; + receiver_.reset(); + receiver_.Bind(std::move(receiver)); +} + +void PageContentExtractor::OnPageTextExtracted( + chat::mojom::PageContentExtractor::ExtractPageContentCallback callback, + const std::optional& content) { + // Validate + if (!content.has_value()) { + LOG(INFO) << "null content"; + std::move(callback).Run({}); + return; + } + if (content->empty()) { + LOG(INFO) << "Empty content"; + std::move(callback).Run({}); + return; + } + LOG(INFO) << "Got a distill result of character length: " + << content->length(); + // Successful text extraction + auto result = chat::mojom::PageContent::New(); + result->type = std::move(chat::mojom::PageContentType::Text); + result->content = chat::mojom::PageContentData::NewContent(content.value()); + std::move(callback).Run(std::move(result)); } } // namespace ai_chat diff --git a/chrome/renderer/chat/page_content_extractor.h b/chrome/renderer/chat/page_content_extractor.h index 6f57436a6cd990..41b7e97bb5d944 100644 --- a/chrome/renderer/chat/page_content_extractor.h +++ b/chrome/renderer/chat/page_content_extractor.h @@ -27,7 +27,8 @@ class PageContentExtractor public content::RenderFrameObserverTracker { public: PageContentExtractor(content::RenderFrame* render_frame, - service_manager::BinderRegistry* registry); + service_manager::BinderRegistry* registry, + int32_t isolated_world_id); PageContentExtractor(const PageContentExtractor&) = delete; PageContentExtractor& operator=(const PageContentExtractor&) = delete; @@ -41,17 +42,26 @@ class PageContentExtractor chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) override; + // RenderFrameObserver implementation: + void OnDestruct() override; + void BindReceiver( - mojo::PendingReceiver receiver); + mojo::PendingReceiver receiver); - chat::mojo::Receiver receiver_{this}; + mojo::Receiver receiver_{this}; - base::WeakPtrFactory weak_ptr_factory_{this}; + int32_t isolated_world_id_; void ExtractPageText( content::RenderFrame* render_frame, int32_t isolated_world_id, base::OnceCallback& text)>); + + void OnPageTextExtracted( + chat::mojom::PageContentExtractor::ExtractPageContentCallback callback, + const std::optional& text); + + base::WeakPtrFactory weak_ptr_factory_{this}; }; } // namespace ai_chat diff --git a/chrome/renderer/chrome_content_renderer_client.cc b/chrome/renderer/chrome_content_renderer_client.cc index cc8af23c1e29de..a3dc3dfa0d6375 100644 --- a/chrome/renderer/chrome_content_renderer_client.cc +++ b/chrome/renderer/chrome_content_renderer_client.cc @@ -51,6 +51,7 @@ #include "chrome/renderer/benchmarking_extension.h" #include "chrome/renderer/browser_exposed_renderer_interfaces.h" #include "chrome/renderer/cart/commerce_hint_agent.h" +#include "chrome/renderer/chat/page_content_extractor.h" #include "chrome/renderer/chrome_content_settings_agent_delegate.h" #include "chrome/renderer/chrome_render_frame_observer.h" #include "chrome/renderer/chrome_render_thread_observer.h" @@ -63,6 +64,7 @@ #include "chrome/renderer/net_benchmarking_extension.h" #include "chrome/renderer/plugins/non_loadable_plugin_placeholder.h" #include "chrome/renderer/plugins/pdf_plugin_placeholder.h" +#include "chrome/renderer/process_state.h" #include "chrome/renderer/supervised_user/supervised_user_error_page_controller_delegate_impl.h" #include "chrome/renderer/trusted_vault_encryption_keys_extension.h" #include "chrome/renderer/url_loader_throttle_provider_impl.h" @@ -821,6 +823,13 @@ void ChromeContentRendererClient::RenderFrameCreated( new wallet::BoardingPassExtractor(render_frame, registry); } #endif + +#if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) + if (render_frame->IsMainFrame() && !IsIncognitoProcess()) { + new ai_chat::PageContentExtractor(render_frame, registry, + ISOLATED_WORLD_ID_CHROME_INTERNAL); + } +#endif } void ChromeContentRendererClient::WebViewCreated( From 5bffa22fcd6cfdf406c6e3e184ce28019c1442f3 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 13 Nov 2024 15:53:52 +0800 Subject: [PATCH 17/34] Refactor content extraction - use DVLOG --- .../chat/chat_side_panel_web_view.cc | 59 ++++++++++--------- .../chat/chat_side_panel_web_view.h | 4 +- .../ui/webui/side_panel/chat/chat_ui.h | 3 - .../renderer/chat/page_content_extractor.cc | 32 ++++------ 4 files changed, 44 insertions(+), 54 deletions(-) diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc index ca0d490df58ae9..e6d442acf317e2 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc @@ -48,7 +48,8 @@ ChatSidePanelWebView::ChatSidePanelWebView(Browser* browser, browser->profile(), IDS_AI_CHAT_TITLE, /*esc_closes_ui=*/false)), - browser_(browser) { + browser_(browser), + weak_ptr_factory_(this) { SetProperty(views::kElementIdentifierKey, kChatSidePanelWebViewElementId); browser_->tab_strip_model()->AddObserver(this); } @@ -90,40 +91,42 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( site_info->is_content_usable_in_conversations = false; } - //todo : to capture content here and hold it in SiteInfo - - auto* primary_rfh = contents->GetPrimaryMainFrame(); - DCHECK(primary_rfh->IsRenderFrameLive()); - - mojo::Remote extractor; - primary_rfh->GetRemoteInterfaces()->GetInterface( - extractor.BindNewPipeAndPassReceiver()); + if (primary_rfh->IsRenderFrameLive()) { + mojo::Remote extractor; + primary_rfh->GetRemoteInterfaces()->GetInterface( + extractor.BindNewPipeAndPassReceiver()); - extractor->ExtractPageContent( - base::BindOnce(&ChatSidePanelWebView::OnPageContentExtracted, - base::Unretained(this))); + extractor->ExtractPageContent( + base::BindOnce(&ChatSidePanelWebView::OnPageContentExtracted, + weak_ptr_factory_.GetWeakPtr())); - controller->GetAs()->SetSiteInfo(site_info.Clone()); + controller->GetAs()->SetSiteInfo(site_info.Clone()); + } } +base::WeakPtr ChatSidePanelWebView::GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); +} void ChatSidePanelWebView::OnPageContentExtracted(chat::mojom::PageContentPtr data) { - if (!data) { - VLOG(1) << __func__ << " no data."; - return; - } - DVLOG(1) << "OnPageContentExtracted: " << data.get(); - const bool is_video = base::Contains(kVideoPageContentTypes, data->type); - DVLOG(1) << "Is video? " << is_video; - // Handle text mode response - if (!is_video) { - DCHECK(data->content->is_content()); - auto content = data->content->get_content(); - DVLOG(1) << __func__ << ": Got content with char length of " - << content.length(); - LOG(INFO) << content; - return; + LOG(INFO) << "$$$$$#### log"; + if (!data) { + VLOG(1) << __func__ << " no data."; + return; + } + DVLOG(0) << "$$$$$$******************************OnPageContentExtracted: " + << data.get(); + const bool is_video = base::Contains(kVideoPageContentTypes, data->type); + DVLOG(1) << "Is video? " << is_video; + // Handle text mode response + if (!is_video) { + DCHECK(data->content->is_content()); + auto content = data->content->get_content(); + DVLOG(1) << __func__ << ": Got content with char length of " + << content.length(); + LOG(INFO) << content; + return; } LOG(INFO) << "content is url"; diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h index d2c2a6c7b32328..cf27c13f41572e 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h @@ -32,7 +32,6 @@ class ChatSidePanelWebView : public SidePanelWebUIViewT, ChatSidePanelWebView& operator=(const ChatSidePanelWebView&) = delete; ~ChatSidePanelWebView() override; - // TabStripModelObserver: void OnTabStripModelChanged( TabStripModel* tab_strip_model, const TabStripModelChange& change, @@ -44,10 +43,11 @@ class ChatSidePanelWebView : public SidePanelWebUIViewT, void UpdateActiveSiteInfo(content::WebContents* contents); void UpdateActiveWebContents(); + base::WeakPtr GetWeakPtr(); private: const raw_ptr browser_; - base::WeakPtrFactory weak_factory_{this}; + base::WeakPtrFactory weak_ptr_factory_{this}; void OnPageContentExtracted(chat::mojom::PageContentPtr data); }; diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h index 6d1ac9086bed95..1e9190e8ba43f2 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -13,7 +13,6 @@ #include "mojo/public/cpp/bindings/receiver.h" #include "ui/webui/mojo_web_ui_controller.h" -// Forward declaration so that config definition can come before controller. class ChatUI; class ChatUIConfig : public DefaultTopChromeWebUIConfig { @@ -36,8 +35,6 @@ class ChatUI : public TopChromeWebUIController, ~ChatUI() override; - // Instantiates the implementor of the mojom::PageHandlerFactory mojo - // interface passing the pending receiver that will be internally bound. void BindInterface( mojo::PendingReceiver receiver); diff --git a/chrome/renderer/chat/page_content_extractor.cc b/chrome/renderer/chat/page_content_extractor.cc index 52f5d1c18de575..1105435263d91d 100644 --- a/chrome/renderer/chat/page_content_extractor.cc +++ b/chrome/renderer/chat/page_content_extractor.cc @@ -136,7 +136,7 @@ base::WeakPtr PageContentExtractor::GetWeakPtr() { void PageContentExtractor::ExtractPageContent( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) { - LOG(INFO) << "AI Chat renderer has been asked for page content.xxx"; + DVLOG(0) << __func__ << "The current page will be extracted for Yep Chat."; ExtractPageText( render_frame(), isolated_world_id_, @@ -148,7 +148,6 @@ void PageContentExtractor::ExtractPageText( content::RenderFrame* render_frame, int32_t isolated_world_id, base::OnceCallback&)> callback) { - LOG(INFO) << "ExtractPageText()"; auto snapshotter = render_frame->CreateAXTreeSnapshotter( ui::AXMode::kWebContents | ui::AXMode::kHTML | ui::AXMode::kScreenReader); ui::AXTreeUpdate snapshot; @@ -182,35 +181,24 @@ void PageContentExtractor::ExtractPageText( std::string contents_text = base::UTF16ToUTF8(base::JoinString(text_node_contents, u" ")); - LOG(INFO) << contents_text; if (contents_text.empty()) { blink::WebLocalFrame* main_frame = render_frame->GetWebFrame(); v8::HandleScope handle_scope( main_frame->GetAgentGroupScheduler()->Isolate()); -// blink::WebScriptSource source = blink::WebScriptSource( -// blink::WebString::FromASCII("document.addEventListener('DOMContentLoaded', function() { document.body.innerText; });")); blink::WebScriptSource source = blink::WebScriptSource( blink::WebString::FromASCII("document.body.innerText")); auto on_script_executed = [](base::OnceCallback&)> callback, std::optional value, base::TimeTicks start_time) { - LOG(INFO) << "on_script_executed"; - LOG(INFO) << value->DebugString(); - if (value->is_string()) { - LOG(INFO) << "value is string ...."; + if (value && value->is_string()) { std::move(callback).Run(value->GetString()); return; } - LOG(INFO) << "Value is not string"; std::move(callback).Run({}); }; - if (render_frame->GetWebFrame()) { - LOG(INFO) << "has local web frame"; - } - render_frame->GetWebFrame()->RequestExecuteScript( isolated_world_id, UNSAFE_TODO(base::make_span(&source, 1u)), blink::mojom::UserActivationOption::kDoNotActivate, @@ -220,7 +208,6 @@ void PageContentExtractor::ExtractPageText( blink::BackForwardCacheAware::kAllow, blink::mojom::WantResultOption::kWantResult, blink::mojom::PromiseResultOption::kAwait); - LOG(INFO) << "called RequestExecuteScript"; } else { std::move(callback).Run(contents_text); } @@ -228,7 +215,7 @@ void PageContentExtractor::ExtractPageText( void PageContentExtractor::BindReceiver( mojo::PendingReceiver receiver) { - VLOG(1) << "AIChat PageContentExtractor handler bound."; + VLOG(1) << "Yep Chat PageContentExtractor handler bound."; receiver_.reset(); receiver_.Bind(std::move(receiver)); } @@ -236,19 +223,22 @@ void PageContentExtractor::BindReceiver( void PageContentExtractor::OnPageTextExtracted( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback, const std::optional& content) { - // Validate if (!content.has_value()) { - LOG(INFO) << "null content"; + DVLOG(0) << "Extracted content is null."; std::move(callback).Run({}); return; } + if (content->empty()) { - LOG(INFO) << "Empty content"; + DVLOG(0) << "Extracted content is empty."; std::move(callback).Run({}); return; } - LOG(INFO) << "Got a distill result of character length: " - << content->length(); + + DVLOG(0) << "--------> The content of the current opening page."; + DVLOG(0) << content.value(); + DVLOG(0) << "The length of extracted content: " << content->length(); + // Successful text extraction auto result = chat::mojom::PageContent::New(); result->type = std::move(chat::mojom::PageContentType::Text); From 06dc0ebc6acb7bc7c4844b2035162c47818ec47e Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Thu, 14 Nov 2024 09:24:31 +0800 Subject: [PATCH 18/34] Refactor content extraction - extraction called in page handler (Mojo pipe issue needs to be fixed though) --- .../chat/chat_side_panel_web_view.cc | 23 +++-- .../side_panel/chat/chat_page_handler.cc | 61 ++++++++----- .../webui/side_panel/chat/chat_page_handler.h | 45 +++++----- .../ui/webui/side_panel/chat/chat_ui.cc | 85 +++++++++++++++++-- .../ui/webui/side_panel/chat/chat_ui.h | 9 ++ .../renderer/chat/page_content_extractor.cc | 27 +++--- 6 files changed, 176 insertions(+), 74 deletions(-) diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc index e6d442acf317e2..08f198af4e68d1 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc @@ -91,18 +91,17 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( site_info->is_content_usable_in_conversations = false; } - auto* primary_rfh = contents->GetPrimaryMainFrame(); - if (primary_rfh->IsRenderFrameLive()) { - mojo::Remote extractor; - primary_rfh->GetRemoteInterfaces()->GetInterface( - extractor.BindNewPipeAndPassReceiver()); - - extractor->ExtractPageContent( - base::BindOnce(&ChatSidePanelWebView::OnPageContentExtracted, - weak_ptr_factory_.GetWeakPtr())); - - controller->GetAs()->SetSiteInfo(site_info.Clone()); - } + // auto* primary_rfh = contents->GetPrimaryMainFrame(); + // if (primary_rfh->IsRenderFrameLive()) { + // mojo::Remote extractor; + // primary_rfh->GetRemoteInterfaces()->GetInterface( + // extractor.BindNewPipeAndPassReceiver()); + // + // extractor->ExtractPageContent( + // base::BindOnce(&ChatSidePanelWebView::OnPageContentExtracted, + // weak_ptr_factory_.GetWeakPtr())); + + controller->GetAs()->SetSiteInfo(site_info.Clone()); } base::WeakPtr ChatSidePanelWebView::GetWeakPtr() { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 8348ec04bca5ac..d302a82313f9b5 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -27,16 +27,18 @@ constexpr auto kVideoPageContentTypes = chat::mojom::PageContentType::VideoTranscriptVTT}); ChatPageHandler::ChatPageHandler( - mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui, - content::WebUI* web_ui) - : receiver_(this, std::move(receiver)), - page_(std::move(page)), - chat_ui_(chat_ui), - web_ui_(web_ui), - web_contents_(web_ui->GetWebContents()), - profile_(Profile::FromWebUI(web_ui)) {} + mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui, + content::WebContents* owner_web_contents, + content::WebContents* chat_context_web_contents) + : receiver_(this, std::move(receiver)), + page_(std::move(page)), + chat_ui_(chat_ui), + owner_web_contents_(owner_web_contents), + chat_context_web_contents_(chat_context_web_contents), + profile_(Profile::FromWebUI(web_ui)) {} ChatPageHandler::~ChatPageHandler() = default; @@ -61,16 +63,23 @@ void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { // todo: to remove probably, not correct url and title here void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { - chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); - site_info->title = base::UTF16ToUTF8(web_contents_->GetTitle()); - const GURL gurl = web_contents_->GetLastCommittedURL(); + DCHECK(chat_context_web_contents_); + chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); + site_info->url = ""; + site_info->is_content_usable_in_conversations = false; + + if (chat_context_web_contents_) { + site_info->title = + base::UTF16ToUTF8(chat_context_web_contents_->GetTitle()); + const GURL gurl = chat_context_web_contents_->GetLastCommittedURL(); if (gurl.SchemeIsHTTPOrHTTPS()) { - site_info->url = gurl.spec(); - site_info->is_content_usable_in_conversations = true; + site_info->url = gurl.spec(); + site_info->is_content_usable_in_conversations = true; } else { - site_info->url = ""; - site_info->is_content_usable_in_conversations = false; + site_info->url = ""; + site_info->is_content_usable_in_conversations = false; } + } std::move(callback).Run(site_info.Clone()); } @@ -117,16 +126,22 @@ void ChatPageHandler::SubmitQueryCompletedCallback( LOG(INFO) << result.has_value(); } +base::WeakPtr ChatPageHandler::GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); +} + void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { if (page_.is_bound()) { LOG(INFO) << action_type; if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { LOG(INFO) << "ActionType: Summarize_page"; - LOG(INFO) << "****" << base::UTF16ToUTF8(web_contents_->GetTitle()); - const GURL gurl = web_contents_->GetLastCommittedURL(); - LOG(INFO) << "****" << gurl.spec(); - auto* primary_rfh = web_contents_->GetPrimaryMainFrame(); + LOG(INFO) << "****" + << base::UTF16ToUTF8( + chat_context_web_contents_->GetTitle()); + const GURL gurl = chat_context_web_contents_->GetLastCommittedURL(); + LOG(INFO) << "****" << gurl.spec(); + auto* primary_rfh = chat_context_web_contents_->GetPrimaryMainFrame(); DCHECK(primary_rfh->IsRenderFrameLive()); mojo::Remote extractor; @@ -135,7 +150,7 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { extractor->ExtractPageContent( base::BindOnce(&ChatPageHandler::OnPageContentExtracted, - base::Unretained(this))); + weak_ptr_factory_.GetWeakPtr())); } else { // todo: to implement for other action types later @@ -153,7 +168,7 @@ void ChatPageHandler::OnPageContentExtracted(chat::mojom::PageContentPtr data) { VLOG(1) << __func__ << " no data."; return; } - DVLOG(1) << "OnPageContentExtracted: " << data.get(); + DVLOG(1) << "##################OnPageContentExtracted: " << data.get(); const bool is_video = base::Contains(kVideoPageContentTypes, data->type); DVLOG(1) << "Is video? " << is_video; // Handle text mode response diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 01d5faaa670271..b40d6cc68dbb65 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -25,38 +25,43 @@ namespace content { class ChatPageHandler : public chat::mojom::PageHandler { public: - ChatPageHandler(mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui, - content::WebUI* web_ui); + ChatPageHandler(mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui, + content::WebContents* owner_web_contents, + content::WebContents* chat_context_web_contents); - ChatPageHandler(const ChatPageHandler&) = delete; - ChatPageHandler& operator=(const ChatPageHandler&) = delete; + ChatPageHandler(const ChatPageHandler&) = delete; + ChatPageHandler& operator=(const ChatPageHandler&) = delete; - ~ChatPageHandler() override; + ~ChatPageHandler() override; - void GetSiteInfo(GetSiteInfoCallback callback) override; - void GetActionList(GetActionListCallback callback) override; - void SubmitAction(chat::mojom::ActionType action_type) override; - void SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) override; - void ShowUI() override; - void CloseUI() override; + void GetSiteInfo(GetSiteInfoCallback callback) override; + void GetActionList(GetActionListCallback callback) override; + void SubmitAction(chat::mojom::ActionType action_type) override; + void SubmitQuery(chat::mojom::ActionType action_type, + const std::string& query) override; + void ShowUI() override; + void CloseUI() override; - void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); - void SubmitQueryCallback(std::string completion); - void SubmitQueryCompletedCallback( - base::expected result); + void SubmitQueryCallback(std::string completion); + void SubmitQueryCompletedCallback( + base::expected result); + + base::WeakPtr GetWeakPtr(); private: mojo::Receiver receiver_; mojo::Remote page_; const raw_ptr chat_ui_; - const raw_ptr web_ui_; - raw_ptr web_contents_; + raw_ptr owner_web_contents_ = nullptr; + raw_ptr chat_context_web_contents_ = nullptr; const raw_ptr profile_; std::unique_ptr api_client_ = nullptr; - + base::WeakPtrFactory weak_ptr_factory_{this}; void OnPageContentExtracted(chat::mojom::PageContentPtr data); }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc index 26ef74af39fc91..525ebd9c2ae1a3 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc @@ -1,22 +1,65 @@ #include "chat_ui.h" + #include "chat_page_handler.h" +#include "chrome/browser/profiles/profile.h" +#include "chrome/browser/ui/browser.h" +#include "chrome/browser/ui/browser_window.h" +#include "chrome/browser/ui/browser_window/public/browser_window_features.h" +#include "chrome/browser/ui/views/frame/browser_view.h" +#include "chrome/browser/ui/views/side_panel/side_panel_ui.h" #include "chrome/browser/ui/webui/webui_util.h" #include "chrome/common/webui_url_constants.h" -#include "content/public/common/url_constants.h" +#include "chrome/grit/generated_resources.h" +#include "chrome/grit/side_panel_chat_resources.h" +#include "chrome/grit/side_panel_chat_resources_map.h" #include "content/public/browser/browser_context.h" #include "content/public/browser/web_contents.h" #include "content/public/browser/web_ui.h" -#include "content/public/browser/web_ui_data_source.h" #include "content/public/browser/web_ui_controller.h" +#include "content/public/browser/web_ui_data_source.h" #include "content/public/browser/webui_config.h" -#include "chrome/grit/side_panel_chat_resources.h" -#include "chrome/grit/side_panel_chat_resources_map.h" +#include "content/public/common/url_constants.h" #include "ui/webui/mojo_web_ui_controller.h" -#include "chrome/browser/profiles/profile.h" -#include "chrome/grit/generated_resources.h" #pragma allow_unsafe_buffers +namespace { + +#if BUILDFLAG(IS_ANDROID) +content::WebContents* GetActiveWebContents(content::BrowserContext* context) { + auto tab_models = TabModelList::models(); + auto iter = base::ranges::find_if( + tab_models, [](const auto& model) { return model->IsActiveModel(); }); + if (iter == tab_models.end()) { + return nullptr; + } + + auto* active_contents = (*iter)->GetActiveWebContents(); + if (!active_contents) { + return nullptr; + } + DCHECK_EQ(active_contents->GetBrowserContext(), context); + return active_contents; +} +#endif + +Browser* GetBrowserForWebContents(content::WebContents* web_contents) { + if (!web_contents) { + return nullptr; + } + + auto* browser_window = + BrowserWindow::FindBrowserWindowWithWebContents(web_contents); + auto* browser_view = static_cast(browser_window); + if (!browser_view) { + return nullptr; + } + + return browser_view->browser(); +} + +} // namespace + ChatUI::ChatUI(content::WebUI* web_ui) : TopChromeWebUIController(web_ui) { Profile* const profile = Profile::FromWebUI(web_ui); @@ -49,9 +92,33 @@ void ChatUI::BindInterface( void ChatUI::CreatePageHandler( mojo::PendingRemote page, mojo::PendingReceiver receiver) { - DCHECK(page); - page_handler_ = std::make_unique( - std::move(receiver), std::move(page), this, web_ui()); + DCHECK(page); + // ShowUI() is called before creating the PageHandler. + // This ensures the WebContents is added to a Browser, + // allowing us to provide the Browser reference to the PageHandler. + if (embedder_) { + embedder_->ShowUI(); + } + + content::WebContents* web_contents = nullptr; +#if !BUILDFLAG(IS_ANDROID) + Browser* browser = GetBrowserForWebContents(web_ui()->GetWebContents()); + if (!browser) { + return; + } + + TabStripModel* tab_strip_model = browser->tab_strip_model(); + DCHECK(tab_strip_model); + web_contents = tab_strip_model->GetActiveWebContents(); +#else + web_contents = GetActiveWebContents(profile_); +#endif + if (web_contents == web_ui()->GetWebContents()) { + web_contents = nullptr; + } + page_handler_ = std::make_unique( + std::move(receiver), std::move(page), this, web_ui(), + web_ui()->GetWebContents(), web_contents); } void ChatUI::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h index 1e9190e8ba43f2..bf403ab743e2d4 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -38,6 +38,13 @@ class ChatUI : public TopChromeWebUIController, void BindInterface( mojo::PendingReceiver receiver); + // Set by WebUIContentsWrapperT. TopChromeWebUIController provides default + // implementation for this but we don't use it. + void set_embedder( + base::WeakPtr embedder) { + embedder_ = embedder; + } + static constexpr std::string GetWebUIName() { return "Chat"; } @@ -53,6 +60,8 @@ class ChatUI : public TopChromeWebUIController, mojo::Receiver page_factory_receiver_{ this}; + base::WeakPtr embedder_; + WEB_UI_CONTROLLER_TYPE_DECL(); }; #endif //CHROMIUM_CHAT_UI_H diff --git a/chrome/renderer/chat/page_content_extractor.cc b/chrome/renderer/chat/page_content_extractor.cc index 1105435263d91d..06a89577c6a3f8 100644 --- a/chrome/renderer/chat/page_content_extractor.cc +++ b/chrome/renderer/chat/page_content_extractor.cc @@ -134,14 +134,29 @@ base::WeakPtr PageContentExtractor::GetWeakPtr() { return weak_ptr_factory_.GetWeakPtr(); } +void PageContentExtractor::BindReceiver( + mojo::PendingReceiver receiver) { + VLOG(1) << "Yep Chat PageContentExtractor handler bound."; + receiver_.reset(); + receiver_.Bind(std::move(receiver)); +} + void PageContentExtractor::ExtractPageContent( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) { DVLOG(0) << __func__ << "The current page will be extracted for Yep Chat."; + // auto result = chat::mojom::PageContent::New(); + // result->type = std::move(chat::mojom::PageContentType::Text); + // result->content = + // chat::mojom::PageContentData::NewContent("#######################33"); + // DCHECK(callback); + // std::move(callback).Run(std::move(result)); ExtractPageText( render_frame(), isolated_world_id_, base::BindOnce(&PageContentExtractor::OnPageTextExtracted, weak_ptr_factory_.GetWeakPtr(), std::move(callback))); + + DVLOG(0) << "End of " << __func__ << "\n"; } void PageContentExtractor::ExtractPageText( @@ -193,10 +208,9 @@ void PageContentExtractor::ExtractPageText( std::optional value, base::TimeTicks start_time) { if (value && value->is_string()) { std::move(callback).Run(value->GetString()); - return; + } else { + std::move(callback).Run({}); } - - std::move(callback).Run({}); }; render_frame->GetWebFrame()->RequestExecuteScript( @@ -213,13 +227,6 @@ void PageContentExtractor::ExtractPageText( } } -void PageContentExtractor::BindReceiver( - mojo::PendingReceiver receiver) { - VLOG(1) << "Yep Chat PageContentExtractor handler bound."; - receiver_.reset(); - receiver_.Bind(std::move(receiver)); -} - void PageContentExtractor::OnPageTextExtracted( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback, const std::optional& content) { From cf7ec77a6499d837a1feae3ed3e1a17748a062e3 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 15 Nov 2024 15:56:48 +0800 Subject: [PATCH 19/34] Extract content from the active page for summarization and display results in the debug console --- .../chat/chat_side_panel_coordinator.h | 1 - .../chat/chat_side_panel_web_view.cc | 38 ------ .../chat/chat_side_panel_web_view.h | 2 - .../chat/api/completion_api_client.cc | 12 +- .../chat/api/completion_api_client.h | 1 + .../side_panel/chat/chat_page_handler.cc | 115 +++++++++++++----- .../webui/side_panel/chat/chat_page_handler.h | 2 +- .../common/chat/page_content_extractor.mojom | 7 ++ .../renderer/chat/page_content_extractor.cc | 17 +-- 9 files changed, 106 insertions(+), 89 deletions(-) diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h index a068ee9d28fb48..a8bb080af7c68b 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_coordinator.h @@ -23,7 +23,6 @@ class ChatSidePanelCoordinator private: friend class BrowserUserData; - std::unique_ptr CreateChatWebView(); BROWSER_USER_DATA_KEY_DECL(); diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc index 08f198af4e68d1..c355e5c5c7a113 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc @@ -28,11 +28,6 @@ #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" -constexpr auto kVideoPageContentTypes = - base::MakeFixedFlatSet( - {chat::mojom::PageContentType::VideoTranscriptYouTube, - chat::mojom::PageContentType::VideoTranscriptVTT}); - using SidePanelWebUIViewT_ChatUI = SidePanelWebUIViewT; BEGIN_TEMPLATE_METADATA(SidePanelWebUIViewT_ChatUI, SidePanelWebUIViewT) END_METADATA @@ -91,16 +86,6 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( site_info->is_content_usable_in_conversations = false; } - // auto* primary_rfh = contents->GetPrimaryMainFrame(); - // if (primary_rfh->IsRenderFrameLive()) { - // mojo::Remote extractor; - // primary_rfh->GetRemoteInterfaces()->GetInterface( - // extractor.BindNewPipeAndPassReceiver()); - // - // extractor->ExtractPageContent( - // base::BindOnce(&ChatSidePanelWebView::OnPageContentExtracted, - // weak_ptr_factory_.GetWeakPtr())); - controller->GetAs()->SetSiteInfo(site_info.Clone()); } @@ -108,29 +93,6 @@ base::WeakPtr ChatSidePanelWebView::GetWeakPtr() { return weak_ptr_factory_.GetWeakPtr(); } -void ChatSidePanelWebView::OnPageContentExtracted(chat::mojom::PageContentPtr data) { - LOG(INFO) << "$$$$$#### log"; - if (!data) { - VLOG(1) << __func__ << " no data."; - return; - } - DVLOG(0) << "$$$$$$******************************OnPageContentExtracted: " - << data.get(); - const bool is_video = base::Contains(kVideoPageContentTypes, data->type); - DVLOG(1) << "Is video? " << is_video; - // Handle text mode response - if (!is_video) { - DCHECK(data->content->is_content()); - auto content = data->content->get_content(); - DVLOG(1) << __func__ << ": Got content with char length of " - << content.length(); - LOG(INFO) << content; - return; - } - - LOG(INFO) << "content is url"; -} - void ChatSidePanelWebView::UpdateActiveWebContents() { UpdateActiveSiteInfo(browser_->tab_strip_model()->GetActiveWebContents()); } diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h index cf27c13f41572e..51878db9f7707d 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.h @@ -48,7 +48,5 @@ class ChatSidePanelWebView : public SidePanelWebUIViewT, private: const raw_ptr browser_; base::WeakPtrFactory weak_ptr_factory_{this}; - - void OnPageContentExtracted(chat::mojom::PageContentPtr data); }; #endif // CHROMIUM_CHAT_SIDE_PANEL_WEB_VIEW_H diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc index 4fa438376f4151..5ae1cab1372cfc 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc @@ -118,6 +118,7 @@ void CompletionApiClient::OnQueryDataReceived( if (!result.has_value() || !result->is_dict()) { return; } + const base::Value::List* list = result->GetDict().FindList("choices"); if (list) { for (const auto& item : *list) { @@ -126,7 +127,8 @@ void CompletionApiClient::OnQueryDataReceived( if (delta) { const std::string* content = delta->FindString("content"); if (content) { - LOG(INFO) << "delta content: " << *content; + //LOG(INFO) << "delta content: " << *content; + entire_completion_result.push_back(*content); } } } @@ -156,8 +158,14 @@ void CompletionApiClient::OnQueryCompleted( completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); } } + std::string entire_message; + for (const auto& s : entire_completion_result) { + entire_message += s; + } + DVLOG(0) << std::endl << std::endl << entire_message << std::endl << std::endl; - // std::move(callback).Run(base::ok(std::move(completion))); + entire_completion_result.clear(); + // std::move(callback).Run(base::ok(std::move(completion))); return; } diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h index c53f3a93808f7b..5dd09ae5666c51 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h @@ -52,6 +52,7 @@ class CompletionApiClient { APIRequestResult result); api_request_helper::APIRequestHelper api_request_helper_; + std::vector entire_completion_result; base::WeakPtrFactory weak_ptr_factory_{this}; }; diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index d302a82313f9b5..2725f7a90cd1f8 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -21,11 +21,77 @@ #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" +namespace { constexpr auto kVideoPageContentTypes = base::MakeFixedFlatSet( {chat::mojom::PageContentType::VideoTranscriptYouTube, chat::mojom::PageContentType::VideoTranscriptVTT}); +using ExtractPageContentCallback = + base::OnceCallback; + +class PageContentExtractorHelper { + public: + PageContentExtractorHelper() {} + + void Start(mojo::Remote content_extractor, + ExtractPageContentCallback callback) { + content_extractor_ = std::move(content_extractor); + if (!content_extractor_) { + DeleteSelf(); + return; + } + + // Ref: + // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks + // Once a `mojo::Remote` is destroyed, it is guaranteed that pending + // callbacks as well as the connection error handler (if registered) won't + // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed that + // no more method calls are dispatched to the implementation and the + // connection error handler (if registered) won't be called. + content_extractor_.set_disconnect_handler(base::BindOnce( + &PageContentExtractorHelper::DeleteSelf, base::Unretained(this))); + content_extractor_->ExtractPageContent( + base::BindOnce(&PageContentExtractorHelper::OnPageContentExtracted, + base::Unretained(this), std::move(callback))); + } + + void OnPageContentExtracted(ExtractPageContentCallback callback, + chat::mojom::PageContentPtr data) { + if (!data) { + DVLOG(0) << __func__ << " no extracted page content."; + SendResultAndDeleteSelf(std::move(callback)); + return; + } + + DVLOG(1) << "OnTabContentResult: " << data.get(); + const bool is_video = base::Contains(kVideoPageContentTypes, data->type); + DVLOG(1) << "Is video? " << is_video; + + if (!is_video) { + DCHECK(data->content->is_content()); + auto content = data->content->get_content(); + DVLOG(1) << __func__ << ": Got content with char length of " + << content.length(); + SendResultAndDeleteSelf(std::move(callback), content); + return; + } + + SendResultAndDeleteSelf(std::move(callback)); + } + + private: + void DeleteSelf() { delete this; } + void SendResultAndDeleteSelf(ExtractPageContentCallback callback, + std::string content = "") { + std::move(callback).Run(content); + delete this; + } + mojo::Remote content_extractor_; + base::WeakPtrFactory weak_ptr_factory_{this}; +}; +} // namespace + ChatPageHandler::ChatPageHandler( mojo::PendingReceiver receiver, mojo::PendingRemote page, @@ -38,7 +104,13 @@ ChatPageHandler::ChatPageHandler( chat_ui_(chat_ui), owner_web_contents_(owner_web_contents), chat_context_web_contents_(chat_context_web_contents), - profile_(Profile::FromWebUI(web_ui)) {} + profile_(Profile::FromWebUI(web_ui)) { + scoped_refptr url_loader_factory = + profile_->GetDefaultStoragePartition() + ->GetURLLoaderFactoryForBrowserProcess(); + api_client_ = + std::make_unique(std::move(url_loader_factory)); +} ChatPageHandler::~ChatPageHandler() = default; @@ -61,7 +133,6 @@ void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { } } -// todo: to remove probably, not correct url and title here void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { DCHECK(chat_context_web_contents_); chat::mojom::SiteInfoPtr site_info = chat::mojom::SiteInfo::New(); @@ -141,6 +212,7 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { chat_context_web_contents_->GetTitle()); const GURL gurl = chat_context_web_contents_->GetLastCommittedURL(); LOG(INFO) << "****" << gurl.spec(); + auto* primary_rfh = chat_context_web_contents_->GetPrimaryMainFrame(); DCHECK(primary_rfh->IsRenderFrameLive()); @@ -148,9 +220,11 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { primary_rfh->GetRemoteInterfaces()->GetInterface( extractor.BindNewPipeAndPassReceiver()); - extractor->ExtractPageContent( + auto* extractor_helper = new PageContentExtractorHelper(); + extractor_helper->Start( + std::move(extractor), base::BindOnce(&ChatPageHandler::OnPageContentExtracted, - weak_ptr_factory_.GetWeakPtr())); + base::Unretained(this))); } else { // todo: to implement for other action types later @@ -163,34 +237,15 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { } } -void ChatPageHandler::OnPageContentExtracted(chat::mojom::PageContentPtr data) { - if (!data) { - VLOG(1) << __func__ << " no data."; - return; - } - DVLOG(1) << "##################OnPageContentExtracted: " << data.get(); - const bool is_video = base::Contains(kVideoPageContentTypes, data->type); - DVLOG(1) << "Is video? " << is_video; - // Handle text mode response - if (!is_video) { - DCHECK(data->content->is_content()); - auto content = data->content->get_content(); - DVLOG(1) << __func__ << ": Got content with char length of " - << content.length(); - LOG(INFO) << content; - return; - } - - LOG(INFO) << "content is url"; +void ChatPageHandler::OnPageContentExtracted(std::string content) { + // todo: to put prompt in resource file + api_client_->QueryPrompt( + "Provide a brief summary of the key takeaways for the following:" + + content, + base::NullCallback(), base::NullCallback()); } void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { - scoped_refptr url_loader_factory = - profile_->GetDefaultStoragePartition() - ->GetURLLoaderFactoryForBrowserProcess(); - api_client_ = - std::make_unique(std::move(url_loader_factory)); - - // Fix: use proper callback + // todo: use proper callback api_client_->QueryPrompt(query, base::NullCallback(), base::NullCallback()); } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index b40d6cc68dbb65..1de83090b5c64b 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -62,6 +62,6 @@ class ChatPageHandler : public chat::mojom::PageHandler { const raw_ptr profile_; std::unique_ptr api_client_ = nullptr; base::WeakPtrFactory weak_ptr_factory_{this}; - void OnPageContentExtracted(chat::mojom::PageContentPtr data); + void OnPageContentExtracted(std::string content); }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/common/chat/page_content_extractor.mojom b/chrome/common/chat/page_content_extractor.mojom index a5e31c80c4542f..480c7e06bb23e6 100644 --- a/chrome/common/chat/page_content_extractor.mojom +++ b/chrome/common/chat/page_content_extractor.mojom @@ -22,3 +22,10 @@ struct PageContent { interface PageContentExtractor { ExtractPageContent() => (PageContent? page_content); }; + +// Enables the renderer to signal content updates to the browser process. +interface PageContentExtractorHost { + // This is just to notify that the content has changed. + // The current page's content will be extracted via PageContentExtractor.ExtractPageContent. + OnInterceptedPageContentChanged(); +}; diff --git a/chrome/renderer/chat/page_content_extractor.cc b/chrome/renderer/chat/page_content_extractor.cc index 06a89577c6a3f8..eef73cd394eef5 100644 --- a/chrome/renderer/chat/page_content_extractor.cc +++ b/chrome/renderer/chat/page_content_extractor.cc @@ -143,20 +143,11 @@ void PageContentExtractor::BindReceiver( void PageContentExtractor::ExtractPageContent( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) { - DVLOG(0) << __func__ << "The current page will be extracted for Yep Chat."; - - // auto result = chat::mojom::PageContent::New(); - // result->type = std::move(chat::mojom::PageContentType::Text); - // result->content = - // chat::mojom::PageContentData::NewContent("#######################33"); - // DCHECK(callback); - // std::move(callback).Run(std::move(result)); + DVLOG(0) << __func__ << " The current page will be extracted for Yep Chat."; ExtractPageText( render_frame(), isolated_world_id_, base::BindOnce(&PageContentExtractor::OnPageTextExtracted, weak_ptr_factory_.GetWeakPtr(), std::move(callback))); - - DVLOG(0) << "End of " << __func__ << "\n"; } void PageContentExtractor::ExtractPageText( @@ -223,7 +214,7 @@ void PageContentExtractor::ExtractPageText( blink::mojom::WantResultOption::kWantResult, blink::mojom::PromiseResultOption::kAwait); } else { - std::move(callback).Run(contents_text); + std::move(callback).Run(std::move(contents_text)); } } @@ -242,10 +233,6 @@ void PageContentExtractor::OnPageTextExtracted( return; } - DVLOG(0) << "--------> The content of the current opening page."; - DVLOG(0) << content.value(); - DVLOG(0) << "The length of extracted content: " << content->length(); - // Successful text extraction auto result = chat::mojom::PageContent::New(); result->type = std::move(chat::mojom::PageContentType::Text); From ceba18eca7a7bf8e87c72389decd26e9f5a8b99a Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Mon, 18 Nov 2024 08:56:39 +0800 Subject: [PATCH 20/34] Removed start-up page app --- chrome/browser/resources/new_tab_page/new_tab_page.html | 1 - 1 file changed, 1 deletion(-) diff --git a/chrome/browser/resources/new_tab_page/new_tab_page.html b/chrome/browser/resources/new_tab_page/new_tab_page.html index 6e261416165ac0..072e28a5e6dca1 100644 --- a/chrome/browser/resources/new_tab_page/new_tab_page.html +++ b/chrome/browser/resources/new_tab_page/new_tab_page.html @@ -26,7 +26,6 @@ - From ea48bdbaebe82ccf0f18a5bd230f0dcc03563fd4 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 19 Nov 2024 18:36:24 +0800 Subject: [PATCH 21/34] Observe tab change to provide chat context --- .../browser/chrome_content_browser_client.cc | 20 ++ ...ontent_browser_client_receiver_bindings.cc | 13 + chrome/browser/ui/BUILD.gn | 2 + .../side_panel/chat/chat_context_observer.cc | 260 ++++++++++++++++++ .../side_panel/chat/chat_context_observer.h | 96 +++++++ .../side_panel/chat/chat_page_handler.cc | 204 ++++++++------ .../webui/side_panel/chat/chat_page_handler.h | 102 ++++--- 7 files changed, 572 insertions(+), 125 deletions(-) create mode 100644 chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc create mode 100644 chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc index 5c0a66583d4153..2b33fde759891d 100644 --- a/chrome/browser/chrome_content_browser_client.cc +++ b/chrome/browser/chrome_content_browser_client.cc @@ -420,6 +420,10 @@ #include "url/origin.h" #include "url/third_party/mozilla/url_parse.h" #include "url/url_constants.h" +// #include "chrome/common/chat/page_content_extractor.mojom.h" +// #include "chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h" +// #include +// "third_party/blink/public/common/associated_interfaces/associated_interface_registry.h" #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) #include "chrome/browser/enterprise/platform_auth/platform_auth_navigation_throttle.h" @@ -8858,3 +8862,19 @@ base::ReadOnlySharedMemoryRegion ChromeContentBrowserClient::GetGlobalPerformanceScenarioRegion() { return performance_manager::GetGlobalSharedScenarioRegion(); } + +// void ChromeContentBrowserClient:: +// RegisterAssociatedInterfaceBindersForRenderFrameHost( +// content::RenderFrameHost& render_frame_host, +// blink::AssociatedInterfaceRegistry& associated_registry) { +// // AI Chat page content extraction renderer -> browser interface +// associated_registry.AddInterface( +// base::BindRepeating( +// [](content::RenderFrameHost* render_frame_host, +// mojo::PendingAssociatedReceiver< +// chat::mojom::PageContentExtractorHost> receiver) { +// ai_chat::ChatContextObserver::BindPageContentExtractorHost( +// render_frame_host, std::move(receiver)); +// }, +// &render_frame_host)); +// } diff --git a/chrome/browser/chrome_content_browser_client_receiver_bindings.cc b/chrome/browser/chrome_content_browser_client_receiver_bindings.cc index 0340dc97e87cdb..e266624e8a230c 100644 --- a/chrome/browser/chrome_content_browser_client_receiver_bindings.cc +++ b/chrome/browser/chrome_content_browser_client_receiver_bindings.cc @@ -25,7 +25,9 @@ #include "chrome/browser/signin/google_accounts_private_api_host.h" #include "chrome/browser/supervised_user/supervised_user_navigation_observer.h" #include "chrome/browser/trusted_vault/trusted_vault_encryption_keys_tab_helper.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h" #include "chrome/common/buildflags.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" #include "chrome/common/chrome_features.h" #include "components/autofill/content/browser/content_autofill_driver_factory.h" #include "components/content_capture/browser/onscreen_content_provider.h" @@ -627,6 +629,17 @@ void ChromeContentBrowserClient:: std::move(receiver), render_frame_host); }, &render_frame_host)); + + // AI Chat page content extraction renderer -> browser interface + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver< + chat::mojom::PageContentExtractorHost> receiver) { + ai_chat::ChatContextObserver::BindPageContentExtractorHost( + render_frame_host, std::move(receiver)); + }, + &render_frame_host)); } void ChromeContentBrowserClient::BindGpuHostReceiver( diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index bca3d2514f32fb..0d0e62820a153a 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -1569,6 +1569,8 @@ static_library("ui") { "webui/side_panel/chat/api/api_request_helper.h", "webui/side_panel/chat/api/completion_api_client.cc", "webui/side_panel/chat/api/completion_api_client.h", + "webui/side_panel/chat/chat_context_observer.cc", + "webui/side_panel/chat/chat_context_observer.h", "webui/side_panel/chat/chat_page_handler.cc", "webui/side_panel/chat/chat_page_handler.h", "webui/side_panel/chat/chat_ui.cc", diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc new file mode 100644 index 00000000000000..aadd2c17670e4c --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc @@ -0,0 +1,260 @@ +#include "chat_context_observer.h" + +#include +#include +#include +#include + +#include "base/containers/fixed_flat_set.h" +#include "base/functional/bind.h" +#include "base/memory/weak_ptr.h" +#include "base/ranges/algorithm.h" +#include "base/strings/string_util.h" +#include "chrome/grit/generated_resources.h" +#include "chrome/renderer/chat/page_content_extractor.h" +#include "components/strings/grit/components_strings.h" +#include "content/public/browser/browser_accessibility_state.h" +#include "content/public/browser/browser_context.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/navigation_details.h" +#include "content/public/browser/navigation_entry.h" +#include "content/public/browser/scoped_accessibility_mode.h" +#include "content/public/browser/storage_partition.h" +#include "content/public/browser/web_contents.h" +#include "content/public/browser/web_ui.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/service_manager/public/cpp/interface_provider.h" +#include "ui/accessibility/ax_mode.h" +#include "ui/accessibility/ax_updates_and_events.h" +#include "ui/base/l10n/l10n_util.h" +#include "url/gurl.h" + +namespace ai_chat { + +namespace { +constexpr auto kVideoPageContentTypes = + base::MakeFixedFlatSet( + {chat::mojom::PageContentType::VideoTranscriptYouTube, + chat::mojom::PageContentType::VideoTranscriptVTT}); + +using ExtractPageContentCallback = + ChatContextObserver::ExtractPageContentCallback; + +class PageContentExtractorHelper { + public: + PageContentExtractorHelper() {} + + void Start(mojo::Remote content_extractor, + ExtractPageContentCallback callback) { + content_extractor_ = std::move(content_extractor); + if (!content_extractor_) { + DeleteSelf(); + return; + } + + // Ref: + // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks + // Once a `mojo::Remote` is destroyed, it is guaranteed that pending + // callbacks as well as the connection error handler (if registered) won't + // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed that + // no more method calls are dispatched to the implementation and the + // connection error handler (if registered) won't be called. + content_extractor_.set_disconnect_handler(base::BindOnce( + &PageContentExtractorHelper::DeleteSelf, base::Unretained(this))); + content_extractor_->ExtractPageContent( + base::BindOnce(&PageContentExtractorHelper::OnPageContentExtracted, + base::Unretained(this), std::move(callback))); + } + + void OnPageContentExtracted(ExtractPageContentCallback callback, + chat::mojom::PageContentPtr data) { + if (!data) { + DVLOG(0) << __func__ << " no extracted page content."; + SendResultAndDeleteSelf(std::move(callback)); + return; + } + + DVLOG(1) << "OnTabContentResult: " << data.get(); + const bool is_video = base::Contains(kVideoPageContentTypes, data->type); + DVLOG(1) << "Is video? " << is_video; + + if (!is_video) { + DCHECK(data->content->is_content()); + auto content = data->content->get_content(); + DVLOG(1) << __func__ << ": Got content with char length of " + << content.length(); + SendResultAndDeleteSelf(std::move(callback), content); + return; + } + + SendResultAndDeleteSelf(std::move(callback)); + } + + private: + void DeleteSelf() { delete this; } + void SendResultAndDeleteSelf(ExtractPageContentCallback callback, + std::string content = "") { + std::move(callback).Run(content); + delete this; + } + mojo::Remote content_extractor_; + base::WeakPtrFactory weak_ptr_factory_{this}; +}; +} // namespace + +// static +void ChatContextObserver::BindPageContentExtractorHost( + content::RenderFrameHost* rfh, + mojo::PendingAssociatedReceiver + receiver) { + CHECK(rfh); + if (!rfh->IsInPrimaryMainFrame()) { + DVLOG(1) << "Render frame is not in primary main frame. Not binding to " + "extractor host."; + return; + } + auto* sender = content::WebContents::FromRenderFrameHost(rfh); + if (!sender) { + DVLOG(1) << "Cannot bind extractor host, no valid WebContents"; + return; + } + auto* chat_context_observer = ChatContextObserver::FromWebContents(sender); + if (!chat_context_observer) { + DVLOG(1) << "Cannot bind extractor host, no ChatContextObserver - " + << sender->GetVisibleURL(); + return; + } + DVLOG(1) << "Binding extractor host to ChatContextObserver"; + chat_context_observer->BindPageContentExtractorReceiver(std::move(receiver)); +} + +void ChatContextObserver::BindPageContentExtractorReceiver( + mojo::PendingAssociatedReceiver + receiver) { + page_content_extractor_receiver_.reset(); + page_content_extractor_receiver_.Bind(std::move(receiver)); +} + +ChatContextObserver::ChatContextObserver(content::WebContents* web_contents) + : content::WebContentsObserver(web_contents), + content::WebContentsUserData(*web_contents) { + previous_page_title_ = web_contents->GetTitle(); +} + +ChatContextObserver::~ChatContextObserver() = default; + +void ChatContextObserver::SetPendingGetContentCallback( + ExtractPageContentCallback callback) { + if (pending_extract_page_content_callback_) { + std::move(pending_extract_page_content_callback_).Run(""); + } + pending_extract_page_content_callback_ = std::move(callback); +} + +GURL ChatContextObserver::GetPageURL() const { + return web_contents()->GetLastCommittedURL(); +} + +void ChatContextObserver::GetPageContent(ExtractPageContentCallback callback) { + // fix: this doesn't work properly, need to introduce intermediate class to capture related WebContents + auto* primary_rfh = web_contents()->GetPrimaryMainFrame(); + DCHECK(primary_rfh->IsRenderFrameLive()); + + mojo::Remote extractor; + primary_rfh->GetRemoteInterfaces()->GetInterface( + extractor.BindNewPipeAndPassReceiver()); + + auto* extractor_helper = new PageContentExtractorHelper(); + extractor_helper->Start(std::move(extractor), std::move(callback)); +} + +std::u16string ChatContextObserver::GetPageTitle() const { + return web_contents()->GetTitle(); +} + +// begin content::WebContentsObserver +void ChatContextObserver::WebContentsDestroyed() { + inner_web_contents_ = nullptr; +} + +void ChatContextObserver::NavigationEntryCommitted( + const content::LoadCommittedDetails& load_details) { + if (!load_details.is_main_frame) { + return; + } + // UniqueID will provide a consistent value for the entry when navigating + // through history, allowing us to re-join conversations and navigations. + int pending_navigation_id = load_details.entry->GetUniqueID(); + pending_navigation_id_ = pending_navigation_id; + DVLOG(1) << __func__ << " id: " << pending_navigation_id_ + << "\n url: " << load_details.entry->GetVirtualURL() + << "\n current page title: " << GetPageTitle() + << "\n previous page title: " << previous_page_title_ + << "\n same document? " << load_details.is_same_document; + + // Allow same-document navigation, as content often changes as a result + // of framgment / pushState / replaceState navigations. + // Content won't be retrieved immediately and we don't have a similar + // "DOM Content Loaded" event, so let's wait for something else such as + // page title changing before committing to starting a new conversation + // and treating it as a "fresh page". + is_same_document_navigation_ = load_details.is_same_document; + // Experimentally only call |OnNewPage| for same-page navigations _if_ + // it results in a page title change (see |TtileWasSet|). Title detection + // also done within the navigation entry so that back/forward navigations + // are handled correctly. + + // Page loaded is only considered changing when full document changes + if (!is_same_document_navigation_) { + is_page_loaded_ = false; + } + if (!is_same_document_navigation_ || previous_page_title_ != GetPageTitle()) { + // OnNewPage(pending_navigation_id_); + } + previous_page_title_ = GetPageTitle(); +} + +void ChatContextObserver::TitleWasSet(content::NavigationEntry* entry) { + DVLOG(1) << __func__ << ": id=" << entry->GetUniqueID() + << " title=" << entry->GetTitle(); + MaybeSameDocumentIsNewPage(); + previous_page_title_ = GetPageTitle(); +} + +void ChatContextObserver::DidFinishLoad( + content::RenderFrameHost* render_frame_host, + const GURL& validated_url) { + DVLOG(1) << __func__ << ": " << validated_url.spec(); + if (validated_url == GetPageURL()) { + is_page_loaded_ = true; + if (pending_extract_page_content_callback_) { + GetPageContent(std::move(pending_extract_page_content_callback_)); + } + } +} +// end content::WebContentsObserver + +void ChatContextObserver::MaybeSameDocumentIsNewPage() { + if (is_same_document_navigation_) { + DVLOG(2) << "Same document navigation detected new \"page\" - calling " + "OnNewPage()"; + // Cancel knowledge that the current navigation should be associated + // with any conversation that's associated with the previous navigation. + // Tell any conversation that it shouldn't be associated with this + // content anymore, as we've moved on. + // OnNewPage(pending_navigation_id_); + // Don't respond to further TitleWasSet + is_same_document_navigation_ = false; + } +} + +// mojom::PageContentExtractorHost +void ChatContextObserver::OnInterceptedPageContentChanged() { + // Maybe mark that the page changed, if we didn't detect it already via title + // change after a same-page navigation. This is the main benefit of this + // function. + MaybeSameDocumentIsNewPage(); +} + +WEB_CONTENTS_USER_DATA_KEY_IMPL(ChatContextObserver); +} // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h new file mode 100644 index 00000000000000..8c768dc3638022 --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h @@ -0,0 +1,96 @@ +#ifndef CHROMIUM_CHAT_CONTEXT_OBSERVER_H +#define CHROMIUM_CHAT_CONTEXT_OBSERVER_H + +#include +#include +#include +#include + +#include "base/functional/callback_forward.h" +#include "base/memory/raw_ptr.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" +#include "content/public/browser/navigation_handle.h" +#include "content/public/browser/render_frame_host.h" +#include "content/public/browser/web_contents.h" +#include "content/public/browser/web_contents_observer.h" +#include "content/public/browser/web_contents_user_data.h" +#include "mojo/public/cpp/bindings/associated_receiver.h" +#include "mojo/public/cpp/bindings/pending_associated_receiver.h" + +namespace ai_chat { +// Monitors changes in web content and updates the AI chat with the latest +// information. +class ChatContextObserver + : public content::WebContentsObserver, + public content::WebContentsUserData, + public chat::mojom::PageContentExtractorHost { + public: + using ExtractPageContentCallback = + base::OnceCallback; + + static void BindPageContentExtractorHost( + content::RenderFrameHost* rfh, + mojo::PendingAssociatedReceiver + receiver); + + ChatContextObserver(const ChatContextObserver&) = delete; + + ChatContextObserver& operator=(const ChatContextObserver&) = delete; + + ~ChatContextObserver() override; + + // chat::mojom::PageContentExtractorHost + void OnInterceptedPageContentChanged() override; + + void GetPageContent(ExtractPageContentCallback callback); + + private: + ChatContextObserver(content::WebContents* web_contents); + + friend class content::WebContentsUserData; + + // begin content::WebContentsObserver + void WebContentsDestroyed() override; + + // Called when an event of significance occurs that, if the page is a + // same-document navigation, should result in that previous navigation + // being considered as a new page. + void MaybeSameDocumentIsNewPage(); + + void NavigationEntryCommitted( + const content::LoadCommittedDetails& load_details) override; + + void TitleWasSet(content::NavigationEntry* entry) override; + + void DidFinishLoad(content::RenderFrameHost* render_frame_host, + const GURL& validated_url) override; + // end content::WebContentsObserver + + void BindPageContentExtractorReceiver( + mojo::PendingAssociatedReceiver + receiver); + + void SetPendingGetContentCallback(ExtractPageContentCallback callback); + + GURL GetPageURL() const; + + std::u16string GetPageTitle() const; + + void OnNewPage(int64_t navigation_id); + + int pending_navigation_id_; + bool is_same_document_navigation_ = false; + std::u16string previous_page_title_; + bool is_page_loaded_ = false; + raw_ptr inner_web_contents_ = nullptr; + ExtractPageContentCallback pending_extract_page_content_callback_; + + mojo::AssociatedReceiver + page_content_extractor_receiver_{this}; + + base::WeakPtrFactory weak_ptr_factory_{this}; + WEB_CONTENTS_USER_DATA_KEY_DECL(); +}; +} // namespace ai_chat + +#endif // CHROMIUM_CHAT_CONTEXT_OBSERVER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 2725f7a90cd1f8..9db85fc13da94d 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -15,82 +15,94 @@ #include "content/public/browser/browser_thread.h" #include "content/public/browser/storage_partition.h" #include "content/public/browser/web_contents.h" +#include "content/public/browser/web_contents_user_data.h" #include "content/public/browser/web_ui.h" #include "services/network/public/cpp/shared_url_loader_factory.h" #include "services/service_manager/public/cpp/interface_provider.h" #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" -namespace { -constexpr auto kVideoPageContentTypes = - base::MakeFixedFlatSet( - {chat::mojom::PageContentType::VideoTranscriptYouTube, - chat::mojom::PageContentType::VideoTranscriptVTT}); - -using ExtractPageContentCallback = - base::OnceCallback; - -class PageContentExtractorHelper { - public: - PageContentExtractorHelper() {} - - void Start(mojo::Remote content_extractor, - ExtractPageContentCallback callback) { - content_extractor_ = std::move(content_extractor); - if (!content_extractor_) { - DeleteSelf(); - return; - } - - // Ref: - // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks - // Once a `mojo::Remote` is destroyed, it is guaranteed that pending - // callbacks as well as the connection error handler (if registered) won't - // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed that - // no more method calls are dispatched to the implementation and the - // connection error handler (if registered) won't be called. - content_extractor_.set_disconnect_handler(base::BindOnce( - &PageContentExtractorHelper::DeleteSelf, base::Unretained(this))); - content_extractor_->ExtractPageContent( - base::BindOnce(&PageContentExtractorHelper::OnPageContentExtracted, - base::Unretained(this), std::move(callback))); - } - - void OnPageContentExtracted(ExtractPageContentCallback callback, - chat::mojom::PageContentPtr data) { - if (!data) { - DVLOG(0) << __func__ << " no extracted page content."; - SendResultAndDeleteSelf(std::move(callback)); - return; - } - - DVLOG(1) << "OnTabContentResult: " << data.get(); - const bool is_video = base::Contains(kVideoPageContentTypes, data->type); - DVLOG(1) << "Is video? " << is_video; - - if (!is_video) { - DCHECK(data->content->is_content()); - auto content = data->content->get_content(); - DVLOG(1) << __func__ << ": Got content with char length of " - << content.length(); - SendResultAndDeleteSelf(std::move(callback), content); - return; - } - - SendResultAndDeleteSelf(std::move(callback)); - } - - private: - void DeleteSelf() { delete this; } - void SendResultAndDeleteSelf(ExtractPageContentCallback callback, - std::string content = "") { - std::move(callback).Run(content); - delete this; - } - mojo::Remote content_extractor_; - base::WeakPtrFactory weak_ptr_factory_{this}; -}; -} // namespace +// namespace { +// constexpr auto kVideoPageContentTypes = +// base::MakeFixedFlatSet( +// {chat::mojom::PageContentType::VideoTranscriptYouTube, +// chat::mojom::PageContentType::VideoTranscriptVTT}); +// +// using ExtractPageContentCallback = +// base::OnceCallback; +// +// class PageContentExtractorHelper { +// public: +// PageContentExtractorHelper() {} +// +// void Start(mojo::Remote +// content_extractor, +// ExtractPageContentCallback callback) { +// content_extractor_ = std::move(content_extractor); +// if (!content_extractor_) { +// DeleteSelf(); +// return; +// } +// +// // Ref: +// // +// https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks +// // Once a `mojo::Remote` is destroyed, it is guaranteed that pending +// // callbacks as well as the connection error handler (if registered) +// won't +// // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed +// that +// // no more method calls are dispatched to the implementation and the +// // connection error handler (if registered) won't be called. +// content_extractor_.set_disconnect_handler(base::BindOnce( +// &PageContentExtractorHelper::DeleteSelf, base::Unretained(this))); +// content_extractor_->ExtractPageContent( +// base::BindOnce(&PageContentExtractorHelper::OnPageContentExtracted, +// base::Unretained(this), std::move(callback))); +// } +// +// void OnPageContentExtracted(ExtractPageContentCallback callback, +// chat::mojom::PageContentPtr data) { +// if (!data) { +// DVLOG(0) << __func__ << " no extracted page content."; +// SendResultAndDeleteSelf(std::move(callback)); +// return; +// } +// +// DVLOG(1) << "OnTabContentResult: " << data.get(); +// const bool is_video = base::Contains(kVideoPageContentTypes, data->type); +// DVLOG(1) << "Is video? " << is_video; +// +// if (!is_video) { +// DCHECK(data->content->is_content()); +// auto content = data->content->get_content(); +// DVLOG(1) << __func__ << ": Got content with char length of " +// << content.length(); +// SendResultAndDeleteSelf(std::move(callback), content); +// return; +// } +// +// SendResultAndDeleteSelf(std::move(callback)); +// } +// +// private: +// void DeleteSelf() { delete this; } +// void SendResultAndDeleteSelf(ExtractPageContentCallback callback, +// std::string content = "") { +// std::move(callback).Run(content); +// delete this; +// } +// mojo::Remote content_extractor_; +// base::WeakPtrFactory weak_ptr_factory_{this}; +// }; +// } // namespace + +ChatPageHandler::ChatWebContentsObserver::ChatWebContentsObserver( + content::WebContents* web_contents, + ChatPageHandler& page_handler) + : content::WebContentsObserver(web_contents), page_handler_(page_handler) {} + +ChatPageHandler::ChatWebContentsObserver::~ChatWebContentsObserver() = default; ChatPageHandler::ChatPageHandler( mojo::PendingReceiver receiver, @@ -110,10 +122,24 @@ ChatPageHandler::ChatPageHandler( ->GetURLLoaderFactoryForBrowserProcess(); api_client_ = std::make_unique(std::move(url_loader_factory)); + + active_chat_context_observer_ = + ai_chat::ChatContextObserver::FromWebContents(chat_context_web_contents); + web_content_observer_ = std::make_unique( + chat_context_web_contents, *this); } ChatPageHandler::~ChatPageHandler() = default; +void ChatPageHandler::ChatWebContentsObserver::WebContentsDestroyed() { + page_handler_->HandleWebContentsDestroyed(); +} + +void ChatPageHandler::HandleWebContentsDestroyed() { + active_chat_context_observer_ = nullptr; + web_content_observer_.reset(); +} + void ChatPageHandler::ShowUI() { auto embedder = chat_ui_->embedder(); if (embedder) { @@ -207,22 +233,28 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { LOG(INFO) << "ActionType: Summarize_page"; - LOG(INFO) << "****" - << base::UTF16ToUTF8( - chat_context_web_contents_->GetTitle()); - const GURL gurl = chat_context_web_contents_->GetLastCommittedURL(); - LOG(INFO) << "****" << gurl.spec(); - - auto* primary_rfh = chat_context_web_contents_->GetPrimaryMainFrame(); - DCHECK(primary_rfh->IsRenderFrameLive()); - - mojo::Remote extractor; - primary_rfh->GetRemoteInterfaces()->GetInterface( - extractor.BindNewPipeAndPassReceiver()); - - auto* extractor_helper = new PageContentExtractorHelper(); - extractor_helper->Start( - std::move(extractor), + // LOG(INFO) << "****" + // << base::UTF16ToUTF8( + // chat_context_web_contents_->GetTitle()); + // const GURL gurl = + // chat_context_web_contents_->GetLastCommittedURL(); + // LOG(INFO) << "****" << gurl.spec(); + + // auto* primary_rfh = + // chat_context_web_contents_->GetPrimaryMainFrame(); + // DCHECK(primary_rfh->IsRenderFrameLive()); + // + // mojo::Remote extractor; + // primary_rfh->GetRemoteInterfaces()->GetInterface( + // extractor.BindNewPipeAndPassReceiver()); + // + // auto* extractor_helper = new PageContentExtractorHelper(); + // extractor_helper->Start( + // std::move(extractor), + // base::BindOnce(&ChatPageHandler::OnPageContentExtracted, + // base::Unretained(this))); + + active_chat_context_observer_->GetPageContent( base::BindOnce(&ChatPageHandler::OnPageContentExtracted, base::Unretained(this))); diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 1de83090b5c64b..9c67ff0284b92d 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -7,12 +7,16 @@ #include "base/functional/callback_forward.h" #include "base/memory/raw_ptr.h" +#include "base/scoped_observation.h" #include "base/types/expected.h" +#include "chat_context_observer.h" #include "chat_ui.h" #include "chrome/browser/profiles/profile.h" #include "chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h" #include "chrome/browser/ui/webui/side_panel/chat/chat.mojom.h" #include "chrome/common/chat/page_content_extractor.mojom.h" +#include "content/public/browser/web_contents_observer.h" +#include "content/public/browser/web_contents_user_data.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" @@ -24,44 +28,64 @@ namespace content { } // namespace content class ChatPageHandler : public chat::mojom::PageHandler { -public: - ChatPageHandler(mojo::PendingReceiver receiver, - mojo::PendingRemote page, - ChatUI* chat_ui, - content::WebUI* web_ui, - content::WebContents* owner_web_contents, - content::WebContents* chat_context_web_contents); - - ChatPageHandler(const ChatPageHandler&) = delete; - ChatPageHandler& operator=(const ChatPageHandler&) = delete; - - ~ChatPageHandler() override; - - void GetSiteInfo(GetSiteInfoCallback callback) override; - void GetActionList(GetActionListCallback callback) override; - void SubmitAction(chat::mojom::ActionType action_type) override; - void SubmitQuery(chat::mojom::ActionType action_type, - const std::string& query) override; - void ShowUI() override; - void CloseUI() override; - - void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); - - void SubmitQueryCallback(std::string completion); - void SubmitQueryCompletedCallback( - base::expected result); - - base::WeakPtr GetWeakPtr(); - -private: - mojo::Receiver receiver_; - mojo::Remote page_; - const raw_ptr chat_ui_; - raw_ptr owner_web_contents_ = nullptr; - raw_ptr chat_context_web_contents_ = nullptr; - const raw_ptr profile_; - std::unique_ptr api_client_ = nullptr; - base::WeakPtrFactory weak_ptr_factory_{this}; - void OnPageContentExtracted(std::string content); + public: + ChatPageHandler(mojo::PendingReceiver receiver, + mojo::PendingRemote page, + ChatUI* chat_ui, + content::WebUI* web_ui, + content::WebContents* owner_web_contents, + content::WebContents* chat_context_web_contents); + + ChatPageHandler(const ChatPageHandler&) = delete; + ChatPageHandler& operator=(const ChatPageHandler&) = delete; + + ~ChatPageHandler() override; + + void GetSiteInfo(GetSiteInfoCallback callback) override; + void GetActionList(GetActionListCallback callback) override; + void SubmitAction(chat::mojom::ActionType action_type) override; + void SubmitQuery(chat::mojom::ActionType action_type, + const std::string& query) override; + void ShowUI() override; + void CloseUI() override; + + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); + + void SubmitQueryCallback(std::string completion); + void SubmitQueryCompletedCallback( + base::expected result); + + base::WeakPtr GetWeakPtr(); + + private: + class ChatWebContentsObserver : public content::WebContentsObserver { + public: + explicit ChatWebContentsObserver(content::WebContents* web_contents, + ChatPageHandler& page_handler); + ~ChatWebContentsObserver() override; + + private: + // content::WebContentsObserver + void WebContentsDestroyed() override; + + raw_ref page_handler_; + }; + + void HandleWebContentsDestroyed(); + + void OnPageContentExtracted(std::string content); + + mojo::Receiver receiver_; + mojo::Remote page_; + const raw_ptr chat_ui_; + raw_ptr owner_web_contents_ = nullptr; + raw_ptr chat_context_web_contents_ = nullptr; + const raw_ptr profile_; + std::unique_ptr api_client_ = nullptr; + + raw_ptr active_chat_context_observer_ = nullptr; + std::unique_ptr web_content_observer_; + + base::WeakPtrFactory weak_ptr_factory_{this}; }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H From caadcb3fd82bd49271d8ba13c1f22b9ffbd8ff47 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 20 Nov 2024 16:52:35 +0800 Subject: [PATCH 22/34] Fix issues in observing chat context --- chrome/browser/ui/BUILD.gn | 2 + chrome/browser/ui/tabs/tab_features.cc | 3 + .../chat/chat_side_panel_web_view.cc | 2 +- .../side_panel/chat/chat_context_observer.cc | 98 ++------------ .../side_panel/chat/chat_context_observer.h | 15 ++- .../side_panel/chat/chat_page_handler.cc | 67 +++++----- .../webui/side_panel/chat/chat_page_handler.h | 2 +- .../ui/webui/side_panel/chat/chat_ui.cc | 4 +- .../ui/webui/side_panel/chat/chat_ui.h | 2 +- .../chat/page_content_extractor_helper.cc | 120 ++++++++++++++++++ .../chat/page_content_extractor_helper.h | 28 ++++ 11 files changed, 212 insertions(+), 131 deletions(-) create mode 100644 chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc create mode 100644 chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index 0d0e62820a153a..b5914795d71fcf 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -1573,6 +1573,8 @@ static_library("ui") { "webui/side_panel/chat/chat_context_observer.h", "webui/side_panel/chat/chat_page_handler.cc", "webui/side_panel/chat/chat_page_handler.h", + "webui/side_panel/chat/page_content_extractor_helper.cc", + "webui/side_panel/chat/page_content_extractor_helper.h", "webui/side_panel/chat/chat_ui.cc", "webui/side_panel/chat/chat_ui.h", "webui/side_panel/companion/companion_page_handler.cc", diff --git a/chrome/browser/ui/tabs/tab_features.cc b/chrome/browser/ui/tabs/tab_features.cc index 6f8f662f93deff..ef6d8614060c52 100644 --- a/chrome/browser/ui/tabs/tab_features.cc +++ b/chrome/browser/ui/tabs/tab_features.cc @@ -39,6 +39,7 @@ #include "components/fingerprinting_protection_filter/common/fingerprinting_protection_filter_features.h" #include "components/image_fetcher/core/image_fetcher_service.h" #include "components/permissions/permission_indicators_tab_data.h" +#include "chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h" namespace tabs { @@ -155,6 +156,8 @@ void TabFeatures::Init(TabInterface& tab, Profile* profile) { if (web_app::WebAppMetricsTabHelper::IsEnabled(tab.GetContents())) { web_app::WebAppMetricsTabHelper::CreateForWebContents(tab.GetContents()); } + + ai_chat::ChatContextObserver::CreateForWebContents(tab.GetContents()); } TabFeatures::TabFeatures() = default; diff --git a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc index c355e5c5c7a113..72b0e2f98aaa09 100644 --- a/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc +++ b/chrome/browser/ui/views/side_panel/chat/chat_side_panel_web_view.cc @@ -86,7 +86,7 @@ void ChatSidePanelWebView::UpdateActiveSiteInfo( site_info->is_content_usable_in_conversations = false; } - controller->GetAs()->SetSiteInfo(site_info.Clone()); + controller->GetAs()->SetSiteInfo(site_info.Clone(), contents); } base::WeakPtr ChatSidePanelWebView::GetWeakPtr() { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc index aadd2c17670e4c..f939fd4a583a2f 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc @@ -28,80 +28,10 @@ #include "ui/accessibility/ax_updates_and_events.h" #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" +#include "page_content_extractor_helper.h" namespace ai_chat { -namespace { -constexpr auto kVideoPageContentTypes = - base::MakeFixedFlatSet( - {chat::mojom::PageContentType::VideoTranscriptYouTube, - chat::mojom::PageContentType::VideoTranscriptVTT}); - -using ExtractPageContentCallback = - ChatContextObserver::ExtractPageContentCallback; - -class PageContentExtractorHelper { - public: - PageContentExtractorHelper() {} - - void Start(mojo::Remote content_extractor, - ExtractPageContentCallback callback) { - content_extractor_ = std::move(content_extractor); - if (!content_extractor_) { - DeleteSelf(); - return; - } - - // Ref: - // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks - // Once a `mojo::Remote` is destroyed, it is guaranteed that pending - // callbacks as well as the connection error handler (if registered) won't - // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed that - // no more method calls are dispatched to the implementation and the - // connection error handler (if registered) won't be called. - content_extractor_.set_disconnect_handler(base::BindOnce( - &PageContentExtractorHelper::DeleteSelf, base::Unretained(this))); - content_extractor_->ExtractPageContent( - base::BindOnce(&PageContentExtractorHelper::OnPageContentExtracted, - base::Unretained(this), std::move(callback))); - } - - void OnPageContentExtracted(ExtractPageContentCallback callback, - chat::mojom::PageContentPtr data) { - if (!data) { - DVLOG(0) << __func__ << " no extracted page content."; - SendResultAndDeleteSelf(std::move(callback)); - return; - } - - DVLOG(1) << "OnTabContentResult: " << data.get(); - const bool is_video = base::Contains(kVideoPageContentTypes, data->type); - DVLOG(1) << "Is video? " << is_video; - - if (!is_video) { - DCHECK(data->content->is_content()); - auto content = data->content->get_content(); - DVLOG(1) << __func__ << ": Got content with char length of " - << content.length(); - SendResultAndDeleteSelf(std::move(callback), content); - return; - } - - SendResultAndDeleteSelf(std::move(callback)); - } - - private: - void DeleteSelf() { delete this; } - void SendResultAndDeleteSelf(ExtractPageContentCallback callback, - std::string content = "") { - std::move(callback).Run(content); - delete this; - } - mojo::Remote content_extractor_; - base::WeakPtrFactory weak_ptr_factory_{this}; -}; -} // namespace - // static void ChatContextObserver::BindPageContentExtractorHost( content::RenderFrameHost* rfh, @@ -109,22 +39,22 @@ void ChatContextObserver::BindPageContentExtractorHost( receiver) { CHECK(rfh); if (!rfh->IsInPrimaryMainFrame()) { - DVLOG(1) << "Render frame is not in primary main frame. Not binding to " + DVLOG(0) << "Render frame is not in primary main frame. Not binding to " "extractor host."; return; } auto* sender = content::WebContents::FromRenderFrameHost(rfh); if (!sender) { - DVLOG(1) << "Cannot bind extractor host, no valid WebContents"; + DVLOG(0) << "Cannot bind extractor host, no valid WebContents"; return; } auto* chat_context_observer = ChatContextObserver::FromWebContents(sender); if (!chat_context_observer) { - DVLOG(1) << "Cannot bind extractor host, no ChatContextObserver - " + DVLOG(0) << "Cannot bind extractor host, no ChatContextObserver - " << sender->GetVisibleURL(); return; } - DVLOG(1) << "Binding extractor host to ChatContextObserver"; + DVLOG(0) << "Binding extractor host to ChatContextObserver"; chat_context_observer->BindPageContentExtractorReceiver(std::move(receiver)); } @@ -137,7 +67,10 @@ void ChatContextObserver::BindPageContentExtractorReceiver( ChatContextObserver::ChatContextObserver(content::WebContents* web_contents) : content::WebContentsObserver(web_contents), - content::WebContentsUserData(*web_contents) { + content::WebContentsUserData(*web_contents), + page_content_extractor_helper_delegate_( + std::make_unique(web_contents)) + { previous_page_title_ = web_contents->GetTitle(); } @@ -156,16 +89,8 @@ GURL ChatContextObserver::GetPageURL() const { } void ChatContextObserver::GetPageContent(ExtractPageContentCallback callback) { - // fix: this doesn't work properly, need to introduce intermediate class to capture related WebContents - auto* primary_rfh = web_contents()->GetPrimaryMainFrame(); - DCHECK(primary_rfh->IsRenderFrameLive()); - - mojo::Remote extractor; - primary_rfh->GetRemoteInterfaces()->GetInterface( - extractor.BindNewPipeAndPassReceiver()); - - auto* extractor_helper = new PageContentExtractorHelper(); - extractor_helper->Start(std::move(extractor), std::move(callback)); + page_content_extractor_helper_delegate_->ExtractPageContent( + std::move(callback)); } std::u16string ChatContextObserver::GetPageTitle() const { @@ -227,6 +152,7 @@ void ChatContextObserver::DidFinishLoad( DVLOG(1) << __func__ << ": " << validated_url.spec(); if (validated_url == GetPageURL()) { is_page_loaded_ = true; + page_content_extractor_helper_delegate_ = std::make_unique(web_contents()); if (pending_extract_page_content_callback_) { GetPageContent(std::move(pending_extract_page_content_callback_)); } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h index 8c768dc3638022..286fa2ef063d45 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h @@ -25,14 +25,21 @@ class ChatContextObserver public content::WebContentsUserData, public chat::mojom::PageContentExtractorHost { public: - using ExtractPageContentCallback = - base::OnceCallback; + + using ExtractPageContentCallback = + base::OnceCallback; static void BindPageContentExtractorHost( content::RenderFrameHost* rfh, mojo::PendingAssociatedReceiver receiver); + class PageContentExtractorHelperDelegate { + public: + virtual ~PageContentExtractorHelperDelegate() = default; + virtual void ExtractPageContent(ExtractPageContentCallback callback) = 0; + }; + ChatContextObserver(const ChatContextObserver&) = delete; ChatContextObserver& operator=(const ChatContextObserver&) = delete; @@ -83,12 +90,16 @@ class ChatContextObserver std::u16string previous_page_title_; bool is_page_loaded_ = false; raw_ptr inner_web_contents_ = nullptr; + + std::unique_ptr + page_content_extractor_helper_delegate_; ExtractPageContentCallback pending_extract_page_content_callback_; mojo::AssociatedReceiver page_content_extractor_receiver_{this}; base::WeakPtrFactory weak_ptr_factory_{this}; + WEB_CONTENTS_USER_DATA_KEY_DECL(); }; } // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 9db85fc13da94d..a1ffc3a9e3f22a 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -153,7 +153,13 @@ void ChatPageHandler::CloseUI() { embedder->CloseUI(); } -void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { +void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents) { + + //ai_chat::ChatContextObserver::CreateForWebContents(contents); + active_chat_context_observer_ = + ai_chat::ChatContextObserver::FromWebContents(contents); + web_content_observer_ = std::make_unique( + contents, *this); if (page_.is_bound()) { page_->OnSiteInfoChanged(std::move(site_info)); } @@ -229,43 +235,28 @@ base::WeakPtr ChatPageHandler::GetWeakPtr() { void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { if (page_.is_bound()) { - LOG(INFO) << action_type; - - if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { - LOG(INFO) << "ActionType: Summarize_page"; - // LOG(INFO) << "****" - // << base::UTF16ToUTF8( - // chat_context_web_contents_->GetTitle()); - // const GURL gurl = - // chat_context_web_contents_->GetLastCommittedURL(); - // LOG(INFO) << "****" << gurl.spec(); - - // auto* primary_rfh = - // chat_context_web_contents_->GetPrimaryMainFrame(); - // DCHECK(primary_rfh->IsRenderFrameLive()); - // - // mojo::Remote extractor; - // primary_rfh->GetRemoteInterfaces()->GetInterface( - // extractor.BindNewPipeAndPassReceiver()); - // - // auto* extractor_helper = new PageContentExtractorHelper(); - // extractor_helper->Start( - // std::move(extractor), - // base::BindOnce(&ChatPageHandler::OnPageContentExtracted, - // base::Unretained(this))); - - active_chat_context_observer_->GetPageContent( - base::BindOnce(&ChatPageHandler::OnPageContentExtracted, - base::Unretained(this))); - - } else { - // todo: to implement for other action types later - chat::mojom::ActionResponsePtr response = - chat::mojom::ActionResponse::New(); - response->action_type = action_type; - response->result = "MOCK Result"; - page_->OnSubmitActionResponse(response.Clone()); - } + DVLOG(0) << action_type; + + if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { + DVLOG(0) << "ActionType: Summarize_page"; + // LOG(INFO) << "****" + // << base::UTF16ToUTF8( + // chat_context_web_contents_->GetTitle()); + // const GURL gurl = + // chat_context_web_contents_->GetLastCommittedURL(); + // LOG(INFO) << "****" << gurl.spec(); + + active_chat_context_observer_->GetPageContent(base::BindOnce( + &ChatPageHandler::OnPageContentExtracted, base::Unretained(this))); + + } else { + // todo: to implement for other action types later + chat::mojom::ActionResponsePtr response = + chat::mojom::ActionResponse::New(); + response->action_type = action_type; + response->result = "MOCK Result"; + page_->OnSubmitActionResponse(response.Clone()); + } } } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 9c67ff0284b92d..1ca2634ee32d1f 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -49,7 +49,7 @@ class ChatPageHandler : public chat::mojom::PageHandler { void ShowUI() override; void CloseUI() override; - void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents); void SubmitQueryCallback(std::string completion); void SubmitQueryCompletedCallback( diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc index 525ebd9c2ae1a3..7a6d58fd5ffe00 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc @@ -121,8 +121,8 @@ void ChatUI::CreatePageHandler( web_ui()->GetWebContents(), web_contents); } -void ChatUI::SetSiteInfo(chat::mojom::SiteInfoPtr site_info) { +void ChatUI::SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents) { if (page_handler_) { - page_handler_->SetSiteInfo(std::move(site_info)); + page_handler_->SetSiteInfo(std::move(site_info), contents); } } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h index bf403ab743e2d4..ff26d1adf6d0d5 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -48,7 +48,7 @@ class ChatUI : public TopChromeWebUIController, static constexpr std::string GetWebUIName() { return "Chat"; } - void SetSiteInfo(chat::mojom::SiteInfoPtr site_info); + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents ); private: void CreatePageHandler( diff --git a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc new file mode 100644 index 00000000000000..e2333897c85f22 --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc @@ -0,0 +1,120 @@ +#include "page_content_extractor_helper.h" + +#include +#include +#include +#include +#include + +#include "base/functional/callback_forward.h" +#include "base/memory/raw_ptr.h" +#include "base/containers/fixed_flat_set.h" +#include "base/containers/contains.h" +#include "base/functional/bind.h" +#include "base/memory/weak_ptr.h" +#include "base/ranges/algorithm.h" +#include "base/strings/string_util.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" +#include "content/public/browser/navigation_handle.h" +#include "content/public/browser/render_frame_host.h" +#include "content/public/browser/web_contents.h" +#include "content/public/browser/web_contents_observer.h" +#include "content/public/browser/web_contents_user_data.h" +#include "mojo/public/cpp/bindings/associated_receiver.h" +#include "mojo/public/cpp/bindings/pending_associated_receiver.h" +#include "services/service_manager/public/cpp/interface_provider.h" +#include "content/public/browser/web_contents.h" + +namespace ai_chat { + +namespace { +constexpr auto kVideoPageContentTypes = + base::MakeFixedFlatSet( + {chat::mojom::PageContentType::VideoTranscriptYouTube, + chat::mojom::PageContentType::VideoTranscriptVTT}); + +using ExtractPageContentCallback = + ChatContextObserver::ExtractPageContentCallback; + +class PageContentExtractorInternal { + public: + PageContentExtractorInternal() {} + + void Start(mojo::Remote content_extractor, + ExtractPageContentCallback callback) { + content_extractor_ = std::move(content_extractor); + if (!content_extractor_) { + DeleteSelf(); + return; + } + + // Ref: + // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks + // Once a `mojo::Remote` is destroyed, it is guaranteed that pending + // callbacks as well as the connection error handler (if registered) won't + // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed that + // no more method calls are dispatched to the implementation and the + // connection error handler (if registered) won't be called. + content_extractor_.set_disconnect_handler(base::BindOnce( + &PageContentExtractorInternal::DeleteSelf, base::Unretained(this))); + content_extractor_->ExtractPageContent( + base::BindOnce(&PageContentExtractorInternal::OnPageContentExtracted, + base::Unretained(this), std::move(callback))); + } + + void OnPageContentExtracted(ExtractPageContentCallback callback, + chat::mojom::PageContentPtr data) { + if (!data) { + DVLOG(0) << __func__ << " no extracted page content."; + SendResultAndDeleteSelf(std::move(callback)); + return; + } + + DVLOG(1) << "OnTabContentResult: " << data.get(); + const bool is_video = base::Contains(kVideoPageContentTypes, data->type); + DVLOG(1) << "Is video? " << is_video; + + if (!is_video) { + DCHECK(data->content->is_content()); + auto content = data->content->get_content(); + DVLOG(1) << __func__ << ": Got content with char length of " + << content.length(); + SendResultAndDeleteSelf(std::move(callback), content); + return; + } + + SendResultAndDeleteSelf(std::move(callback)); + } + + private: + void DeleteSelf() { delete this; } + void SendResultAndDeleteSelf(ExtractPageContentCallback callback, + std::string content = "") { + std::move(callback).Run(content); + delete this; + } + mojo::Remote content_extractor_; + base::WeakPtrFactory weak_ptr_factory_{this}; +}; +} // namespace + +PageContentExtractorHelper::PageContentExtractorHelper( + content::WebContents* web_contents) + : web_contents_(web_contents) {} + +PageContentExtractorHelper::~PageContentExtractorHelper() = default; + +void PageContentExtractorHelper::ExtractPageContent( + ChatContextObserver::ExtractPageContentCallback callback) { + auto* primary_rfh = web_contents_->GetPrimaryMainFrame(); + DCHECK(primary_rfh->IsRenderFrameLive()); + + mojo::Remote extractor; + primary_rfh->GetRemoteInterfaces()->GetInterface( + extractor.BindNewPipeAndPassReceiver()); + + auto* internal_extractor = new PageContentExtractorInternal(); + internal_extractor->Start(std::move(extractor), std::move(callback)); +} + +} // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h new file mode 100644 index 00000000000000..3e7b2ff2977a67 --- /dev/null +++ b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h @@ -0,0 +1,28 @@ +#ifndef CHROMIUM_PAGE_CONTENT_EXTRACTOR_HELPER_H +#define CHROMIUM_PAGE_CONTENT_EXTRACTOR_HELPER_H + +#include + +#include "base/functional/callback_forward.h" +#include "chat_context_observer.h" +#include "chrome/renderer/chat/page_content_extractor.h" +#include "chat_context_observer.h" + +namespace ai_chat { +class PageContentExtractorHelper +: public ChatContextObserver::PageContentExtractorHelperDelegate { + public: + + explicit PageContentExtractorHelper(content::WebContents* web_contents); + ~PageContentExtractorHelper() override; + PageContentExtractorHelper(const PageContentExtractorHelper&) = delete; + PageContentExtractorHelper& operator=(const PageContentExtractorHelper&) = + delete; + void ExtractPageContent(ChatContextObserver::ExtractPageContentCallback callback) override; + + private: + raw_ptr web_contents_; +}; +} // namespace ai_chat + +#endif // CHROMIUM_PAGE_CONTENT_EXTRACTOR_HELPER_H From d9179b3328985ca2ab498a6718a049cda45bc2d1 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Wed, 20 Nov 2024 18:20:23 +0800 Subject: [PATCH 23/34] Display completion result in Chat UI - WIP --- .../side_panel/chat/chat_app.html.ts | 4 +- .../resources/side_panel/chat/chat_app.ts | 6 +- .../chat/api/completion_api_client.cc | 8 +- .../side_panel/chat/chat_context_observer.cc | 1 - .../side_panel/chat/chat_page_handler.cc | 108 +++--------------- .../webui/side_panel/chat/chat_page_handler.h | 6 +- 6 files changed, 30 insertions(+), 103 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 799a7bcc454484..752ad193f67db3 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -9,7 +9,9 @@ export function getHtml(this: ChatAppElement) {
${this.siteInfo_.url}
${this.siteInfo_.title}
- ${this.submitResponse_.result} +

+ ${this.completionResult_} +

diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 462fbc080485a0..4175457f405e80 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -27,6 +27,7 @@ export class ChatAppElement extends CrLitElement { actionType: ActionType.NONE, result: "" }; + protected completionResult_ : string = ""; protected textareaValue_?: string; constructor() { @@ -48,6 +49,7 @@ export class ChatAppElement extends CrLitElement { actionList_: {type: Array}, submitResponse_: {type: Object}, textareaValue_: {type: String}, + completionResult_: {type: String}, }; } @@ -61,11 +63,13 @@ export class ChatAppElement extends CrLitElement { } private async updateSubmitResponse(response: ActionResponse) { - this.submitResponse_ = response; + //this.submitResponse_ = response; + this.completionResult_ += response.result; } protected onSubmitAction_(e: Event) { e.stopPropagation(); + this.completionResult_ += "\n\n"; this.chatApiProxy_.submitAction(ActionType.SUMMARIZE_PAGE); } diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc index 5ae1cab1372cfc..7b929c01795a45 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc @@ -127,19 +127,13 @@ void CompletionApiClient::OnQueryDataReceived( if (delta) { const std::string* content = delta->FindString("content"); if (content) { - //LOG(INFO) << "delta content: " << *content; entire_completion_result.push_back(*content); + callback.Run(std::move(*content)); } } } } } - - // This client only supports completion events - const std::string* completion = result->GetDict().FindString("choices"); - if (completion) { - callback.Run(std::move(*completion)); - } } void CompletionApiClient::OnQueryCompleted( diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc index f939fd4a583a2f..19a6d757ae24e8 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc @@ -152,7 +152,6 @@ void ChatContextObserver::DidFinishLoad( DVLOG(1) << __func__ << ": " << validated_url.spec(); if (validated_url == GetPageURL()) { is_page_loaded_ = true; - page_content_extractor_helper_delegate_ = std::make_unique(web_contents()); if (pending_extract_page_content_callback_) { GetPageContent(std::move(pending_extract_page_content_callback_)); } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index a1ffc3a9e3f22a..15eb8f027500f3 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -22,81 +22,6 @@ #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" -// namespace { -// constexpr auto kVideoPageContentTypes = -// base::MakeFixedFlatSet( -// {chat::mojom::PageContentType::VideoTranscriptYouTube, -// chat::mojom::PageContentType::VideoTranscriptVTT}); -// -// using ExtractPageContentCallback = -// base::OnceCallback; -// -// class PageContentExtractorHelper { -// public: -// PageContentExtractorHelper() {} -// -// void Start(mojo::Remote -// content_extractor, -// ExtractPageContentCallback callback) { -// content_extractor_ = std::move(content_extractor); -// if (!content_extractor_) { -// DeleteSelf(); -// return; -// } -// -// // Ref: -// // -// https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks -// // Once a `mojo::Remote` is destroyed, it is guaranteed that pending -// // callbacks as well as the connection error handler (if registered) -// won't -// // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed -// that -// // no more method calls are dispatched to the implementation and the -// // connection error handler (if registered) won't be called. -// content_extractor_.set_disconnect_handler(base::BindOnce( -// &PageContentExtractorHelper::DeleteSelf, base::Unretained(this))); -// content_extractor_->ExtractPageContent( -// base::BindOnce(&PageContentExtractorHelper::OnPageContentExtracted, -// base::Unretained(this), std::move(callback))); -// } -// -// void OnPageContentExtracted(ExtractPageContentCallback callback, -// chat::mojom::PageContentPtr data) { -// if (!data) { -// DVLOG(0) << __func__ << " no extracted page content."; -// SendResultAndDeleteSelf(std::move(callback)); -// return; -// } -// -// DVLOG(1) << "OnTabContentResult: " << data.get(); -// const bool is_video = base::Contains(kVideoPageContentTypes, data->type); -// DVLOG(1) << "Is video? " << is_video; -// -// if (!is_video) { -// DCHECK(data->content->is_content()); -// auto content = data->content->get_content(); -// DVLOG(1) << __func__ << ": Got content with char length of " -// << content.length(); -// SendResultAndDeleteSelf(std::move(callback), content); -// return; -// } -// -// SendResultAndDeleteSelf(std::move(callback)); -// } -// -// private: -// void DeleteSelf() { delete this; } -// void SendResultAndDeleteSelf(ExtractPageContentCallback callback, -// std::string content = "") { -// std::move(callback).Run(content); -// delete this; -// } -// mojo::Remote content_extractor_; -// base::WeakPtrFactory weak_ptr_factory_{this}; -// }; -// } // namespace - ChatPageHandler::ChatWebContentsObserver::ChatWebContentsObserver( content::WebContents* web_contents, ChatPageHandler& page_handler) @@ -154,8 +79,6 @@ void ChatPageHandler::CloseUI() { } void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents) { - - //ai_chat::ChatContextObserver::CreateForWebContents(contents); active_chat_context_observer_ = ai_chat::ChatContextObserver::FromWebContents(contents); web_content_observer_ = std::make_unique( @@ -220,9 +143,6 @@ void ChatPageHandler::GetActionList(GetActionListCallback callback) { std::move(callback).Run(std::move(action_items)); } -void ChatPageHandler::SubmitQueryCallback(std::string completion) { - LOG(INFO) << completion; -} void ChatPageHandler::SubmitQueryCompletedCallback( base::expected result) { @@ -239,15 +159,9 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { DVLOG(0) << "ActionType: Summarize_page"; - // LOG(INFO) << "****" - // << base::UTF16ToUTF8( - // chat_context_web_contents_->GetTitle()); - // const GURL gurl = - // chat_context_web_contents_->GetLastCommittedURL(); - // LOG(INFO) << "****" << gurl.spec(); - - active_chat_context_observer_->GetPageContent(base::BindOnce( - &ChatPageHandler::OnPageContentExtracted, base::Unretained(this))); + active_chat_context_observer_->GetPageContent( + base::BindOnce(&ChatPageHandler::OnPageContentExtracted, + base::Unretained(this), action_type)); } else { // todo: to implement for other action types later @@ -260,12 +174,24 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { } } -void ChatPageHandler::OnPageContentExtracted(std::string content) { +void ChatPageHandler::OnPageContentExtracted( + chat::mojom::ActionType action_type, + std::string content) { // todo: to put prompt in resource file api_client_->QueryPrompt( "Provide a brief summary of the key takeaways for the following:" + content, - base::NullCallback(), base::NullCallback()); + base::NullCallback(), + base::BindRepeating(&ChatPageHandler::SubmitQueryCallback, + base::Unretained(this), action_type)); +} + +void ChatPageHandler::SubmitQueryCallback(chat::mojom::ActionType action_type, + std::string completion) { + chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); + response->action_type = action_type; + response->result = completion; + page_->OnSubmitActionResponse(response.Clone()); } void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 1ca2634ee32d1f..47cac113397c33 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -51,7 +51,8 @@ class ChatPageHandler : public chat::mojom::PageHandler { void SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents); - void SubmitQueryCallback(std::string completion); + void SubmitQueryCallback(chat::mojom::ActionType action_type, + std::string completion); void SubmitQueryCompletedCallback( base::expected result); @@ -73,7 +74,8 @@ class ChatPageHandler : public chat::mojom::PageHandler { void HandleWebContentsDestroyed(); - void OnPageContentExtracted(std::string content); + void OnPageContentExtracted(chat::mojom::ActionType action_type, + std::string content); mojo::Receiver receiver_; mojo::Remote page_; From 03d8a38452a7dd009138fd6f4025110de1628366 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 22 Nov 2024 17:15:18 +0800 Subject: [PATCH 24/34] Display conversations in Chat UI - WIP --- .../resources/side_panel/chat/chat_app.css | 44 +++++++++++-- .../side_panel/chat/chat_app.html.ts | 43 +++++++----- .../resources/side_panel/chat/chat_app.ts | 53 ++++++++++++--- .../chat/api/completion_api_client.cc | 65 +++++++++++-------- .../chat/api/completion_api_client.h | 41 ++++++------ .../ui/webui/side_panel/chat/chat.mojom | 33 +++++++--- .../side_panel/chat/chat_page_handler.cc | 39 ++++++++--- .../webui/side_panel/chat/chat_page_handler.h | 3 +- 8 files changed, 223 insertions(+), 98 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index 53099167026e89..932a4828fddbfa 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -65,6 +65,7 @@ cr-button { flex: 1; display: flex; flex-direction: column; + gap: 12px; /* Expands to fill the remaining space */ } @@ -82,30 +83,63 @@ cr-button { align-items: flex-start; } +.query-container { + display: block; + padding: 6px; + width: fit-content; + border-radius: 8px; + border: 1px solid #e3e3e3; + font-size: 14px; + background-color: #efeff2; +} + +.response-container { + display: block; + padding: 6px; + width: fit-content; + font-size: 12px; +} + .action-button { display: inline-block; - padding: 10px 10px; - border: 1px solid #000000; - border-radius: 12px; + padding: 8px; + border: 1px solid #E3E3E3; + border-radius: 8px; cursor: pointer; font-size: 12px; text-decoration: none; + background-color: #FFFFFF; +} + +.site-info { + width: 80vw; + display: flex; + flex-direction: column; + align-items: flex-start; + font-size: 10px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; } .typing-container { bottom: 0; width: 100%; display: flex; + flex-direction: column; padding: 20px 10px; gap: 12px; - background: var(--outgoing-chat-bg); - border-top: 1px solid var(--incoming-chat-border); + background-color: #efeded; + border: 1px solid #cfcece; + border-radius: 16px; align-items: flex-start; } .typing-container .typing-content { display: flex; width: 100%; + flex-direction: row; + gap: 12px; } .typing-container .typing-textarea { diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 752ad193f67db3..09f68c4a9a2765 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -4,36 +4,49 @@ import {html} from '//resources/lit/v3_0/lit.rollup.js'; export function getHtml(this: ChatAppElement) { return html`
-
-
-
${this.siteInfo_.url}
-
${this.siteInfo_.title}
-
+
+ ${ + this.conversations_.map((conversation,_) => { + return conversation.query.length > 0 + ? html` +

${conversation.query}

+ ${conversation.response.length > 0 ? html`

${conversation.response}

` : html``}` + : + html`${conversation.response.length > 0 ? html`

${conversation.response}

` : html``}` + + }) + }

${this.completionResult_}

- ${ this.siteInfo_.isContentUsableInConversations ? + ${ this.siteInfo_.isContentUsableInConversations && this.conversations_ && this.conversations_.length == 0 ? this.actionList_.map((item,_) => html` - + ${item.label} `) : html``}
- - +
+

${this.siteInfo_.url}

+

${this.siteInfo_.title}

+
+
+ + Send +
`; diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 4175457f405e80..bd14259a7cd5c9 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -11,12 +11,18 @@ import {getCss} from './chat_app.css.js'; import {getHtml} from './chat_app.html.js'; import type {ChatApiProxy} from "./chat_api_proxy.js"; import {ChatApiProxyImpl} from "./chat_api_proxy.js"; -import {SiteInfo, ActionItem, ActionType, ActionResponse} from "./chat.mojom-webui.js"; +import {SiteInfo, ActionItem, ActionType, ActionResponse, ResponseType} from "./chat.mojom-webui.js"; + +type conversationRecord = { + query: string, + response: string, +} export class ChatAppElement extends CrLitElement { private chatApiProxy_: ChatApiProxy = ChatApiProxyImpl.getInstance(); private listenerIds_: number[] = []; protected actionList_: ActionItem[] = []; + protected conversations_: conversationRecord[] = []; protected askAnythingLabel_ = loadTimeData.getString('askAnything'); protected siteInfo_ : SiteInfo = { url: "", @@ -25,10 +31,12 @@ export class ChatAppElement extends CrLitElement { }; protected submitResponse_ : ActionResponse = { actionType: ActionType.NONE, - result: "" + responseType: ResponseType.NONE, + result: "", }; protected completionResult_ : string = ""; - protected textareaValue_?: string; + protected query_?: string; + protected submittedQuery_?: string; constructor() { super(); @@ -48,7 +56,8 @@ export class ChatAppElement extends CrLitElement { askAnythingLabel_: {type: String}, actionList_: {type: Array}, submitResponse_: {type: Object}, - textareaValue_: {type: String}, + query_: {type: String}, + submittedQuery_: {type: String}, completionResult_: {type: String}, }; } @@ -63,23 +72,47 @@ export class ChatAppElement extends CrLitElement { } private async updateSubmitResponse(response: ActionResponse) { - //this.submitResponse_ = response; - this.completionResult_ += response.result; + // todo: to properly display error message + if (response.responseType == ResponseType.DELTA) { + this.completionResult_ += response.result; + } else if (response.responseType == ResponseType.COMPLETED) { + this.completionResult_ += "\n"; + } else if (response.responseType == ResponseType.ERROR) { + this.completionResult_ += "\n" ; + } } protected onSubmitAction_(e: Event) { e.stopPropagation(); - this.completionResult_ += "\n\n"; + if (this.conversations_ != null) { + this.conversations_.push({query: "Summarise this page", response: ""}); + } this.chatApiProxy_.submitAction(ActionType.SUMMARIZE_PAGE); } protected onTextareaValueChanged_(e: CustomEvent<{value: string}>) { - this.textareaValue_ = e.detail.value; + this.query_ = e.detail.value; } protected async onSubmitQuery_() { - console.log(this.textareaValue_); - this.chatApiProxy_.submitQuery(ActionType.QUERY, this.textareaValue_ ?? "" ); + if (this.completionResult_ && this.completionResult_.length > 0) { + if (this.conversations_ != null) { + if (this.conversations_.length == 0) { + this.conversations_.push({query: this.query_ ?? "", response: this.completionResult_}); + } else { + const lastIndex = this.conversations_.length - 1; + const lastConversation = this.conversations_[lastIndex]; + if (lastConversation) { + lastConversation.response = this.completionResult_; + } + } + } + } + this.completionResult_ = ""; + this.submittedQuery_ = this.query_; + this.conversations_.push({query: this.query_ ?? "", response: ""}); + this.query_ = ""; + this.chatApiProxy_.submitQuery(ActionType.QUERY, this.submittedQuery_ ?? "" ); } override connectedCallback() { diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc index 7b929c01795a45..fc31beac1ed9d3 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc @@ -142,41 +142,52 @@ void CompletionApiClient::OnQueryCompleted( const bool success = result.Is2XXResponseCode(); // Handle successful request if (success) { - std::string completion = ""; - // We're checking for a value body in case for non-streaming API results. - if (result.value_body().is_dict()) { - const std::string* value = - result.value_body().GetDict().FindString("choices"); - if (value) { - // Trimming necessary for Llama 2 which prepends responses with a " ". - completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); - } - } - std::string entire_message; - for (const auto& s : entire_completion_result) { - entire_message += s; - } - DVLOG(0) << std::endl << std::endl << entire_message << std::endl << std::endl; - - entire_completion_result.clear(); - // std::move(callback).Run(base::ok(std::move(completion))); - return; + // std::string completion = ""; + // // We're checking for a value body in case for non-streaming API + // results. if (result.value_body().is_dict()) { + // const std::string* value = + // result.value_body().GetDict().FindString("choices"); + // if (value) { + // // Trimming necessary for Llama 2 which prepends + // responses with a " ". completion = + // base::TrimWhitespaceASCII(*value, base::TRIM_ALL); + // } + // } + // std::string entire_message; + // for (const auto& s : entire_completion_result) { + // entire_message += s; + // } + // // DVLOG(0) << std::endl << std::endl << entire_message << + // std::endl << std::endl; + + entire_completion_result.clear(); + std::move(callback).Run(base::ok("")); + return; } // Handle error - chat::mojom::APIError error; - + chat::mojom::APIErrorType error; + + LOG(INFO) << "Error response_code: " << result.response_code(); + LOG(INFO) << "Error error_code: " << result.error_code(); + if (result.value_body().is_dict()) { + const std::string* value = + result.value_body().GetDict().FindString("message"); + if (value) { + // Trimming necessary for Llama 2 which prepends responses with a " ". + auto completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); + LOG(INFO) << "Error message: " << completion; + } + } if (net::HTTP_TOO_MANY_REQUESTS == result.response_code()) { - error = chat::mojom::APIError::RateLimitReached; + error = chat::mojom::APIErrorType::RateLimitReached; } else if (net::HTTP_REQUEST_ENTITY_TOO_LARGE == result.response_code()) { - error = chat::mojom::APIError::ContextLimitReached; + error = chat::mojom::APIErrorType::ContextLimitReached; } else { - error = chat::mojom::APIError::ConnectionError; + error = chat::mojom::APIErrorType::ConnectionError; } - auto _ = error; - - // std::move(callback).Run(base::unexpected(std::move(error))); + std::move(callback).Run(base::unexpected(std::move(error))); } diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h index 5dd09ae5666c51..e5c45ab01355d1 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h @@ -19,31 +19,30 @@ namespace network { class SharedURLLoaderFactory; } // namespace network - using api_request_helper::APIRequestResult; class CompletionApiClient { public: - using GenerationResult = base::expected; - using GenerationDataCallback = base::RepeatingCallback; - using GenerationCompletedCallback = - base::OnceCallback; - - CompletionApiClient( - scoped_refptr url_loader_factory); - - CompletionApiClient(const CompletionApiClient&) = delete; - CompletionApiClient& operator=(const CompletionApiClient&) = delete; - virtual ~CompletionApiClient(); - - // This function queries both types of APIs: SSE and non-SSE. - // In non-SSE cases, only the data_completed_callback will be triggered. - virtual void QueryPrompt( - const std::string& prompt, - GenerationCompletedCallback data_completed_callback, - GenerationDataCallback data_received_callback = base::NullCallback()); - // Clears all in-progress requests - void ClearAllQueries(); + using GenerationResult = + base::expected; + using GenerationDataCallback = base::RepeatingCallback; + using GenerationCompletedCallback = base::OnceCallback; + + CompletionApiClient( + scoped_refptr url_loader_factory); + + CompletionApiClient(const CompletionApiClient&) = delete; + CompletionApiClient& operator=(const CompletionApiClient&) = delete; + virtual ~CompletionApiClient(); + + // This function queries both types of APIs: SSE and non-SSE. + // In non-SSE cases, only the data_completed_callback will be triggered. + virtual void QueryPrompt( + const std::string& prompt, + GenerationCompletedCallback data_completed_callback, + GenerationDataCallback data_received_callback = base::NullCallback()); + // Clears all in-progress requests + void ClearAllQueries(); private: void OnQueryDataReceived(GenerationDataCallback callback, diff --git a/chrome/browser/ui/webui/side_panel/chat/chat.mojom b/chrome/browser/ui/webui/side_panel/chat/chat.mojom index 140d69fe39c986..627142709c544b 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat.mojom +++ b/chrome/browser/ui/webui/side_panel/chat/chat.mojom @@ -2,22 +2,18 @@ module chat.mojom; import "url/mojom/url.mojom"; -enum APIError { +enum APIErrorType { None, ConnectionError, RateLimitReached, ContextLimitReached, }; -struct SiteInfo { - // The title of the currently active tab, if it has an open page - string? title; - - // The url of the currently active tab if it has opening page or probaly file which we might support in future - string? url; - - // Indicates whether the current URL contains content that can be used in conversations with Yep Chat - bool is_content_usable_in_conversations; +enum ResponseType { + NONE, + DELTA, + COMPLETED, + ERROR, }; enum ActionType { @@ -30,6 +26,22 @@ enum ActionType { NONE, }; +struct APIError { + APIErrorType error_type; + string error_message; +}; + +struct SiteInfo { + // The title of the currently active tab, if it has an open page + string? title; + + // The url of the currently active tab if it has opening page or probaly file which we might support in future + string? url; + + // Indicates whether the current URL contains content that can be used in conversations with Yep Chat + bool is_content_usable_in_conversations; +}; + struct ActionItem { ActionType action_type; string label; @@ -37,6 +49,7 @@ struct ActionItem { struct ActionResponse { ActionType action_type; + ResponseType response_type; string result; }; diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 15eb8f027500f3..e1cb17ca9e2cc3 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -79,6 +79,8 @@ void ChatPageHandler::CloseUI() { } void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents) { + // call the constructor of ChatContextObserver and set current WebContents to PageContentExtractorHelperDelegate + // todo: this should be removed when refactor and pass WebContents directly to extractor or extractor_helper active_chat_context_observer_ = ai_chat::ChatContextObserver::FromWebContents(contents); web_content_observer_ = std::make_unique( @@ -143,12 +145,6 @@ void ChatPageHandler::GetActionList(GetActionListCallback callback) { std::move(callback).Run(std::move(action_items)); } - -void ChatPageHandler::SubmitQueryCompletedCallback( - base::expected result) { - LOG(INFO) << result.has_value(); -} - base::WeakPtr ChatPageHandler::GetWeakPtr() { return weak_ptr_factory_.GetWeakPtr(); } @@ -181,7 +177,8 @@ void ChatPageHandler::OnPageContentExtracted( api_client_->QueryPrompt( "Provide a brief summary of the key takeaways for the following:" + content, - base::NullCallback(), + base::BindOnce(&ChatPageHandler::SubmitQueryCompletedCallback, + base::Unretained(this), action_type), base::BindRepeating(&ChatPageHandler::SubmitQueryCallback, base::Unretained(this), action_type)); } @@ -190,11 +187,35 @@ void ChatPageHandler::SubmitQueryCallback(chat::mojom::ActionType action_type, std::string completion) { chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; + response->response_type = chat::mojom::ResponseType::DELTA; response->result = completion; page_->OnSubmitActionResponse(response.Clone()); } +void ChatPageHandler::SubmitQueryCompletedCallback( + chat::mojom::ActionType action_type, + base::expected result) { + chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); + response->action_type = action_type; + if (result.has_value()) { + LOG(INFO) << __func__ << " success -> " << result.value(); + response->response_type = chat::mojom::ResponseType::COMPLETED; + response->result = result.value(); + } else { + LOG(INFO) << __func__ << " error -> " << result.error(); + response->response_type = chat::mojom::ResponseType::ERROR; + response->result = + "error_message_here"; // todo: to get error message from api response + // and pass it to chat UI + } + page_->OnSubmitActionResponse(response.Clone()); +} + void ChatPageHandler::SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) { - // todo: use proper callback - api_client_->QueryPrompt(query, base::NullCallback(), base::NullCallback()); + api_client_->QueryPrompt( + query, + base::BindOnce(&ChatPageHandler::SubmitQueryCompletedCallback, + base::Unretained(this), action_type), + base::BindRepeating(&ChatPageHandler::SubmitQueryCallback, + base::Unretained(this), action_type)); } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 47cac113397c33..1a4015c638880f 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -54,7 +54,8 @@ class ChatPageHandler : public chat::mojom::PageHandler { void SubmitQueryCallback(chat::mojom::ActionType action_type, std::string completion); void SubmitQueryCompletedCallback( - base::expected result); + chat::mojom::ActionType action_type, + base::expected result); base::WeakPtr GetWeakPtr(); From 22c7b8b904eef8005d08b9d9a3be51f39333c174 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 26 Nov 2024 11:30:33 +0800 Subject: [PATCH 25/34] Get latest site info when chat UI is shown --- chrome/browser/resources/side_panel/chat/chat_app.ts | 7 ++++++- .../browser/ui/webui/side_panel/chat/chat_page_handler.cc | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index bd14259a7cd5c9..77abc528e7ed93 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -118,7 +118,12 @@ export class ChatAppElement extends CrLitElement { override connectedCallback() { super.connectedCallback(); - setTimeout(() => this.chatApiProxy_.showUI(), 0); + setTimeout(async () => { + this.chatApiProxy_.showUI(); + const {siteInfo} = await this.chatApiProxy_.getSiteInfo(); + await this.updateSiteInfo(siteInfo); + this.updateComplete; + }, 0); this.listenerIds_.push( this.chatApiProxy_.getCallbackRouter().onSiteInfoChanged.addListener( diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index e1cb17ca9e2cc3..a643e36401e047 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -79,12 +79,13 @@ void ChatPageHandler::CloseUI() { } void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents) { - // call the constructor of ChatContextObserver and set current WebContents to PageContentExtractorHelperDelegate // todo: this should be removed when refactor and pass WebContents directly to extractor or extractor_helper active_chat_context_observer_ = ai_chat::ChatContextObserver::FromWebContents(contents); web_content_observer_ = std::make_unique( contents, *this); + + chat_context_web_contents_ = contents; if (page_.is_bound()) { page_->OnSiteInfoChanged(std::move(site_info)); } From 7f5848000fdaacbb24f2189141464c3157aa844b Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 26 Nov 2024 12:49:27 +0800 Subject: [PATCH 26/34] Refactor content extraction --- ...ontent_browser_client_receiver_bindings.cc | 426 +++++++++--------- chrome/browser/ui/BUILD.gn | 2 - chrome/browser/ui/tabs/tab_features.cc | 3 - .../side_panel/chat/chat_page_handler.cc | 38 +- .../webui/side_panel/chat/chat_page_handler.h | 21 +- .../chat/page_content_extractor_helper.cc | 13 +- .../chat/page_content_extractor_helper.h | 15 +- 7 files changed, 227 insertions(+), 291 deletions(-) diff --git a/chrome/browser/chrome_content_browser_client_receiver_bindings.cc b/chrome/browser/chrome_content_browser_client_receiver_bindings.cc index e266624e8a230c..d229d22dbf66dc 100644 --- a/chrome/browser/chrome_content_browser_client_receiver_bindings.cc +++ b/chrome/browser/chrome_content_browser_client_receiver_bindings.cc @@ -25,7 +25,6 @@ #include "chrome/browser/signin/google_accounts_private_api_host.h" #include "chrome/browser/supervised_user/supervised_user_navigation_observer.h" #include "chrome/browser/trusted_vault/trusted_vault_encryption_keys_tab_helper.h" -#include "chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h" #include "chrome/common/buildflags.h" #include "chrome/common/chat/page_content_extractor.mojom.h" #include "chrome/common/chrome_features.h" @@ -391,255 +390,244 @@ void ChromeContentBrowserClient:: RegisterAssociatedInterfaceBindersForRenderFrameHost( content::RenderFrameHost& render_frame_host, blink::AssociatedInterfaceRegistry& associated_registry) { - for (auto& ep : extra_parts_) { - ep->ExposeInterfacesToRendererForRenderFrameHost(render_frame_host, - associated_registry); - } + for (auto &ep: extra_parts_) { + ep->ExposeInterfacesToRendererForRenderFrameHost(render_frame_host, + associated_registry); + } - associated_registry.AddInterface( - base::BindRepeating( - &autofill::ContentAutofillDriverFactory::BindAutofillDriver, - &render_frame_host)); - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - autofill::mojom::PasswordGenerationDriver> receiver) { - ChromePasswordManagerClient::BindPasswordGenerationDriver( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface< - autofill::mojom::PasswordManagerDriver>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - password_manager::ContentPasswordManagerDriverFactory:: - BindPasswordManagerDriver(std::move(receiver), render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - chrome_browser_net::NetErrorTabHelper::BindNetworkDiagnostics( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - chrome_browser_net::NetErrorTabHelper::BindNetworkEasterEgg( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - chrome_browser_net::NetErrorTabHelper::BindNetErrorPageSupport( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + &autofill::ContentAutofillDriverFactory::BindAutofillDriver, + &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + autofill::mojom::PasswordGenerationDriver> receiver) { + ChromePasswordManagerClient::BindPasswordGenerationDriver( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); + associated_registry.AddInterface< + autofill::mojom::PasswordManagerDriver>(base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + password_manager::ContentPasswordManagerDriverFactory:: + BindPasswordManagerDriver(std::move(receiver), render_frame_host); + }, + &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + chrome_browser_net::NetErrorTabHelper::BindNetworkDiagnostics( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + chrome_browser_net::NetErrorTabHelper::BindNetworkEasterEgg( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + chrome_browser_net::NetErrorTabHelper::BindNetErrorPageSupport( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); #if BUILDFLAG(ENABLE_PLUGINS) - associated_registry.AddInterface< - chrome::mojom::PluginAuthHost>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - extensions::ChromeWebViewPermissionHelperDelegate::BindPluginAuthHost( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface< + chrome::mojom::PluginAuthHost>(base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + extensions::ChromeWebViewPermissionHelperDelegate::BindPluginAuthHost( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); #endif #if BUILDFLAG(ENABLE_PLUGINS) || BUILDFLAG(IS_ANDROID) #if BUILDFLAG(IS_ANDROID) - using PluginObserverImpl = PluginObserverAndroid; + using PluginObserverImpl = PluginObserverAndroid; #else - using PluginObserverImpl = PluginObserver; + using PluginObserverImpl = PluginObserver; #endif - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - PluginObserverImpl::BindPluginHost(std::move(receiver), - render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + PluginObserverImpl::BindPluginHost(std::move(receiver), + render_frame_host); + }, + &render_frame_host)); #endif // BUILDFLAG(ENABLE_PLUGINS) || BUILDFLAG(IS_ANDROID) - associated_registry.AddInterface< - chrome::mojom::TrustedVaultEncryptionKeysExtension>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - chrome::mojom::TrustedVaultEncryptionKeysExtension> receiver) { - TrustedVaultEncryptionKeysTabHelper:: - BindTrustedVaultEncryptionKeysExtension(std::move(receiver), - render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface< - chrome::mojom::GoogleAccountsPrivateApiExtension>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - chrome::mojom::GoogleAccountsPrivateApiExtension> receiver) { - GoogleAccountsPrivateApiHost::BindHost(std::move(receiver), - render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface< - content_capture::mojom::ContentCaptureReceiver>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - content_capture::mojom::ContentCaptureReceiver> receiver) { - content_capture::OnscreenContentProvider::BindContentCaptureReceiver( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface< + chrome::mojom::TrustedVaultEncryptionKeysExtension>(base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + chrome::mojom::TrustedVaultEncryptionKeysExtension> receiver) { + TrustedVaultEncryptionKeysTabHelper:: + BindTrustedVaultEncryptionKeysExtension(std::move(receiver), + render_frame_host); + }, + &render_frame_host)); + associated_registry.AddInterface< + chrome::mojom::GoogleAccountsPrivateApiExtension>(base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + chrome::mojom::GoogleAccountsPrivateApiExtension> receiver) { + GoogleAccountsPrivateApiHost::BindHost(std::move(receiver), + render_frame_host); + }, + &render_frame_host)); + associated_registry.AddInterface< + content_capture::mojom::ContentCaptureReceiver>(base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + content_capture::mojom::ContentCaptureReceiver> receiver) { + content_capture::OnscreenContentProvider::BindContentCaptureReceiver( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); #if BUILDFLAG(ENABLE_EXTENSIONS_CORE) - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - extensions::ExtensionWebContentsObserver::BindLocalFrameHost( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + extensions::ExtensionWebContentsObserver::BindLocalFrameHost( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); #endif // BUILDFLAG(ENABLE_EXTENSIONS_CORE) #if BUILDFLAG(ENABLE_OFFLINE_PAGES) - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - offline_pages::mojom::MhtmlPageNotifier> receiver) { - offline_pages::OfflinePageTabHelper::BindHtmlPageNotifier( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver< + offline_pages::mojom::MhtmlPageNotifier> receiver) { + offline_pages::OfflinePageTabHelper::BindHtmlPageNotifier( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); #endif // BUILDFLAG(ENABLE_OFFLINE_PAGES) - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - page_load_metrics::mojom::PageLoadMetrics> receiver) { - page_load_metrics::MetricsWebContentsObserver::BindPageLoadMetrics( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + page_load_metrics::mojom::PageLoadMetrics> receiver) { + page_load_metrics::MetricsWebContentsObserver::BindPageLoadMetrics( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); #if BUILDFLAG(ENABLE_PDF) - associated_registry.AddInterface(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver receiver) { - pdf::PDFDocumentHelper::BindPdfHost( - std::move(receiver), render_frame_host, - std::make_unique()); - }, - &render_frame_host)); + associated_registry.AddInterface(base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver receiver) { + pdf::PDFDocumentHelper::BindPdfHost( + std::move(receiver), render_frame_host, + std::make_unique()); + }, + &render_frame_host)); #endif // BUILDFLAG(ENABLE_PDF) #if !BUILDFLAG(IS_ANDROID) - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - search::mojom::EmbeddedSearchConnector> receiver) { - SearchTabHelper::BindEmbeddedSearchConnecter(std::move(receiver), - render_frame_host); - }, - &render_frame_host)); + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + search::mojom::EmbeddedSearchConnector> receiver) { + SearchTabHelper::BindEmbeddedSearchConnecter(std::move(receiver), + render_frame_host); + }, + &render_frame_host)); #endif // !BUILDFLAG(IS_ANDROID) #if BUILDFLAG(ENABLE_PRINTING) - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver - receiver) { - if (headless::IsHeadlessMode()) { - headless::HeadlessPrintManager::BindPrintManagerHost( - std::move(receiver), render_frame_host); - } else { + associated_registry.AddInterface( + base::BindRepeating( + [](content::RenderFrameHost* render_frame_host, + mojo::PendingAssociatedReceiver + receiver) { + if (headless::IsHeadlessMode()) { + headless::HeadlessPrintManager::BindPrintManagerHost( + std::move(receiver), render_frame_host); + } else { #if BUILDFLAG(IS_CHROMEOS) - if (base::FeatureList::IsEnabled( - ::features::kPrintPreviewCrosPrimary)) { + if (base::FeatureList::IsEnabled( + ::features::kPrintPreviewCrosPrimary)) { #if BUILDFLAG(ENABLE_PRINT_PREVIEW) - chromeos::PrintViewManagerCros::BindPrintManagerHost( - std::move(receiver), render_frame_host); + chromeos::PrintViewManagerCros::BindPrintManagerHost( + std::move(receiver), render_frame_host); #else - chromeos::PrintViewManagerCrosBasic::BindPrintManagerHost( - std::move(receiver), render_frame_host); + chromeos::PrintViewManagerCrosBasic::BindPrintManagerHost( + std::move(receiver), render_frame_host); #endif // BUILDFLAG(ENABLE_PRINT_PREVIEW) - return; - } + return; + } #endif // BUILDFLAG(CHROMEOS) #if BUILDFLAG(ENABLE_PRINT_PREVIEW) - printing::PrintViewManager::BindPrintManagerHost( - std::move(receiver), render_frame_host); + printing::PrintViewManager::BindPrintManagerHost( + std::move(receiver), render_frame_host); #else - printing::PrintViewManagerBasic::BindPrintManagerHost( - std::move(receiver), render_frame_host); + printing::PrintViewManagerBasic::BindPrintManagerHost( + std::move(receiver), render_frame_host); #endif // BUILDFLAG(ENABLE_PRINT_PREVIEW) - } - }, - &render_frame_host)); + } + }, + &render_frame_host)); #endif // BUILDFLAG(ENABLE_PRINTING) - associated_registry.AddInterface< - security_interstitials::mojom::InterstitialCommands>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - security_interstitials::mojom::InterstitialCommands> receiver) { - security_interstitials::SecurityInterstitialTabHelper:: - BindInterstitialCommands(std::move(receiver), render_frame_host); - }, - &render_frame_host)); - associated_registry.AddInterface< - subresource_filter::mojom::SubresourceFilterHost>(base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - subresource_filter::mojom::SubresourceFilterHost> receiver) { - subresource_filter::ContentSubresourceFilterThrottleManager:: - BindReceiver(std::move(receiver), render_frame_host); - }, - &render_frame_host)); - if (fingerprinting_protection_filter::features:: - IsFingerprintingProtectionFeatureEnabled()) { associated_registry.AddInterface< - fingerprinting_protection_filter::mojom::FingerprintingProtectionHost>( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, + security_interstitials::mojom::InterstitialCommands>(base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, mojo::PendingAssociatedReceiver< - fingerprinting_protection_filter::mojom:: - FingerprintingProtectionHost> receiver) { - fingerprinting_protection_filter::ThrottleManager::BindReceiver( - std::move(receiver), render_frame_host); + security_interstitials::mojom::InterstitialCommands> receiver) { + security_interstitials::SecurityInterstitialTabHelper:: + BindInterstitialCommands(std::move(receiver), render_frame_host); }, &render_frame_host)); - } - associated_registry - .AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - supervised_user::mojom::SupervisedUserCommands> receiver) { - SupervisedUserNavigationObserver::BindSupervisedUserCommands( - std::move(receiver), render_frame_host); - }, - &render_frame_host)); - - // AI Chat page content extraction renderer -> browser interface - associated_registry.AddInterface( - base::BindRepeating( - [](content::RenderFrameHost* render_frame_host, - mojo::PendingAssociatedReceiver< - chat::mojom::PageContentExtractorHost> receiver) { - ai_chat::ChatContextObserver::BindPageContentExtractorHost( - render_frame_host, std::move(receiver)); - }, - &render_frame_host)); + associated_registry.AddInterface< + subresource_filter::mojom::SubresourceFilterHost>(base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + subresource_filter::mojom::SubresourceFilterHost> receiver) { + subresource_filter::ContentSubresourceFilterThrottleManager:: + BindReceiver(std::move(receiver), render_frame_host); + }, + &render_frame_host)); + if (fingerprinting_protection_filter::features:: + IsFingerprintingProtectionFeatureEnabled()) { + associated_registry.AddInterface< + fingerprinting_protection_filter::mojom::FingerprintingProtectionHost>( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + fingerprinting_protection_filter::mojom:: + FingerprintingProtectionHost> receiver) { + fingerprinting_protection_filter::ThrottleManager::BindReceiver( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); + } + associated_registry + .AddInterface( + base::BindRepeating( + [](content::RenderFrameHost *render_frame_host, + mojo::PendingAssociatedReceiver< + supervised_user::mojom::SupervisedUserCommands> receiver) { + SupervisedUserNavigationObserver::BindSupervisedUserCommands( + std::move(receiver), render_frame_host); + }, + &render_frame_host)); } void ChromeContentBrowserClient::BindGpuHostReceiver( diff --git a/chrome/browser/ui/BUILD.gn b/chrome/browser/ui/BUILD.gn index b5914795d71fcf..5dfc032501558d 100644 --- a/chrome/browser/ui/BUILD.gn +++ b/chrome/browser/ui/BUILD.gn @@ -1569,8 +1569,6 @@ static_library("ui") { "webui/side_panel/chat/api/api_request_helper.h", "webui/side_panel/chat/api/completion_api_client.cc", "webui/side_panel/chat/api/completion_api_client.h", - "webui/side_panel/chat/chat_context_observer.cc", - "webui/side_panel/chat/chat_context_observer.h", "webui/side_panel/chat/chat_page_handler.cc", "webui/side_panel/chat/chat_page_handler.h", "webui/side_panel/chat/page_content_extractor_helper.cc", diff --git a/chrome/browser/ui/tabs/tab_features.cc b/chrome/browser/ui/tabs/tab_features.cc index ef6d8614060c52..6f8f662f93deff 100644 --- a/chrome/browser/ui/tabs/tab_features.cc +++ b/chrome/browser/ui/tabs/tab_features.cc @@ -39,7 +39,6 @@ #include "components/fingerprinting_protection_filter/common/fingerprinting_protection_filter_features.h" #include "components/image_fetcher/core/image_fetcher_service.h" #include "components/permissions/permission_indicators_tab_data.h" -#include "chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h" namespace tabs { @@ -156,8 +155,6 @@ void TabFeatures::Init(TabInterface& tab, Profile* profile) { if (web_app::WebAppMetricsTabHelper::IsEnabled(tab.GetContents())) { web_app::WebAppMetricsTabHelper::CreateForWebContents(tab.GetContents()); } - - ai_chat::ChatContextObserver::CreateForWebContents(tab.GetContents()); } TabFeatures::TabFeatures() = default; diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index a643e36401e047..35f08b6d680e94 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -22,13 +22,6 @@ #include "ui/base/l10n/l10n_util.h" #include "url/gurl.h" -ChatPageHandler::ChatWebContentsObserver::ChatWebContentsObserver( - content::WebContents* web_contents, - ChatPageHandler& page_handler) - : content::WebContentsObserver(web_contents), page_handler_(page_handler) {} - -ChatPageHandler::ChatWebContentsObserver::~ChatWebContentsObserver() = default; - ChatPageHandler::ChatPageHandler( mojo::PendingReceiver receiver, mojo::PendingRemote page, @@ -41,30 +34,18 @@ ChatPageHandler::ChatPageHandler( chat_ui_(chat_ui), owner_web_contents_(owner_web_contents), chat_context_web_contents_(chat_context_web_contents), - profile_(Profile::FromWebUI(web_ui)) { + profile_(Profile::FromWebUI(web_ui)), + page_content_extractor_helper_(std::make_unique(chat_context_web_contents)) + { scoped_refptr url_loader_factory = profile_->GetDefaultStoragePartition() ->GetURLLoaderFactoryForBrowserProcess(); api_client_ = std::make_unique(std::move(url_loader_factory)); - - active_chat_context_observer_ = - ai_chat::ChatContextObserver::FromWebContents(chat_context_web_contents); - web_content_observer_ = std::make_unique( - chat_context_web_contents, *this); } ChatPageHandler::~ChatPageHandler() = default; -void ChatPageHandler::ChatWebContentsObserver::WebContentsDestroyed() { - page_handler_->HandleWebContentsDestroyed(); -} - -void ChatPageHandler::HandleWebContentsDestroyed() { - active_chat_context_observer_ = nullptr; - web_content_observer_.reset(); -} - void ChatPageHandler::ShowUI() { auto embedder = chat_ui_->embedder(); if (embedder) { @@ -79,12 +60,7 @@ void ChatPageHandler::CloseUI() { } void ChatPageHandler::SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents) { - // todo: this should be removed when refactor and pass WebContents directly to extractor or extractor_helper - active_chat_context_observer_ = - ai_chat::ChatContextObserver::FromWebContents(contents); - web_content_observer_ = std::make_unique( - contents, *this); - + page_content_extractor_helper_ = std::make_unique(contents); chat_context_web_contents_ = contents; if (page_.is_bound()) { page_->OnSiteInfoChanged(std::move(site_info)); @@ -156,9 +132,9 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { DVLOG(0) << "ActionType: Summarize_page"; - active_chat_context_observer_->GetPageContent( - base::BindOnce(&ChatPageHandler::OnPageContentExtracted, - base::Unretained(this), action_type)); + page_content_extractor_helper_->ExtractPageContent( + base::BindOnce(&ChatPageHandler::OnPageContentExtracted, + base::Unretained(this), action_type)); } else { // todo: to implement for other action types later diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 1a4015c638880f..4cd7b0aa6cbf64 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -21,6 +21,7 @@ #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/remote.h" +#include "page_content_extractor_helper.h" namespace content { class WebContents; @@ -60,21 +61,6 @@ class ChatPageHandler : public chat::mojom::PageHandler { base::WeakPtr GetWeakPtr(); private: - class ChatWebContentsObserver : public content::WebContentsObserver { - public: - explicit ChatWebContentsObserver(content::WebContents* web_contents, - ChatPageHandler& page_handler); - ~ChatWebContentsObserver() override; - - private: - // content::WebContentsObserver - void WebContentsDestroyed() override; - - raw_ref page_handler_; - }; - - void HandleWebContentsDestroyed(); - void OnPageContentExtracted(chat::mojom::ActionType action_type, std::string content); @@ -85,10 +71,7 @@ class ChatPageHandler : public chat::mojom::PageHandler { raw_ptr chat_context_web_contents_ = nullptr; const raw_ptr profile_; std::unique_ptr api_client_ = nullptr; - - raw_ptr active_chat_context_observer_ = nullptr; - std::unique_ptr web_content_observer_; - + std::unique_ptr page_content_extractor_helper_; base::WeakPtrFactory weak_ptr_factory_{this}; }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc index e2333897c85f22..672a25da93282b 100644 --- a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc +++ b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc @@ -25,7 +25,6 @@ #include "services/service_manager/public/cpp/interface_provider.h" #include "content/public/browser/web_contents.h" -namespace ai_chat { namespace { constexpr auto kVideoPageContentTypes = @@ -33,15 +32,12 @@ constexpr auto kVideoPageContentTypes = {chat::mojom::PageContentType::VideoTranscriptYouTube, chat::mojom::PageContentType::VideoTranscriptVTT}); -using ExtractPageContentCallback = - ChatContextObserver::ExtractPageContentCallback; - class PageContentExtractorInternal { public: PageContentExtractorInternal() {} void Start(mojo::Remote content_extractor, - ExtractPageContentCallback callback) { + base::OnceCallback callback) { content_extractor_ = std::move(content_extractor); if (!content_extractor_) { DeleteSelf(); @@ -62,7 +58,7 @@ class PageContentExtractorInternal { base::Unretained(this), std::move(callback))); } - void OnPageContentExtracted(ExtractPageContentCallback callback, + void OnPageContentExtracted(base::OnceCallback callback, chat::mojom::PageContentPtr data) { if (!data) { DVLOG(0) << __func__ << " no extracted page content."; @@ -88,7 +84,7 @@ class PageContentExtractorInternal { private: void DeleteSelf() { delete this; } - void SendResultAndDeleteSelf(ExtractPageContentCallback callback, + void SendResultAndDeleteSelf(base::OnceCallback callback, std::string content = "") { std::move(callback).Run(content); delete this; @@ -105,7 +101,7 @@ PageContentExtractorHelper::PageContentExtractorHelper( PageContentExtractorHelper::~PageContentExtractorHelper() = default; void PageContentExtractorHelper::ExtractPageContent( - ChatContextObserver::ExtractPageContentCallback callback) { + base::OnceCallback callback) { auto* primary_rfh = web_contents_->GetPrimaryMainFrame(); DCHECK(primary_rfh->IsRenderFrameLive()); @@ -117,4 +113,3 @@ void PageContentExtractorHelper::ExtractPageContent( internal_extractor->Start(std::move(extractor), std::move(callback)); } -} // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h index 3e7b2ff2977a67..ad82e4527e74c0 100644 --- a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h +++ b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h @@ -4,25 +4,24 @@ #include #include "base/functional/callback_forward.h" -#include "chat_context_observer.h" #include "chrome/renderer/chat/page_content_extractor.h" -#include "chat_context_observer.h" -namespace ai_chat { -class PageContentExtractorHelper -: public ChatContextObserver::PageContentExtractorHelperDelegate { +namespace content { + class WebContents; +} // namespace content + +class PageContentExtractorHelper{ public: explicit PageContentExtractorHelper(content::WebContents* web_contents); - ~PageContentExtractorHelper() override; + ~PageContentExtractorHelper(); PageContentExtractorHelper(const PageContentExtractorHelper&) = delete; PageContentExtractorHelper& operator=(const PageContentExtractorHelper&) = delete; - void ExtractPageContent(ChatContextObserver::ExtractPageContentCallback callback) override; + void ExtractPageContent( base::OnceCallback callback); private: raw_ptr web_contents_; }; -} // namespace ai_chat #endif // CHROMIUM_PAGE_CONTENT_EXTRACTOR_HELPER_H From 9c43341046ac453a9e498e3930f1f8a582568cf8 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 26 Nov 2024 16:53:24 +0800 Subject: [PATCH 27/34] Remove ChatContextObserver and related code Deleted ChatContextObserver class and its dependencies to simplify codebase. Adjusted related headers and implementation files to reflect the removal, and corrected remaining references and comments accordingly. --- .../browser/chrome_content_browser_client.cc | 20 -- ...ontent_browser_client_receiver_bindings.cc | 1 - .../side_panel/chat/api/api_request_helper.cc | 30 +-- .../chat/api/completion_api_client.cc | 42 +--- .../side_panel/chat/chat_context_observer.cc | 185 ------------------ .../side_panel/chat/chat_context_observer.h | 107 ---------- .../side_panel/chat/chat_page_handler.cc | 11 +- .../webui/side_panel/chat/chat_page_handler.h | 8 +- .../chat/page_content_extractor_helper.cc | 90 ++++----- .../chat/page_content_extractor_helper.h | 5 +- chrome/common/chat/BUILD.gn | 1 - .../common/chat/page_content_extractor.mojom | 28 +-- .../renderer/chat/page_content_extractor.cc | 20 +- chrome/renderer/chat/page_content_extractor.h | 2 +- 14 files changed, 83 insertions(+), 467 deletions(-) delete mode 100644 chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc delete mode 100644 chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h diff --git a/chrome/browser/chrome_content_browser_client.cc b/chrome/browser/chrome_content_browser_client.cc index 2b33fde759891d..5c0a66583d4153 100644 --- a/chrome/browser/chrome_content_browser_client.cc +++ b/chrome/browser/chrome_content_browser_client.cc @@ -420,10 +420,6 @@ #include "url/origin.h" #include "url/third_party/mozilla/url_parse.h" #include "url/url_constants.h" -// #include "chrome/common/chat/page_content_extractor.mojom.h" -// #include "chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h" -// #include -// "third_party/blink/public/common/associated_interfaces/associated_interface_registry.h" #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC) #include "chrome/browser/enterprise/platform_auth/platform_auth_navigation_throttle.h" @@ -8862,19 +8858,3 @@ base::ReadOnlySharedMemoryRegion ChromeContentBrowserClient::GetGlobalPerformanceScenarioRegion() { return performance_manager::GetGlobalSharedScenarioRegion(); } - -// void ChromeContentBrowserClient:: -// RegisterAssociatedInterfaceBindersForRenderFrameHost( -// content::RenderFrameHost& render_frame_host, -// blink::AssociatedInterfaceRegistry& associated_registry) { -// // AI Chat page content extraction renderer -> browser interface -// associated_registry.AddInterface( -// base::BindRepeating( -// [](content::RenderFrameHost* render_frame_host, -// mojo::PendingAssociatedReceiver< -// chat::mojom::PageContentExtractorHost> receiver) { -// ai_chat::ChatContextObserver::BindPageContentExtractorHost( -// render_frame_host, std::move(receiver)); -// }, -// &render_frame_host)); -// } diff --git a/chrome/browser/chrome_content_browser_client_receiver_bindings.cc b/chrome/browser/chrome_content_browser_client_receiver_bindings.cc index d229d22dbf66dc..51a516ab04e4de 100644 --- a/chrome/browser/chrome_content_browser_client_receiver_bindings.cc +++ b/chrome/browser/chrome_content_browser_client_receiver_bindings.cc @@ -26,7 +26,6 @@ #include "chrome/browser/supervised_user/supervised_user_navigation_observer.h" #include "chrome/browser/trusted_vault/trusted_vault_encryption_keys_tab_helper.h" #include "chrome/common/buildflags.h" -#include "chrome/common/chat/page_content_extractor.mojom.h" #include "chrome/common/chrome_features.h" #include "components/autofill/content/browser/content_autofill_driver_factory.h" #include "components/content_capture/browser/onscreen_content_provider.h" diff --git a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc index f33b1d562f044f..1b5318c135bcfa 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/api_request_helper.cc @@ -240,7 +240,7 @@ namespace api_request_helper { request->method = method; } - DVLOG(4) << method << " " << url.spec(); + DVLOG(0) << method << " " << url.spec(); if (!headers.empty()) { for (auto entry : headers) { @@ -250,8 +250,8 @@ namespace api_request_helper { } if (!payload.empty()) { - DVLOG(4) << "Payload type " << payload_content_type << ":"; - DVLOG(4) << payload; + DVLOG(0) << "Payload type " << payload_content_type << ":"; + DVLOG(0) << payload; } auto url_loader = @@ -348,7 +348,7 @@ namespace api_request_helper { std::string json, base::OnceCallback callback) { if (!data_decoder_) { - VLOG(1) << "Creating DataDecoder for APIRequestHelper"; + VLOG(0) << "Creating DataDecoder for APIRequestHelper"; data_decoder_ = std::make_unique(); } @@ -358,11 +358,11 @@ namespace api_request_helper { void APIRequestHelper::URLLoaderHandler::OnDataReceived( std::string_view string_piece, base::OnceClosure resume) { - DVLOG(2) << "[[" << __func__ << "]]" << " Chunk received"; + DVLOG(0) << "[[" << __func__ << "]]" << " Chunk received"; if (is_sse_) { ParseSSE(string_piece); } else { - DVLOG(4) << "Chunk content: \n" << string_piece; + DVLOG(0) << "Chunk content: \n" << string_piece; data_received_callback_.Run(base::Value(string_piece)); } std::move(resume).Run(); @@ -370,7 +370,7 @@ namespace api_request_helper { void APIRequestHelper::URLLoaderHandler::OnComplete(bool success) { DCHECK(result_callback_); - VLOG(1) << "[[" << __func__ << "]]" << " Response completed\n"; + VLOG(0) << "[[" << __func__ << "]]" << " Response completed\n"; request_is_finished_ = true; @@ -386,7 +386,7 @@ namespace api_request_helper { void APIRequestHelper::URLLoaderHandler::OnResponse( ResponseConversionCallback conversion_callback, const std::unique_ptr response_body) { - VLOG(1) << "[[" << __func__ << "]]" << " Response received\n"; + VLOG(0) << "[[" << __func__ << "]]" << " Response received\n"; DCHECK(result_callback_); DCHECK_EQ(current_decoding_operation_count_, 0); @@ -417,7 +417,7 @@ namespace api_request_helper { APIRequestResult result, ValueOrError result_value) { if (!result_value.has_value()) { - VLOG(1) << "Response validation error:" << result_value.error(); + VLOG(0) << "Response validation error:" << result_value.error(); if (result_value.error().starts_with("trailing comma")) { DEBUG_ALIAS_FOR_GURL(url_alias, result.final_url()); DEBUG_ALIAS_FOR_CSTR(result_str, result_value.error().c_str(), 1024); @@ -427,7 +427,7 @@ namespace api_request_helper { return; } if (!result_value.value().is_dict() && !result_value.value().is_list()) { - VLOG(1) << "Response validation error: Invalid top-level type"; + VLOG(0) << "Response validation error: Invalid top-level type"; std::move(result_callback_).Run(std::move(result)); return; } @@ -444,7 +444,7 @@ namespace api_request_helper { if (request_is_finished_ && decoding_is_complete) { std::move(result_callback_).Run(ToAPIRequestResult(std::move(url_loader_))); } else if (decoding_is_complete) { - VLOG(3) << "Did not run URLLoaderHandler completion handler, still have " + VLOG(0) << "Did not run URLLoaderHandler completion handler, still have " << current_decoding_operation_count_ << " decoding operations in progress, waiting for them to" << " complete..."; @@ -462,16 +462,16 @@ namespace api_request_helper { // Remove SSE events that don't look like JSON - could be string or [DONE] // message. - // TODO(@nullhook): Parse both JSON and string values. The below currently + // TODO: Parse both JSON and string values. The below currently // only identifies JSON values. static constexpr char kDataPrefix[] = "data: {"; std::erase_if(stream_data, [](std::string_view item) { - DVLOG(3) << "Received chunk: " << item; + DVLOG(0) << "Received chunk: " << item; if (!base::StartsWith(item, kDataPrefix)) { // This is useful to log in case an API starts // coming back with unknown data type in some // scenarios. - VLOG(1) << "Data did not start with SSE prefix"; + VLOG(0) << "Data did not start with SSE prefix"; return true; } return false; @@ -498,7 +498,7 @@ namespace api_request_helper { handler->MaybeSendResult(); }; - DVLOG(2) << "Going to call ParseJsonImpl"; + DVLOG(0) << "Going to call ParseJsonImpl"; ParseJsonImpl(std::string(json), base::BindOnce(std::move(on_json_parsed), weak_ptr_factory_.GetWeakPtr())); diff --git a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc index fc31beac1ed9d3..66d0d47e81b439 100644 --- a/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc +++ b/chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.cc @@ -1,12 +1,10 @@ #include "completion_api_client.h" - -#include - #include #include #include +#include #include "base/containers/flat_set.h" #include "base/functional/bind.h" #include "base/functional/callback_helpers.h" @@ -45,11 +43,6 @@ namespace { std::string CreateJSONRequestBody(const std::vector& prompt) { base::Value::Dict dict; - - // base::Value::List stop_sequences; - // stop_sequences.Append("\nUser:"); - // stop_sequences.Append("\nAssistant:"); - dict.Set("stream", true); dict.Set("max_tokens", 1280); dict.Set("top_p", 0.7); @@ -70,7 +63,6 @@ namespace { base::JSONWriter::Write(dict, &json); return json; } - } // namespace CompletionApiClient::CompletionApiClient( @@ -140,46 +132,32 @@ void CompletionApiClient::OnQueryCompleted( GenerationCompletedCallback callback, APIRequestResult result) { const bool success = result.Is2XXResponseCode(); + // Handle successful request if (success) { - // std::string completion = ""; - // // We're checking for a value body in case for non-streaming API - // results. if (result.value_body().is_dict()) { - // const std::string* value = - // result.value_body().GetDict().FindString("choices"); - // if (value) { - // // Trimming necessary for Llama 2 which prepends - // responses with a " ". completion = - // base::TrimWhitespaceASCII(*value, base::TRIM_ALL); - // } - // } - // std::string entire_message; - // for (const auto& s : entire_completion_result) { - // entire_message += s; - // } - // // DVLOG(0) << std::endl << std::endl << entire_message << - // std::endl << std::endl; - + // todo: to pass entire_completion_result to callback so that chat_page_handler can cache it for subsequent context for chat entire_completion_result.clear(); std::move(callback).Run(base::ok("")); return; } - // Handle error chat::mojom::APIErrorType error; - LOG(INFO) << "Error response_code: " << result.response_code(); - LOG(INFO) << "Error error_code: " << result.error_code(); + DVLOG(0) << "Error response_code: " << result.response_code(); + DVLOG(0) << "Error error_code: " << result.error_code(); + + // todo: to get error message from Chat API response and pass to callback if (result.value_body().is_dict()) { const std::string* value = result.value_body().GetDict().FindString("message"); if (value) { // Trimming necessary for Llama 2 which prepends responses with a " ". - auto completion = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); - LOG(INFO) << "Error message: " << completion; + auto error_message = base::TrimWhitespaceASCII(*value, base::TRIM_ALL); + DVLOG(0) << "Error message: " << error_message; } } + if (net::HTTP_TOO_MANY_REQUESTS == result.response_code()) { error = chat::mojom::APIErrorType::RateLimitReached; } else if (net::HTTP_REQUEST_ENTITY_TOO_LARGE == result.response_code()) { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc deleted file mode 100644 index 19a6d757ae24e8..00000000000000 --- a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.cc +++ /dev/null @@ -1,185 +0,0 @@ -#include "chat_context_observer.h" - -#include -#include -#include -#include - -#include "base/containers/fixed_flat_set.h" -#include "base/functional/bind.h" -#include "base/memory/weak_ptr.h" -#include "base/ranges/algorithm.h" -#include "base/strings/string_util.h" -#include "chrome/grit/generated_resources.h" -#include "chrome/renderer/chat/page_content_extractor.h" -#include "components/strings/grit/components_strings.h" -#include "content/public/browser/browser_accessibility_state.h" -#include "content/public/browser/browser_context.h" -#include "content/public/browser/browser_thread.h" -#include "content/public/browser/navigation_details.h" -#include "content/public/browser/navigation_entry.h" -#include "content/public/browser/scoped_accessibility_mode.h" -#include "content/public/browser/storage_partition.h" -#include "content/public/browser/web_contents.h" -#include "content/public/browser/web_ui.h" -#include "services/network/public/cpp/shared_url_loader_factory.h" -#include "services/service_manager/public/cpp/interface_provider.h" -#include "ui/accessibility/ax_mode.h" -#include "ui/accessibility/ax_updates_and_events.h" -#include "ui/base/l10n/l10n_util.h" -#include "url/gurl.h" -#include "page_content_extractor_helper.h" - -namespace ai_chat { - -// static -void ChatContextObserver::BindPageContentExtractorHost( - content::RenderFrameHost* rfh, - mojo::PendingAssociatedReceiver - receiver) { - CHECK(rfh); - if (!rfh->IsInPrimaryMainFrame()) { - DVLOG(0) << "Render frame is not in primary main frame. Not binding to " - "extractor host."; - return; - } - auto* sender = content::WebContents::FromRenderFrameHost(rfh); - if (!sender) { - DVLOG(0) << "Cannot bind extractor host, no valid WebContents"; - return; - } - auto* chat_context_observer = ChatContextObserver::FromWebContents(sender); - if (!chat_context_observer) { - DVLOG(0) << "Cannot bind extractor host, no ChatContextObserver - " - << sender->GetVisibleURL(); - return; - } - DVLOG(0) << "Binding extractor host to ChatContextObserver"; - chat_context_observer->BindPageContentExtractorReceiver(std::move(receiver)); -} - -void ChatContextObserver::BindPageContentExtractorReceiver( - mojo::PendingAssociatedReceiver - receiver) { - page_content_extractor_receiver_.reset(); - page_content_extractor_receiver_.Bind(std::move(receiver)); -} - -ChatContextObserver::ChatContextObserver(content::WebContents* web_contents) - : content::WebContentsObserver(web_contents), - content::WebContentsUserData(*web_contents), - page_content_extractor_helper_delegate_( - std::make_unique(web_contents)) - { - previous_page_title_ = web_contents->GetTitle(); -} - -ChatContextObserver::~ChatContextObserver() = default; - -void ChatContextObserver::SetPendingGetContentCallback( - ExtractPageContentCallback callback) { - if (pending_extract_page_content_callback_) { - std::move(pending_extract_page_content_callback_).Run(""); - } - pending_extract_page_content_callback_ = std::move(callback); -} - -GURL ChatContextObserver::GetPageURL() const { - return web_contents()->GetLastCommittedURL(); -} - -void ChatContextObserver::GetPageContent(ExtractPageContentCallback callback) { - page_content_extractor_helper_delegate_->ExtractPageContent( - std::move(callback)); -} - -std::u16string ChatContextObserver::GetPageTitle() const { - return web_contents()->GetTitle(); -} - -// begin content::WebContentsObserver -void ChatContextObserver::WebContentsDestroyed() { - inner_web_contents_ = nullptr; -} - -void ChatContextObserver::NavigationEntryCommitted( - const content::LoadCommittedDetails& load_details) { - if (!load_details.is_main_frame) { - return; - } - // UniqueID will provide a consistent value for the entry when navigating - // through history, allowing us to re-join conversations and navigations. - int pending_navigation_id = load_details.entry->GetUniqueID(); - pending_navigation_id_ = pending_navigation_id; - DVLOG(1) << __func__ << " id: " << pending_navigation_id_ - << "\n url: " << load_details.entry->GetVirtualURL() - << "\n current page title: " << GetPageTitle() - << "\n previous page title: " << previous_page_title_ - << "\n same document? " << load_details.is_same_document; - - // Allow same-document navigation, as content often changes as a result - // of framgment / pushState / replaceState navigations. - // Content won't be retrieved immediately and we don't have a similar - // "DOM Content Loaded" event, so let's wait for something else such as - // page title changing before committing to starting a new conversation - // and treating it as a "fresh page". - is_same_document_navigation_ = load_details.is_same_document; - // Experimentally only call |OnNewPage| for same-page navigations _if_ - // it results in a page title change (see |TtileWasSet|). Title detection - // also done within the navigation entry so that back/forward navigations - // are handled correctly. - - // Page loaded is only considered changing when full document changes - if (!is_same_document_navigation_) { - is_page_loaded_ = false; - } - if (!is_same_document_navigation_ || previous_page_title_ != GetPageTitle()) { - // OnNewPage(pending_navigation_id_); - } - previous_page_title_ = GetPageTitle(); -} - -void ChatContextObserver::TitleWasSet(content::NavigationEntry* entry) { - DVLOG(1) << __func__ << ": id=" << entry->GetUniqueID() - << " title=" << entry->GetTitle(); - MaybeSameDocumentIsNewPage(); - previous_page_title_ = GetPageTitle(); -} - -void ChatContextObserver::DidFinishLoad( - content::RenderFrameHost* render_frame_host, - const GURL& validated_url) { - DVLOG(1) << __func__ << ": " << validated_url.spec(); - if (validated_url == GetPageURL()) { - is_page_loaded_ = true; - if (pending_extract_page_content_callback_) { - GetPageContent(std::move(pending_extract_page_content_callback_)); - } - } -} -// end content::WebContentsObserver - -void ChatContextObserver::MaybeSameDocumentIsNewPage() { - if (is_same_document_navigation_) { - DVLOG(2) << "Same document navigation detected new \"page\" - calling " - "OnNewPage()"; - // Cancel knowledge that the current navigation should be associated - // with any conversation that's associated with the previous navigation. - // Tell any conversation that it shouldn't be associated with this - // content anymore, as we've moved on. - // OnNewPage(pending_navigation_id_); - // Don't respond to further TitleWasSet - is_same_document_navigation_ = false; - } -} - -// mojom::PageContentExtractorHost -void ChatContextObserver::OnInterceptedPageContentChanged() { - // Maybe mark that the page changed, if we didn't detect it already via title - // change after a same-page navigation. This is the main benefit of this - // function. - MaybeSameDocumentIsNewPage(); -} - -WEB_CONTENTS_USER_DATA_KEY_IMPL(ChatContextObserver); -} // namespace ai_chat diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h b/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h deleted file mode 100644 index 286fa2ef063d45..00000000000000 --- a/chrome/browser/ui/webui/side_panel/chat/chat_context_observer.h +++ /dev/null @@ -1,107 +0,0 @@ -#ifndef CHROMIUM_CHAT_CONTEXT_OBSERVER_H -#define CHROMIUM_CHAT_CONTEXT_OBSERVER_H - -#include -#include -#include -#include - -#include "base/functional/callback_forward.h" -#include "base/memory/raw_ptr.h" -#include "chrome/common/chat/page_content_extractor.mojom.h" -#include "content/public/browser/navigation_handle.h" -#include "content/public/browser/render_frame_host.h" -#include "content/public/browser/web_contents.h" -#include "content/public/browser/web_contents_observer.h" -#include "content/public/browser/web_contents_user_data.h" -#include "mojo/public/cpp/bindings/associated_receiver.h" -#include "mojo/public/cpp/bindings/pending_associated_receiver.h" - -namespace ai_chat { -// Monitors changes in web content and updates the AI chat with the latest -// information. -class ChatContextObserver - : public content::WebContentsObserver, - public content::WebContentsUserData, - public chat::mojom::PageContentExtractorHost { - public: - - using ExtractPageContentCallback = - base::OnceCallback; - - static void BindPageContentExtractorHost( - content::RenderFrameHost* rfh, - mojo::PendingAssociatedReceiver - receiver); - - class PageContentExtractorHelperDelegate { - public: - virtual ~PageContentExtractorHelperDelegate() = default; - virtual void ExtractPageContent(ExtractPageContentCallback callback) = 0; - }; - - ChatContextObserver(const ChatContextObserver&) = delete; - - ChatContextObserver& operator=(const ChatContextObserver&) = delete; - - ~ChatContextObserver() override; - - // chat::mojom::PageContentExtractorHost - void OnInterceptedPageContentChanged() override; - - void GetPageContent(ExtractPageContentCallback callback); - - private: - ChatContextObserver(content::WebContents* web_contents); - - friend class content::WebContentsUserData; - - // begin content::WebContentsObserver - void WebContentsDestroyed() override; - - // Called when an event of significance occurs that, if the page is a - // same-document navigation, should result in that previous navigation - // being considered as a new page. - void MaybeSameDocumentIsNewPage(); - - void NavigationEntryCommitted( - const content::LoadCommittedDetails& load_details) override; - - void TitleWasSet(content::NavigationEntry* entry) override; - - void DidFinishLoad(content::RenderFrameHost* render_frame_host, - const GURL& validated_url) override; - // end content::WebContentsObserver - - void BindPageContentExtractorReceiver( - mojo::PendingAssociatedReceiver - receiver); - - void SetPendingGetContentCallback(ExtractPageContentCallback callback); - - GURL GetPageURL() const; - - std::u16string GetPageTitle() const; - - void OnNewPage(int64_t navigation_id); - - int pending_navigation_id_; - bool is_same_document_navigation_ = false; - std::u16string previous_page_title_; - bool is_page_loaded_ = false; - raw_ptr inner_web_contents_ = nullptr; - - std::unique_ptr - page_content_extractor_helper_delegate_; - ExtractPageContentCallback pending_extract_page_content_callback_; - - mojo::AssociatedReceiver - page_content_extractor_receiver_{this}; - - base::WeakPtrFactory weak_ptr_factory_{this}; - - WEB_CONTENTS_USER_DATA_KEY_DECL(); -}; -} // namespace ai_chat - -#endif // CHROMIUM_CHAT_CONTEXT_OBSERVER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 35f08b6d680e94..15dc9df30e9f97 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -74,19 +74,20 @@ void ChatPageHandler::GetSiteInfo(GetSiteInfoCallback callback) { site_info->is_content_usable_in_conversations = false; if (chat_context_web_contents_) { - site_info->title = - base::UTF16ToUTF8(chat_context_web_contents_->GetTitle()); const GURL gurl = chat_context_web_contents_->GetLastCommittedURL(); if (gurl.SchemeIsHTTPOrHTTPS()) { + site_info->title = + base::UTF16ToUTF8(chat_context_web_contents_->GetTitle()); site_info->url = gurl.spec(); site_info->is_content_usable_in_conversations = true; } else { + site_info->title = ""; site_info->url = ""; site_info->is_content_usable_in_conversations = false; } } - std::move(callback).Run(site_info.Clone()); + std::move(callback).Run(site_info.Clone()); } void ChatPageHandler::GetActionList(GetActionListCallback callback) { @@ -175,11 +176,11 @@ void ChatPageHandler::SubmitQueryCompletedCallback( chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; if (result.has_value()) { - LOG(INFO) << __func__ << " success -> " << result.value(); + DVLOG(0) << __func__ << " success -> " << result.value(); response->response_type = chat::mojom::ResponseType::COMPLETED; response->result = result.value(); } else { - LOG(INFO) << __func__ << " error -> " << result.error(); + DVLOG(0) << __func__ << " error -> " << result.error(); response->response_type = chat::mojom::ResponseType::ERROR; response->result = "error_message_here"; // todo: to get error message from api response diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 4cd7b0aa6cbf64..78ace6432dbf81 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -9,7 +9,6 @@ #include "base/memory/raw_ptr.h" #include "base/scoped_observation.h" #include "base/types/expected.h" -#include "chat_context_observer.h" #include "chat_ui.h" #include "chrome/browser/profiles/profile.h" #include "chrome/browser/ui/webui/side_panel/chat/api/completion_api_client.h" @@ -66,12 +65,13 @@ class ChatPageHandler : public chat::mojom::PageHandler { mojo::Receiver receiver_; mojo::Remote page_; - const raw_ptr chat_ui_; + const raw_ptr chat_ui_ = nullptr; raw_ptr owner_web_contents_ = nullptr; raw_ptr chat_context_web_contents_ = nullptr; - const raw_ptr profile_; + const raw_ptr profile_ = nullptr; std::unique_ptr api_client_ = nullptr; - std::unique_ptr page_content_extractor_helper_; + std::unique_ptr page_content_extractor_helper_ = + nullptr; base::WeakPtrFactory weak_ptr_factory_{this}; }; #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc index 672a25da93282b..0a1075d6ff64ad 100644 --- a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc +++ b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.cc @@ -25,70 +25,63 @@ #include "services/service_manager/public/cpp/interface_provider.h" #include "content/public/browser/web_contents.h" - namespace { -constexpr auto kVideoPageContentTypes = - base::MakeFixedFlatSet( - {chat::mojom::PageContentType::VideoTranscriptYouTube, - chat::mojom::PageContentType::VideoTranscriptVTT}); class PageContentExtractorInternal { public: PageContentExtractorInternal() {} - void Start(mojo::Remote content_extractor, - base::OnceCallback callback) { - content_extractor_ = std::move(content_extractor); - if (!content_extractor_) { - DeleteSelf(); - return; - } - - // Ref: - // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks - // Once a `mojo::Remote` is destroyed, it is guaranteed that pending - // callbacks as well as the connection error handler (if registered) won't - // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed that - // no more method calls are dispatched to the implementation and the - // connection error handler (if registered) won't be called. - content_extractor_.set_disconnect_handler(base::BindOnce( - &PageContentExtractorInternal::DeleteSelf, base::Unretained(this))); - content_extractor_->ExtractPageContent( - base::BindOnce(&PageContentExtractorInternal::OnPageContentExtracted, - base::Unretained(this), std::move(callback))); - } - - void OnPageContentExtracted(base::OnceCallback callback, - chat::mojom::PageContentPtr data) { - if (!data) { - DVLOG(0) << __func__ << " no extracted page content."; - SendResultAndDeleteSelf(std::move(callback)); - return; + void Start( + mojo::Remote content_extractor, + base::OnceCallback callback) { + content_extractor_ = std::move(content_extractor); + if (!content_extractor_) { + DeleteSelf(); + return; + } + + // Ref: + // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/mojo/public/cpp/bindings/README.md#a-note-about-endpoint-lifetime-and-callbacks + // Once a `mojo::Remote` is destroyed, it is guaranteed that pending + // callbacks as well as the connection error handler (if registered) won't + // be called. Once a `mojo::Receiver` is destroyed, it is guaranteed + // that no more method calls are dispatched to the implementation and the + // connection error handler (if registered) won't be called. + content_extractor_.set_disconnect_handler(base::BindOnce( + &PageContentExtractorInternal::DeleteSelf, base::Unretained(this))); + content_extractor_->ExtractPageContent( + base::BindOnce(&PageContentExtractorInternal::OnPageContentExtracted, + base::Unretained(this), std::move(callback))); } - DVLOG(1) << "OnTabContentResult: " << data.get(); - const bool is_video = base::Contains(kVideoPageContentTypes, data->type); - DVLOG(1) << "Is video? " << is_video; - - if (!is_video) { - DCHECK(data->content->is_content()); - auto content = data->content->get_content(); - DVLOG(1) << __func__ << ": Got content with char length of " - << content.length(); - SendResultAndDeleteSelf(std::move(callback), content); - return; + void OnPageContentExtracted( + base::OnceCallback callback, + const std::optional& content) { + if (!content.has_value()) { + DVLOG(0) << __func__ << "Extracted content is null."; + SendResultAndDeleteSelf(std::move(callback)); + return; + } + + if (content->empty()) { + DVLOG(0) << __func__ << "Extracted content is empty."; + SendResultAndDeleteSelf(std::move(callback)); + return; + } + + DVLOG(0) << __func__ << "extracted page content: " << content.value(); + SendResultAndDeleteSelf(std::move(callback), content.value()); } - SendResultAndDeleteSelf(std::move(callback)); - } - private: void DeleteSelf() { delete this; } + void SendResultAndDeleteSelf(base::OnceCallback callback, std::string content = "") { std::move(callback).Run(content); delete this; } + mojo::Remote content_extractor_; base::WeakPtrFactory weak_ptr_factory_{this}; }; @@ -101,7 +94,7 @@ PageContentExtractorHelper::PageContentExtractorHelper( PageContentExtractorHelper::~PageContentExtractorHelper() = default; void PageContentExtractorHelper::ExtractPageContent( - base::OnceCallback callback) { + base::OnceCallback callback) { auto* primary_rfh = web_contents_->GetPrimaryMainFrame(); DCHECK(primary_rfh->IsRenderFrameLive()); @@ -112,4 +105,3 @@ void PageContentExtractorHelper::ExtractPageContent( auto* internal_extractor = new PageContentExtractorInternal(); internal_extractor->Start(std::move(extractor), std::move(callback)); } - diff --git a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h index ad82e4527e74c0..522ce7bfd56e90 100644 --- a/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h +++ b/chrome/browser/ui/webui/side_panel/chat/page_content_extractor_helper.h @@ -4,6 +4,7 @@ #include #include "base/functional/callback_forward.h" +#include "chrome/common/chat/page_content_extractor.mojom.h" #include "chrome/renderer/chat/page_content_extractor.h" namespace content { @@ -12,13 +13,13 @@ namespace content { class PageContentExtractorHelper{ public: - explicit PageContentExtractorHelper(content::WebContents* web_contents); ~PageContentExtractorHelper(); PageContentExtractorHelper(const PageContentExtractorHelper&) = delete; PageContentExtractorHelper& operator=(const PageContentExtractorHelper&) = delete; - void ExtractPageContent( base::OnceCallback callback); + void ExtractPageContent( + base::OnceCallback callback); private: raw_ptr web_contents_; diff --git a/chrome/common/chat/BUILD.gn b/chrome/common/chat/BUILD.gn index 272941ec432de9..3fd8db43976b3f 100644 --- a/chrome/common/chat/BUILD.gn +++ b/chrome/common/chat/BUILD.gn @@ -2,5 +2,4 @@ import("//mojo/public/tools/bindings/mojom.gni") mojom("mojo_bindings") { sources = [ "page_content_extractor.mojom" ] - public_deps = [ "//url/mojom:url_mojom_gurl" ] } diff --git a/chrome/common/chat/page_content_extractor.mojom b/chrome/common/chat/page_content_extractor.mojom index 480c7e06bb23e6..189473d9aefb36 100644 --- a/chrome/common/chat/page_content_extractor.mojom +++ b/chrome/common/chat/page_content_extractor.mojom @@ -1,31 +1,5 @@ module chat.mojom; -import "url/mojom/url.mojom"; - -enum PageContentType { - Text, - VideoTranscriptYouTube, - VideoTranscriptVTT -}; - -// It has either text or url -union PageContentData { - string content; - url.mojom.Url content_url; -}; - -struct PageContent { - PageContentData content; - PageContentType type; -}; - interface PageContentExtractor { - ExtractPageContent() => (PageContent? page_content); -}; - -// Enables the renderer to signal content updates to the browser process. -interface PageContentExtractorHost { - // This is just to notify that the content has changed. - // The current page's content will be extracted via PageContentExtractor.ExtractPageContent. - OnInterceptedPageContentChanged(); + ExtractPageContent() => (string? page_content); }; diff --git a/chrome/renderer/chat/page_content_extractor.cc b/chrome/renderer/chat/page_content_extractor.cc index eef73cd394eef5..2a082092ddba73 100644 --- a/chrome/renderer/chat/page_content_extractor.cc +++ b/chrome/renderer/chat/page_content_extractor.cc @@ -136,7 +136,7 @@ base::WeakPtr PageContentExtractor::GetWeakPtr() { void PageContentExtractor::BindReceiver( mojo::PendingReceiver receiver) { - VLOG(1) << "Yep Chat PageContentExtractor handler bound."; + DVLOG(0) << "Yep Chat PageContentExtractor handler bound."; receiver_.reset(); receiver_.Bind(std::move(receiver)); } @@ -221,22 +221,6 @@ void PageContentExtractor::ExtractPageText( void PageContentExtractor::OnPageTextExtracted( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback, const std::optional& content) { - if (!content.has_value()) { - DVLOG(0) << "Extracted content is null."; - std::move(callback).Run({}); - return; - } - - if (content->empty()) { - DVLOG(0) << "Extracted content is empty."; - std::move(callback).Run({}); - return; - } - - // Successful text extraction - auto result = chat::mojom::PageContent::New(); - result->type = std::move(chat::mojom::PageContentType::Text); - result->content = chat::mojom::PageContentData::NewContent(content.value()); - std::move(callback).Run(std::move(result)); + std::move(callback).Run(std::move(content)); } } // namespace ai_chat diff --git a/chrome/renderer/chat/page_content_extractor.h b/chrome/renderer/chat/page_content_extractor.h index 41b7e97bb5d944..8cf3daa71d6daf 100644 --- a/chrome/renderer/chat/page_content_extractor.h +++ b/chrome/renderer/chat/page_content_extractor.h @@ -37,7 +37,7 @@ class PageContentExtractor base::WeakPtr GetWeakPtr(); private: - // PageContentExtractor implementation: + // chat::mojom::PageContentExtractor implementation: void ExtractPageContent( chat::mojom::PageContentExtractor::ExtractPageContentCallback callback) override; From fec3cf0072744a98a271873685404896f6bb53e5 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 26 Nov 2024 17:35:38 +0800 Subject: [PATCH 28/34] Enhance chat actions with specific prompts Implemented different prompts for chat actions: summarize page, explain page, and fact check page. Updated function signatures and button click handlers to utilize the new action-specific prompts. --- .../side_panel/chat/chat_app.html.ts | 5 +++- .../resources/side_panel/chat/chat_app.ts | 16 +++++++++--- .../side_panel/chat/chat_page_handler.cc | 25 ++++++++++++++----- .../webui/side_panel/chat/chat_page_handler.h | 1 + 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 09f68c4a9a2765..9b867b38bf0f93 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -24,7 +24,10 @@ export function getHtml(this: ChatAppElement) {
${ this.siteInfo_.isContentUsableInConversations && this.conversations_ && this.conversations_.length == 0 ? this.actionList_.map((item,_) => html` - + ${item.label} `) : html``} diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 77abc528e7ed93..f30c0269a35726 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -82,12 +82,20 @@ export class ChatAppElement extends CrLitElement { } } - protected onSubmitAction_(e: Event) { - e.stopPropagation(); + protected onSubmitAction_(actionType: ActionType) { + // todo: to handle type and display proper UI if (this.conversations_ != null) { - this.conversations_.push({query: "Summarise this page", response: ""}); + if (actionType == ActionType.SUMMARIZE_PAGE) { + this.conversations_.push({query: "Summarise this page.", response: ""}); + } else if (actionType == ActionType.EXPLAIN) { + this.conversations_.push({query: "Explain this page in simple language.", response: ""}); + } else if (actionType == ActionType.FACT_CHECK) { + this.conversations_.push({query: "Fact check this page.", response: ""}); + } else { + this.conversations_.push({query: "Other actions. To be fixed.", response: ""}); + } } - this.chatApiProxy_.submitAction(ActionType.SUMMARIZE_PAGE); + this.chatApiProxy_.submitAction(actionType); } protected onTextareaValueChanged_(e: CustomEvent<{value: string}>) { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 15dc9df30e9f97..aa9b8da04a516b 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -128,20 +128,34 @@ base::WeakPtr ChatPageHandler::GetWeakPtr() { } void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { + // todo: to put prompt in resource file + std::string summarize_prompt = "Provide a brief summary of the key takeaways for the following:"; + std::string explain_prompt = "Explain the following in simple language:"; + std::string fact_check_prompt = "Fact check the following:"; if (page_.is_bound()) { DVLOG(0) << action_type; if (action_type == chat::mojom::ActionType::SUMMARIZE_PAGE) { - DVLOG(0) << "ActionType: Summarize_page"; page_content_extractor_helper_->ExtractPageContent( base::BindOnce(&ChatPageHandler::OnPageContentExtracted, - base::Unretained(this), action_type)); + base::Unretained(this), action_type, summarize_prompt)); - } else { + } else if (action_type == chat::mojom::ActionType::EXPLAIN) { + page_content_extractor_helper_->ExtractPageContent( + base::BindOnce(&ChatPageHandler::OnPageContentExtracted, + base::Unretained(this), action_type, explain_prompt)); + + } else if (action_type == chat::mojom::ActionType::FACT_CHECK) { + page_content_extractor_helper_->ExtractPageContent( + base::BindOnce(&ChatPageHandler::OnPageContentExtracted, + base::Unretained(this), action_type, fact_check_prompt)); + } + else { // todo: to implement for other action types later chat::mojom::ActionResponsePtr response = chat::mojom::ActionResponse::New(); response->action_type = action_type; + response->response_type = chat::mojom::ResponseType::DELTA; response->result = "MOCK Result"; page_->OnSubmitActionResponse(response.Clone()); } @@ -150,11 +164,10 @@ void ChatPageHandler::SubmitAction(chat::mojom::ActionType action_type) { void ChatPageHandler::OnPageContentExtracted( chat::mojom::ActionType action_type, + const std::string& prompt, std::string content) { - // todo: to put prompt in resource file api_client_->QueryPrompt( - "Provide a brief summary of the key takeaways for the following:" + - content, + prompt + content, base::BindOnce(&ChatPageHandler::SubmitQueryCompletedCallback, base::Unretained(this), action_type), base::BindRepeating(&ChatPageHandler::SubmitQueryCallback, diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 78ace6432dbf81..4f27842633a4ff 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -61,6 +61,7 @@ class ChatPageHandler : public chat::mojom::PageHandler { private: void OnPageContentExtracted(chat::mojom::ActionType action_type, + const std::string& prompt, std::string content); mojo::Receiver receiver_; From 49c6c0c281b6c80b07e448d3d6d4dc53928eabb8 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 26 Nov 2024 17:49:30 +0800 Subject: [PATCH 29/34] Limit chat content length to 32768 tokens. Previously, the full content was passed to the query without length restriction, which could cause issues. Now, the content is truncated to a maximum of 32768 tokens to ensure it adheres to the defined limit. --- .../ui/webui/side_panel/chat/chat_page_handler.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index aa9b8da04a516b..25e2d03d8bdebd 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -166,8 +166,16 @@ void ChatPageHandler::OnPageContentExtracted( chat::mojom::ActionType action_type, const std::string& prompt, std::string content) { + std::string max_content = content; + + //Note: This max tokens seems message + completion. + const size_t max_tokens = 32768; + if (content.length() > max_tokens) { + max_content = content.substr(0, max_tokens); + } + api_client_->QueryPrompt( - prompt + content, + prompt + max_content, base::BindOnce(&ChatPageHandler::SubmitQueryCompletedCallback, base::Unretained(this), action_type), base::BindRepeating(&ChatPageHandler::SubmitQueryCallback, From 73f43bbf31d2e6f6048bfe4c12231e609f6ab68b Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Thu, 28 Nov 2024 18:47:55 +0800 Subject: [PATCH 30/34] Add dynamic color change handling to ChatUI - WIP --- .../resources/side_panel/chat/BUILD.gn | 46 +++++++++++++------ .../resources/side_panel/chat/chat.css | 5 -- .../resources/side_panel/chat/chat.html | 16 +++++-- .../side_panel/chat/chat_api_proxy.ts | 26 ++++++++--- .../resources/side_panel/chat/chat_app.css | 7 +-- .../side_panel/chat/chat_app.html.ts | 40 ++++++++-------- .../resources/side_panel/chat/chat_app.ts | 32 +++++++------ .../side_panel/chat/chat_page_handler.cc | 6 +-- .../webui/side_panel/chat/chat_page_handler.h | 12 ++++- .../ui/webui/side_panel/chat/chat_ui.cc | 20 +++++++- .../ui/webui/side_panel/chat/chat_ui.h | 17 +++++-- 11 files changed, 148 insertions(+), 79 deletions(-) delete mode 100644 chrome/browser/resources/side_panel/chat/chat.css diff --git a/chrome/browser/resources/side_panel/chat/BUILD.gn b/chrome/browser/resources/side_panel/chat/BUILD.gn index 11c6929579e30e..86eba351733c3e 100644 --- a/chrome/browser/resources/side_panel/chat/BUILD.gn +++ b/chrome/browser/resources/side_panel/chat/BUILD.gn @@ -1,22 +1,38 @@ import("//ui/webui/resources/tools/build_webui.gni") +assert(!is_android) + build_webui("build") { - grd_prefix = "side_panel_chat" + grd_prefix = "side_panel_chat" + + static_files = [ "chat.html" ] + + non_web_component_files = [ + "chat_api_proxy.ts", + "chat_app.ts", + "chat_app.html.ts", + ] + css_files = [ "chat_app.css" ] - static_files = [ "chat.html", "chat.css" ] + # Enable the proper webui_context_type depending on whether implementing + # a chrome:// or chrome-untrusted:// page. + webui_context_type = "trusted" - non_web_component_files = [ "chat_api_proxy.ts", "chat_app.ts", "chat_app.html.ts" ] - css_files = [ "chat_app.css" ] + mojo_files_deps = [ + "//chrome/browser/ui/webui/side_panel/chat:mojo_bindings_ts__generator", + ] + mojo_files = [ + "$root_gen_dir/chrome/browser/ui/webui/side_panel/chat/chat.mojom-webui.ts", + ] - # Enable the proper webui_context_type depending on whether implementing - # a chrome:// or chrome-untrusted:// page. - webui_context_type = "trusted" - ts_deps = [ - "//third_party/lit/v3_0:build_ts", - "//ui/webui/resources/js:build_ts", - "//ui/webui/resources/mojo:build_ts", - ] + ts_composite = true - mojo_files_deps = [ "//chrome/browser/ui/webui/side_panel/chat:mojo_bindings_ts__generator" ] - mojo_files = [ "$root_gen_dir/chrome/browser/ui/webui/side_panel/chat/chat.mojom-webui.ts",] -} \ No newline at end of file + ts_deps = [ + "../shared:build_ts", + "//third_party/lit/v3_0:build_ts", + "//ui/webui/resources/cr_components/color_change_listener:build_ts", + "//ui/webui/resources/cr_elements:build_ts", + "//ui/webui/resources/js:build_ts", + "//ui/webui/resources/mojo:build_ts", + ] +} diff --git a/chrome/browser/resources/side_panel/chat/chat.css b/chrome/browser/resources/side_panel/chat/chat.css deleted file mode 100644 index 2d6b08d4b48301..00000000000000 --- a/chrome/browser/resources/side_panel/chat/chat.css +++ /dev/null @@ -1,5 +0,0 @@ - html, - body { - background-color: hsla(0, 0%, 0%, 0); - margin: 0; - } \ No newline at end of file diff --git a/chrome/browser/resources/side_panel/chat/chat.html b/chrome/browser/resources/side_panel/chat/chat.html index ba033d5c1f9d82..c837c219545a73 100644 --- a/chrome/browser/resources/side_panel/chat/chat.html +++ b/chrome/browser/resources/side_panel/chat/chat.html @@ -1,11 +1,17 @@ - - $i18n{title} - - - + + $i18n{title} + + + diff --git a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts index decf77ad1c60ac..1388c70b9ea5bc 100644 --- a/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts +++ b/chrome/browser/resources/side_panel/chat/chat_api_proxy.ts @@ -1,17 +1,29 @@ -import {PageCallbackRouter, PageHandlerFactory, PageHandlerRemote, SiteInfo, - ActionItem, ActionType} from "./chat.mojom-webui.js"; +import { + ActionItem, + ActionType, + PageCallbackRouter, + PageHandlerFactory, + PageHandlerRemote, + SiteInfo +} from "./chat.mojom-webui.js"; export interface ChatApiProxy { getActionList(): Promise<{ actionList: ActionItem[] }>; + submitAction(actionType: ActionType): void; + submitQuery(actionType: ActionType, query: string): void; - getSiteInfo(): Promise<{siteInfo: SiteInfo}> + + getSiteInfo(): Promise<{ siteInfo: SiteInfo }> + showUI(): void; + closeUI(): void; + getCallbackRouter(): PageCallbackRouter; } -let instance: ChatApiProxy|null = null; +let instance: ChatApiProxy | null = null; export class ChatApiProxyImpl implements ChatApiProxy { private readonly callbackRouter: PageCallbackRouter = new PageCallbackRouter(); @@ -35,7 +47,7 @@ export class ChatApiProxyImpl implements ChatApiProxy { } getActionList(): Promise<{ actionList: ActionItem[] }> { - return this.handler.getActionList() ; + return this.handler.getActionList(); } submitAction(actionType: ActionType) { @@ -47,14 +59,14 @@ export class ChatApiProxyImpl implements ChatApiProxy { } getSiteInfo() { - return this.handler.getSiteInfo(); + return this.handler.getSiteInfo(); } showUI() { this.handler.showUI(); } - closeUI(){ + closeUI() { this.handler.closeUI(); } diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index 932a4828fddbfa..ba7d1f6a9c9f17 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -1,4 +1,3 @@ - /* #css_wrapper_metadata_start * #type=style-lit * #import=../chat_app.css.js @@ -6,6 +5,7 @@ * #css_wrapper_metadata_end */ @import url('https://fonts.googleapis.com/css2?family=Poppins:wght@400;500;600&display=swap'); + * { margin: 0; padding: 0; @@ -42,6 +42,7 @@ cr-button { .no-error { --cr-input-error-display: none; } + #container { display: flex; flex-direction: column; @@ -147,7 +148,7 @@ cr-button { display: flex; } -.typing-textarea{ +.typing-textarea { resize: none; height: 55px; width: 100%; @@ -208,7 +209,7 @@ span.material-symbols-rounded { visibility: hidden; } -.typing-textarea textarea:valid~span { +.typing-textarea textarea:valid ~ span { visibility: visible; } diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 9b867b38bf0f93..2ff0a6314ce7d8 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -6,31 +6,33 @@ export function getHtml(this: ChatAppElement) {
${ - this.conversations_.map((conversation,_) => { - return conversation.query.length > 0 - ? html` -

${conversation.query}

- ${conversation.response.length > 0 ? html`

${conversation.response}

` : html``}` - : - html`${conversation.response.length > 0 ? html`

${conversation.response}

` : html``}` - - }) + this.conversations_.map((conversation, _) => { + return conversation.query.length > 0 + ? html` +

${conversation.query}

+ ${conversation.response.length > 0 ? html`

+ ${conversation.response}

` : html``}` + : + html`${conversation.response.length > 0 ? html`

+ ${conversation.response}

` : html``}` + + }) }

- ${this.completionResult_} + ${this.completionResult_}

- ${ this.siteInfo_.isContentUsableInConversations && this.conversations_ && this.conversations_.length == 0 ? - this.actionList_.map((item,_) => html` - - ${item.label} - - `) : html``} + ${this.siteInfo_.isContentUsableInConversations && this.conversations_ && this.conversations_.length == 0 ? + this.actionList_.map((item, _) => html` + + ${item.label} + + `) : html``}
diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index f30c0269a35726..e2679c0313aacf 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -5,13 +5,14 @@ import '//resources/cr_elements/cr_icon_button/cr_icon_button.js'; import '//resources/cr_elements/cr_input/cr_input.js'; import '//resources/cr_elements/cr_textarea/cr_textarea.js'; +// import {ColorChangeUpdater} from 'chrome://resources/cr_components/color_change_listener/colors_css_updater.js'; import {loadTimeData} from 'chrome://resources/js/load_time_data.js'; import {CrLitElement} from '//resources/lit/v3_0/lit.rollup.js'; import {getCss} from './chat_app.css.js'; import {getHtml} from './chat_app.html.js'; import type {ChatApiProxy} from "./chat_api_proxy.js"; import {ChatApiProxyImpl} from "./chat_api_proxy.js"; -import {SiteInfo, ActionItem, ActionType, ActionResponse, ResponseType} from "./chat.mojom-webui.js"; +import {ActionItem, ActionResponse, ActionType, ResponseType, SiteInfo} from "./chat.mojom-webui.js"; type conversationRecord = { query: string, @@ -24,22 +25,23 @@ export class ChatAppElement extends CrLitElement { protected actionList_: ActionItem[] = []; protected conversations_: conversationRecord[] = []; protected askAnythingLabel_ = loadTimeData.getString('askAnything'); - protected siteInfo_ : SiteInfo = { + protected siteInfo_: SiteInfo = { url: "", title: "", isContentUsableInConversations: false, }; - protected submitResponse_ : ActionResponse = { + protected submitResponse_: ActionResponse = { actionType: ActionType.NONE, responseType: ResponseType.NONE, result: "", }; - protected completionResult_ : string = ""; + protected completionResult_: string = ""; protected query_?: string; protected submittedQuery_?: string; constructor() { super(); + // ColorChangeUpdater.forDocument().start(); } static get is() { @@ -50,6 +52,10 @@ export class ChatAppElement extends CrLitElement { return getCss(); } + override render() { + return getHtml.bind(this)(); + } + static override get properties() { return { siteInfo_: {type: Object}, @@ -65,7 +71,7 @@ export class ChatAppElement extends CrLitElement { private async updateSiteInfo(siteInfo: SiteInfo) { this.siteInfo_ = siteInfo; if (this.siteInfo_.isContentUsableInConversations) { - const {actionList} = await this.chatApiProxy_.getActionList(); + const {actionList} = await this.chatApiProxy_.getActionList(); this.actionList_ = actionList; } this.updateComplete; @@ -78,7 +84,7 @@ export class ChatAppElement extends CrLitElement { } else if (response.responseType == ResponseType.COMPLETED) { this.completionResult_ += "\n"; } else if (response.responseType == ResponseType.ERROR) { - this.completionResult_ += "\n" ; + this.completionResult_ += "\n"; } } @@ -98,7 +104,7 @@ export class ChatAppElement extends CrLitElement { this.chatApiProxy_.submitAction(actionType); } - protected onTextareaValueChanged_(e: CustomEvent<{value: string}>) { + protected onTextareaValueChanged_(e: CustomEvent<{ value: string }>) { this.query_ = e.detail.value; } @@ -120,7 +126,7 @@ export class ChatAppElement extends CrLitElement { this.submittedQuery_ = this.query_; this.conversations_.push({query: this.query_ ?? "", response: ""}); this.query_ = ""; - this.chatApiProxy_.submitQuery(ActionType.QUERY, this.submittedQuery_ ?? "" ); + this.chatApiProxy_.submitQuery(ActionType.QUERY, this.submittedQuery_ ?? ""); } override connectedCallback() { @@ -130,15 +136,14 @@ export class ChatAppElement extends CrLitElement { this.chatApiProxy_.showUI(); const {siteInfo} = await this.chatApiProxy_.getSiteInfo(); await this.updateSiteInfo(siteInfo); - this.updateComplete; }, 0); this.listenerIds_.push( this.chatApiProxy_.getCallbackRouter().onSiteInfoChanged.addListener( (siteInfo: SiteInfo) => this.updateSiteInfo(siteInfo)), - this.chatApiProxy_.getCallbackRouter().onSubmitActionResponse.addListener( - (response: ActionResponse) => this.updateSubmitResponse(response)) - ); + this.chatApiProxy_.getCallbackRouter().onSubmitActionResponse.addListener( + (response: ActionResponse) => this.updateSubmitResponse(response)) + ); } override disconnectedCallback() { @@ -148,9 +153,6 @@ export class ChatAppElement extends CrLitElement { id => this.chatApiProxy_.getCallbackRouter().removeListener(id)); } - override render() { - return getHtml.bind(this)(); - } } declare global { diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc index 25e2d03d8bdebd..75c66ba2b57bd6 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.cc @@ -47,9 +47,9 @@ ChatPageHandler::ChatPageHandler( ChatPageHandler::~ChatPageHandler() = default; void ChatPageHandler::ShowUI() { - auto embedder = chat_ui_->embedder(); - if (embedder) { - embedder->ShowUI(); + auto embedder = chat_ui_->embedder(); + if (embedder) { + embedder->ShowUI(); } } diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h index 4f27842633a4ff..404cae69b24cb2 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_page_handler.h @@ -24,6 +24,7 @@ namespace content { class WebContents; + class WebUI; } // namespace content @@ -37,22 +38,30 @@ class ChatPageHandler : public chat::mojom::PageHandler { content::WebContents* chat_context_web_contents); ChatPageHandler(const ChatPageHandler&) = delete; + ChatPageHandler& operator=(const ChatPageHandler&) = delete; ~ChatPageHandler() override; void GetSiteInfo(GetSiteInfoCallback callback) override; + void GetActionList(GetActionListCallback callback) override; + void SubmitAction(chat::mojom::ActionType action_type) override; + void SubmitQuery(chat::mojom::ActionType action_type, const std::string& query) override; + void ShowUI() override; + void CloseUI() override; - void SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents); + void SetSiteInfo(chat::mojom::SiteInfoPtr site_info, + content::WebContents* contents); void SubmitQueryCallback(chat::mojom::ActionType action_type, std::string completion); + void SubmitQueryCompletedCallback( chat::mojom::ActionType action_type, base::expected result); @@ -75,4 +84,5 @@ class ChatPageHandler : public chat::mojom::PageHandler { nullptr; base::WeakPtrFactory weak_ptr_factory_{this}; }; + #endif //CHROMIUM_CHAT_PAGE_HANDLER_H diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc index 7a6d58fd5ffe00..71e781d0a86a95 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.cc @@ -1,3 +1,8 @@ + +#ifdef UNSAFE_BUFFERS_BUILD +#pragma allow_unsafe_buffers +#endif + #include "chat_ui.h" #include "chat_page_handler.h" @@ -12,6 +17,8 @@ #include "chrome/grit/generated_resources.h" #include "chrome/grit/side_panel_chat_resources.h" #include "chrome/grit/side_panel_chat_resources_map.h" +#include "chrome/grit/side_panel_shared_resources.h" +#include "chrome/grit/side_panel_shared_resources_map.h" #include "content/public/browser/browser_context.h" #include "content/public/browser/web_contents.h" #include "content/public/browser/web_ui.h" @@ -19,10 +26,12 @@ #include "content/public/browser/web_ui_data_source.h" #include "content/public/browser/webui_config.h" #include "content/public/common/url_constants.h" +#include "ui/base/ui_base_features.h" +#include "ui/base/webui/web_ui_util.h" +#include "ui/views/style/platform_style.h" +#include "ui/webui/color_change_listener/color_change_handler.h" #include "ui/webui/mojo_web_ui_controller.h" -#pragma allow_unsafe_buffers - namespace { #if BUILDFLAG(IS_ANDROID) @@ -83,6 +92,13 @@ ChatUI::~ChatUI() = default; WEB_UI_CONTROLLER_TYPE_IMPL(ChatUI) +void ChatUI::BindInterface( + mojo::PendingReceiver + pending_receiver) { + color_provider_handler_ = std::make_unique( + web_ui()->GetWebContents(), std::move(pending_receiver)); +} + void ChatUI::BindInterface( mojo::PendingReceiver receiver) { page_factory_receiver_.reset(); diff --git a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h index ff26d1adf6d0d5..e5a4fec4d1b954 100644 --- a/chrome/browser/ui/webui/side_panel/chat/chat_ui.h +++ b/chrome/browser/ui/webui/side_panel/chat/chat_ui.h @@ -12,6 +12,13 @@ #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" #include "ui/webui/mojo_web_ui_controller.h" +#include "ui/webui/resources/cr_components/color_change_listener/color_change_listener.mojom.h" + +class ChatPageHandler; + +namespace ui { +class ColorChangeHandler; +} class ChatUI; @@ -22,8 +29,6 @@ class ChatUIConfig : public DefaultTopChromeWebUIConfig { chrome::kChromeUIChatHost) {} }; -class ChatPageHandler; - class ChatUI : public TopChromeWebUIController, public chat::mojom::PageHandlerFactory { public: @@ -35,6 +40,10 @@ class ChatUI : public TopChromeWebUIController, ~ChatUI() override; + void BindInterface( + mojo::PendingReceiver + pending_receiver); + void BindInterface( mojo::PendingReceiver receiver); @@ -45,9 +54,8 @@ class ChatUI : public TopChromeWebUIController, embedder_ = embedder; } - static constexpr std::string + static constexpr std::string GetWebUIName() { return "Chat"; } - GetWebUIName() { return "Chat"; } void SetSiteInfo(chat::mojom::SiteInfoPtr site_info, content::WebContents* contents ); private: @@ -55,6 +63,7 @@ class ChatUI : public TopChromeWebUIController, mojo::PendingRemote page, mojo::PendingReceiver receiver) override; + std::unique_ptr color_provider_handler_; std::unique_ptr page_handler_; mojo::Receiver page_factory_receiver_{ From 3b0447efa655b0cfa1218c657d2154fe7cff03fd Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 29 Nov 2024 01:17:52 +0800 Subject: [PATCH 31/34] Register chat ui to color change handler --- chrome/browser/chrome_browser_interface_binders.cc | 2 +- chrome/browser/resources/side_panel/chat/chat.html | 3 ++- chrome/browser/resources/side_panel/chat/chat_app.css | 7 ++----- chrome/browser/resources/side_panel/chat/chat_app.ts | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/chrome/browser/chrome_browser_interface_binders.cc b/chrome/browser/chrome_browser_interface_binders.cc index 34ccbaf582d15e..dfbf0a31736132 100644 --- a/chrome/browser/chrome_browser_interface_binders.cc +++ b/chrome/browser/chrome_browser_interface_binders.cc @@ -1244,7 +1244,7 @@ void PopulateChromeWebUIFrameBinders( policy::local_user_files::LocalFilesMigrationUI, #endif NewTabPageUI, OmniboxPopupUI, BookmarksSidePanelUI, CustomizeChromeUI, - InternalsUI, ReadingListUI, TabSearchUI, WebuiGalleryUI, + InternalsUI, ReadingListUI, ChatUI, TabSearchUI, WebuiGalleryUI, HistoryClustersSidePanelUI, ShoppingInsightsSidePanelUI, media_router::AccessCodeCastUI, commerce::ProductSpecificationsUI>(map); diff --git a/chrome/browser/resources/side_panel/chat/chat.html b/chrome/browser/resources/side_panel/chat/chat.html index c837c219545a73..90671cc2b2c16f 100644 --- a/chrome/browser/resources/side_panel/chat/chat.html +++ b/chrome/browser/resources/side_panel/chat/chat.html @@ -8,7 +8,8 @@ diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index ba7d1f6a9c9f17..ae530ddba20ce8 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -4,13 +4,10 @@ * #scheme=relative * #css_wrapper_metadata_end */ -@import url('https://fonts.googleapis.com/css2?family=Poppins:wght@400;500;600&display=swap'); - * { margin: 0; padding: 0; box-sizing: border-box; - font-family: "Poppins", sans-serif; } :root { @@ -29,7 +26,6 @@ cr-input, cr-textarea { } cr-button { - padding: 10px 10px; border: 1px solid #000000; border-radius: 12px; @@ -47,9 +43,10 @@ cr-button { display: flex; flex-direction: column; height: calc(100vh - 8px); - background-color: transparent; + width: 100vw; margin: 4px; padding: 8px; + background: transparent; } .top { diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index e2679c0313aacf..9f4ab3377210d4 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -5,7 +5,7 @@ import '//resources/cr_elements/cr_icon_button/cr_icon_button.js'; import '//resources/cr_elements/cr_input/cr_input.js'; import '//resources/cr_elements/cr_textarea/cr_textarea.js'; -// import {ColorChangeUpdater} from 'chrome://resources/cr_components/color_change_listener/colors_css_updater.js'; +import {ColorChangeUpdater} from 'chrome://resources/cr_components/color_change_listener/colors_css_updater.js'; import {loadTimeData} from 'chrome://resources/js/load_time_data.js'; import {CrLitElement} from '//resources/lit/v3_0/lit.rollup.js'; import {getCss} from './chat_app.css.js'; @@ -41,7 +41,7 @@ export class ChatAppElement extends CrLitElement { constructor() { super(); - // ColorChangeUpdater.forDocument().start(); + ColorChangeUpdater.forDocument().start(); } static get is() { From 69bf09b3ea5b01f2eba467d242ca74f83ceed2ce Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 29 Nov 2024 14:49:55 +0800 Subject: [PATCH 32/34] Refactor chat panel layout and styling --- .../resources/side_panel/chat/chat.html | 6 +- .../resources/side_panel/chat/chat_app.css | 226 ++++-------------- .../side_panel/chat/chat_app.html.ts | 72 +++--- 3 files changed, 89 insertions(+), 215 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat.html b/chrome/browser/resources/side_panel/chat/chat.html index 90671cc2b2c16f..dde8e00d877cd2 100644 --- a/chrome/browser/resources/side_panel/chat/chat.html +++ b/chrome/browser/resources/side_panel/chat/chat.html @@ -8,9 +8,11 @@ diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index ae530ddba20ce8..29249903405a4c 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -4,223 +4,95 @@ * #scheme=relative * #css_wrapper_metadata_end */ -* { - margin: 0; - padding: 0; - box-sizing: border-box; -} - -:root { - --text-color: #343541; - --icon-color: #a9a9bc; - --icon-hover-bg: #f1f1f3; - --placeholder-color: #6c6c6c; - --outgoing-chat-bg: #FFFFFF; - --incoming-chat-bg: #F7F7F8; - --outgoing-chat-border: #FFFFFF; - --incoming-chat-border: #D9D9E3; -} - -cr-input, cr-textarea { - width: 100%; -} - -cr-button { - padding: 10px 10px; - border: 1px solid #000000; - border-radius: 12px; - cursor: pointer; - font-size: 12px; - text-decoration: none; - color: #474444; -} - -.no-error { - --cr-input-error-display: none; -} - #container { display: flex; flex-direction: column; - height: calc(100vh - 8px); - width: 100vw; - margin: 4px; - padding: 8px; - background: transparent; -} - -.top { - flex: 0 0 auto; - display: flex; - font-size: 1rem; - border-radius: 12px; - border: 1px solid #e3e3e3; - padding: 12px; - max-width: fit-content; -} - -.middle { - flex: 1; - display: flex; - flex-direction: column; - gap: 12px; - /* Expands to fill the remaining space */ + height: calc(100vh - 24px); + padding: 10px 12px; } -.bottom { - flex: 0 0 auto; +#conversation-container { + flex-grow: 1; display: flex; flex-direction: column; - gap: 12px; + gap: 8px; } -.button-container { - display: flex; - gap: 6px; - flex-direction: column; - align-items: flex-start; +.conversation-content { + flex-grow: 1; } -.query-container { +.query-prompt-container { display: block; padding: 6px; width: fit-content; border-radius: 8px; - border: 1px solid #e3e3e3; - font-size: 14px; - background-color: #efeff2; + border: 1px solid; } -.response-container { - display: block; - padding: 6px; - width: fit-content; - font-size: 12px; +.action-buttons-container { + display: flex; + gap: 6px; + flex-direction: column; + align-items: flex-start; + padding-bottom: 6px; } .action-button { display: inline-block; - padding: 8px; - border: 1px solid #E3E3E3; - border-radius: 8px; + padding: 10px 12px 10px 12px; + border: 1px solid; + border-radius: 12px; cursor: pointer; - font-size: 12px; text-decoration: none; - background-color: #FFFFFF; } -.site-info { - width: 80vw; +#prompt-container { display: flex; flex-direction: column; + border: 1px solid; + border-radius: 12px; align-items: flex-start; - font-size: 10px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; + padding: 10px 12px; + gap: 12px; } -.typing-container { - bottom: 0; - width: 100%; +.site-info-container { display: flex; flex-direction: column; - padding: 20px 10px; - gap: 12px; - background-color: #efeded; - border: 1px solid #cfcece; - border-radius: 16px; align-items: flex-start; + white-space: nowrap; + width: 100vw; + max-width: 100%; } -.typing-container .typing-content { - display: flex; - width: 100%; - flex-direction: row; - gap: 12px; -} - -.typing-container .typing-textarea { - width: 100%; - display: flex; -} - -.typing-textarea { - resize: none; - height: 55px; - width: 100%; - padding: 15px 45px 15px 20px; - color: var(--text-color); - font-size: 14px; - border-radius: 12px; - max-height: 250px; - overflow-y: auto; - border: 1px solid #e3e3e3; -} - -.typing-textarea textarea::placeholder { - color: var(--placeholder-color); -} - -.typing-textarea textarea::placeholder { - color: var(--placeholder-color); -} - -.typing-content span { - width: 55px; - height: 55px; - display: flex; - border-radius: 12px; - font-size: 1rem; - align-items: center; - justify-content: center; - color: var(--icon-color); -} - -.typing-textarea span { - position: absolute; - right: 0; - bottom: 0; - visibility: hidden; -} - -span.material-symbols-rounded { - font-size: 1.25rem !important; -} - -.typing-content span { - width: 55px; - height: 55px; - display: flex; - border-radius: 4px; - font-size: 1.35rem; - align-items: center; - justify-content: center; - color: var(--icon-color); -} - -.typing-textarea span { - position: absolute; - right: 0; - bottom: 0; - visibility: hidden; -} - -.typing-textarea textarea:valid ~ span { - visibility: visible; +.site-info-container .site-info-content { + width: 98%; + display: inline-block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + vertical-align: top; + margin: 0px; + padding: 0px; + font-size: 12px; + box-sizing: border-box; } -.typing-controls { +.typing-content { display: flex; + flex-direction: row; + gap: 6px; + width: 100vw; + max-width: 100%; } -.typing-controls span { - margin-left: 7px; - font-size: 1rem; - background: var(--incoming-chat-bg); - outline: 1px solid var(--incoming-chat-border); +.typing-content .prompt-input { + display: inline-block; + width: 90%; } -.typing-controls span:hover { - background: var(--icon-hover-bg); +.typing-content .send-btn { + display: inline-block; + width: auto; } \ No newline at end of file diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 2ff0a6314ce7d8..49182944de02e0 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -4,26 +4,26 @@ import {html} from '//resources/lit/v3_0/lit.rollup.js'; export function getHtml(this: ChatAppElement) { return html`
-
- ${ - this.conversations_.map((conversation, _) => { - return conversation.query.length > 0 - ? html` -

${conversation.query}

- ${conversation.response.length > 0 ? html`

+

+
+ ${ + this.conversations_.map((conversation, _) => { + return conversation.query.length > 0 + ? html` +

${conversation.query}

+ ${conversation.response.length > 0 ? html`

+ ${conversation.response}

` : html``}` + : + html`${conversation.response.length > 0 ? html`

${conversation.response}

` : html``}` - : - html`${conversation.response.length > 0 ? html`

- ${conversation.response}

` : html``}` - }) - } -

- ${this.completionResult_} -

-
-
-
+ }) + } +

+ ${this.completionResult_} +

+
+
${this.siteInfo_.isContentUsableInConversations && this.conversations_ && this.conversations_.length == 0 ? this.actionList_.map((item, _) => html` -
-

${this.siteInfo_.url}

-

${this.siteInfo_.title}

-
-
- - - - Send - -
+
+
+
+

${this.siteInfo_.url}

+

${this.siteInfo_.title}

+
+
+ + + + Send +
`; From 2eda1c2f0a3870d865a77d3e58d8651484c5230a Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Fri, 29 Nov 2024 18:04:11 +0800 Subject: [PATCH 33/34] Add site info as context in prompt display in chat panel UI Enhanced the chat panel to conditionally display site information for specific actions such as summarizing, explaining, or drafting social media posts. This includes new CSS for layout, updated TypeScript logic to manage data handling, and HTML changes to render site details when necessary. --- .../resources/side_panel/chat/chat_app.css | 70 ++++++++++++++++++- .../side_panel/chat/chat_app.html.ts | 25 +++++-- .../resources/side_panel/chat/chat_app.ts | 64 +++++++++++++++-- 3 files changed, 145 insertions(+), 14 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index 29249903405a4c..f1540e51769e1f 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -27,7 +27,69 @@ padding: 6px; width: fit-content; border-radius: 8px; - border: 1px solid; + background: var(--color-side-panel-content-background); +} + +.query-prompt-container .siteinfo-section { + display: flex; + flex-direction: row; + gap: 6px; +} + +.query-prompt-container .siteinfo-section .vertline{ + display: block; + flex: 0 0 auto; + width: 4px; + border-radius: 3px; + background-color: rgba(0,0,0,0.1); +} + +.query-prompt-container .siteinfo-section .content { + display: flex; + flex-direction: column; + flex-grow: 1; + /*font-size: 13px;*/ + /*line-height: 18px;*/ + font-weight: 400; + /*color: rgba(0, 0, 0, 1);*/ + width: auto; + max-width: 100%; +} + +.query-prompt-container .siteinfo-section .content .title { + width: 95%; + display: inline-block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + vertical-align: top; + margin: 0px; + padding: 0px; + /*font-size: 12px;*/ + box-sizing: border-box; +} + +.query-prompt-container .siteinfo-section .content .url { + width: 95%; + display: inline-block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + vertical-align: top; + margin: 0px; + padding: 0px; + /*font-size: 12px;*/ + box-sizing: border-box; + /*color: rgba(0, 0, 0, 0.5);*/ +} + +.query-prompt-container .prompt-section { + font-weight: 400; + line-height: 22px; + width: 100%; + word-wrap: break-word; + white-space: normal; + /*color:rgba(0, 0, 0, 0.85);*/ } .action-buttons-container { @@ -41,20 +103,22 @@ .action-button { display: inline-block; padding: 10px 12px 10px 12px; - border: 1px solid; + border: 1px solid var(--color-side-panel--background); border-radius: 12px; cursor: pointer; text-decoration: none; + background: var(--color-side-panel-content-background); + color: var(--color-side-panel-card-primary-foreground); } #prompt-container { display: flex; flex-direction: column; - border: 1px solid; border-radius: 12px; align-items: flex-start; padding: 10px 12px; gap: 12px; + background: var(--color-side-panel-content-background); } .site-info-container { diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index 49182944de02e0..ba8c3e9c6ff4d3 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -10,7 +10,18 @@ export function getHtml(this: ChatAppElement) { this.conversations_.map((conversation, _) => { return conversation.query.length > 0 ? html` -

${conversation.query}

+
+ ${conversation.shouldDisplaySiteInfo ? html` +
+
+
+
${conversation.title}
+
${conversation.url}
+
+
` : html`` + } +
${conversation.query}
+
${conversation.response.length > 0 ? html`

${conversation.response}

` : html``}` : @@ -36,10 +47,14 @@ export function getHtml(this: ChatAppElement) {
-
-

${this.siteInfo_.url}

-

${this.siteInfo_.title}

-
+ ${ + this.siteInfo_.isContentUsableInConversations ? + html` +
+

${this.siteInfo_.url}

+

${this.siteInfo_.title}

+
` : html`` + }
0) { if (this.conversations_ != null) { if (this.conversations_.length == 0) { - this.conversations_.push({query: this.query_ ?? "", response: this.completionResult_}); + this.conversations_.push({ + query: this.query_ ?? "", + shouldDisplaySiteInfo: false, + title: this.siteInfo_.title ?? "", + url: this.siteInfo_.url ?? "", + response: this.completionResult_ + }); } else { const lastIndex = this.conversations_.length - 1; const lastConversation = this.conversations_[lastIndex]; @@ -124,7 +171,12 @@ export class ChatAppElement extends CrLitElement { } this.completionResult_ = ""; this.submittedQuery_ = this.query_; - this.conversations_.push({query: this.query_ ?? "", response: ""}); + this.conversations_.push({ + query: this.query_ ?? "", shouldDisplaySiteInfo: false, + title: this.siteInfo_.title ?? "", + url: this.siteInfo_.url ?? "", + response: "" + }); this.query_ = ""; this.chatApiProxy_.submitQuery(ActionType.QUERY, this.submittedQuery_ ?? ""); } From 2986f68e447d08ece701c9631e1e00f023f58945 Mon Sep 17 00:00:00 2001 From: nyinyithann Date: Tue, 3 Dec 2024 14:00:47 +0800 Subject: [PATCH 34/34] Enhance chat UI and manage color updates. Refactor chat message styling by using a div with consistent padding, improving visual alignment. Remove redundant color update code from the constructor and add color update management during the loading phase to ensure proper stylesheet application. --- .../browser/resources/side_panel/chat/chat_app.css | 7 ++++++- .../resources/side_panel/chat/chat_app.html.ts | 12 ++++++------ chrome/browser/resources/side_panel/chat/chat_app.ts | 10 ++++++++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/chrome/browser/resources/side_panel/chat/chat_app.css b/chrome/browser/resources/side_panel/chat/chat_app.css index f1540e51769e1f..660c5f2730b48c 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.css +++ b/chrome/browser/resources/side_panel/chat/chat_app.css @@ -22,9 +22,14 @@ flex-grow: 1; } +.conversation-content .message-container { + display: block; + padding: 10px 12px; +} + .query-prompt-container { display: block; - padding: 6px; + padding: 10px 12px; width: fit-content; border-radius: 8px; background: var(--color-side-panel-content-background); diff --git a/chrome/browser/resources/side_panel/chat/chat_app.html.ts b/chrome/browser/resources/side_panel/chat/chat_app.html.ts index ba8c3e9c6ff4d3..a528a4bc673720 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.html.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.html.ts @@ -22,17 +22,17 @@ export function getHtml(this: ChatAppElement) { }
${conversation.query}
- ${conversation.response.length > 0 ? html`

- ${conversation.response}

` : html``}` + ${conversation.response.length > 0 ? html`
+ ${conversation.response}
` : html``}` : - html`${conversation.response.length > 0 ? html`

- ${conversation.response}

` : html``}` + html`${conversation.response.length > 0 ? html`
+ ${conversation.response}
` : html``}` }) } -

+

${this.completionResult_} -

+
${this.siteInfo_.isContentUsableInConversations && this.conversations_ && this.conversations_.length == 0 ? diff --git a/chrome/browser/resources/side_panel/chat/chat_app.ts b/chrome/browser/resources/side_panel/chat/chat_app.ts index 789cf747566ae0..4e561b1ac71955 100644 --- a/chrome/browser/resources/side_panel/chat/chat_app.ts +++ b/chrome/browser/resources/side_panel/chat/chat_app.ts @@ -45,7 +45,6 @@ export class ChatAppElement extends CrLitElement { constructor() { super(); - ColorChangeUpdater.forDocument().start(); } static get is() { @@ -181,9 +180,15 @@ export class ChatAppElement extends CrLitElement { this.chatApiProxy_.submitQuery(ActionType.QUERY, this.submittedQuery_ ?? ""); } + refreshColorCss() { + const updater = ColorChangeUpdater.forDocument(); + updater.start(); + updater.refreshColorsCss(); + } + override connectedCallback() { super.connectedCallback(); - + window.addEventListener('load', this.refreshColorCss); setTimeout(async () => { this.chatApiProxy_.showUI(); const {siteInfo} = await this.chatApiProxy_.getSiteInfo(); @@ -201,6 +206,7 @@ export class ChatAppElement extends CrLitElement { override disconnectedCallback() { super.disconnectedCallback(); + window.removeEventListener('load', this.refreshColorCss); this.listenerIds_.forEach( id => this.chatApiProxy_.getCallbackRouter().removeListener(id)); }