lens_overlay_controller.cc | Explore in Territory

// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/ui/lens/lens_overlay_controller.h"

#include "base/containers/contains.h"
#include "base/functional/bind.h"
#include "base/metrics/histogram_functions.h"
#include "base/no_destructor.h"
#include "base/system/sys_info.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "chrome/browser/feedback/show_feedback_page.h"
#include "chrome/browser/lens/core/mojom/geometry.mojom.h"
#include "chrome/browser/lens/core/mojom/overlay_object.mojom.h"
#include "chrome/browser/lens/core/mojom/text.mojom.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/search/search.h"
#include "chrome/browser/task_manager/web_contents_tags.h"
#include "chrome/browser/themes/theme_service.h"
#include "chrome/browser/ui/browser_window/public/browser_window_features.h"
#include "chrome/browser/ui/browser_window/public/browser_window_interface.h"
#include "chrome/browser/ui/exclusive_access/exclusive_access_manager.h"
#include "chrome/browser/ui/lens/lens_overlay_controller_glue.h"
#include "chrome/browser/ui/lens/lens_overlay_entry_point_controller.h"
#include "chrome/browser/ui/lens/lens_overlay_event_handler.h"
#include "chrome/browser/ui/lens/lens_overlay_image_helper.h"
#include "chrome/browser/ui/lens/lens_overlay_query_controller.h"
#include "chrome/browser/ui/lens/lens_overlay_side_panel_coordinator.h"
#include "chrome/browser/ui/lens/lens_overlay_theme_utils.h"
#include "chrome/browser/ui/lens/lens_overlay_url_builder.h"
#include "chrome/browser/ui/lens/lens_permission_bubble_controller.h"
#include "chrome/browser/ui/lens/lens_search_bubble_controller.h"
#include "chrome/browser/ui/views/side_panel/side_panel.h"
#include "chrome/browser/ui/views/side_panel/side_panel_coordinator.h"
#include "chrome/browser/ui/views/side_panel/side_panel_enums.h"
#include "chrome/browser/ui/views/side_panel/side_panel_ui.h"
#include "chrome/browser/ui/views/side_panel/side_panel_util.h"
#include "chrome/browser/ui/webui/util/image_util.h"
#include "chrome/common/chrome_render_frame.mojom.h"
#include "chrome/common/pref_names.h"
#include "chrome/common/webui_url_constants.h"
#include "chrome/grit/generated_resources.h"
#include "components/find_in_page/find_tab_helper.h"
#include "components/lens/lens_features.h"
#include "components/lens/lens_overlay_permission_utils.h"
#include "components/permissions/permission_request_manager.h"
#include "components/sessions/content/session_tab_helper.h"
#include "components/signin/public/identity_manager/identity_manager.h"
#include "components/viz/common/frame_timing_details.h"
#include "components/zoom/zoom_controller.h"
#include "content/public/browser/download_manager.h"
#include "content/public/browser/download_request_utils.h"
#include "content/public/browser/render_frame_host.h"
#include "content/public/browser/render_process_host.h"
#include "content/public/browser/render_view_host.h"
#include "content/public/browser/render_widget_host.h"
#include "content/public/browser/render_widget_host_view.h"
#include "content/public/browser/web_contents_user_data.h"
#include "content/public/browser/web_ui.h"
#include "net/base/network_change_notifier.h"
#include "net/base/url_search_params.h"
#include "net/base/url_util.h"
#include "services/metrics/public/cpp/ukm_builders.h"
#include "services/metrics/public/cpp/ukm_source_id.h"
#include "third_party/blink/public/common/associated_interfaces/associated_interface_provider.h"
#include "third_party/lens_server_proto/lens_overlay_selection_type.pb.h"
#include "third_party/lens_server_proto/lens_overlay_service_deps.pb.h"
#include "ui/base/clipboard/scoped_clipboard_writer.h"
#include "ui/base/l10n/l10n_util.h"
#include "ui/base/webui/web_ui_util.h"
#include "ui/base/window_open_disposition_utils.h"
#include "ui/compositor/compositor.h"
#include "ui/compositor/layer.h"
#include "ui/gfx/geometry/rounded_corners_f.h"
#include "ui/native_theme/native_theme.h"
#include "ui/views/bubble/bubble_dialog_delegate_view.h"
#include "ui/views/controls/webview/web_contents_set_background_color.h"
#include "ui/views/controls/webview/webview.h"
#include "ui/views/layout/flex_layout_types.h"
#include "ui/views/layout/flex_layout_view.h"
#include "ui/views/widget/native_widget.h"

void* kLensOverlayPreselectionWidgetIdentifier = …;

namespace …  // namespace

LensOverlayController::LensOverlayController(
    tabs::TabInterface* tab,
    variations::VariationsClient* variations_client,
    signin::IdentityManager* identity_manager,
    PrefService* pref_service,
    syncer::SyncService* sync_service,
    ThemeService* theme_service)
    : … { … }

LensOverlayController::~LensOverlayController() { … }

DEFINE_CLASS_ELEMENT_IDENTIFIER_VALUE(LensOverlayController, kOverlayId);
DEFINE_CLASS_ELEMENT_IDENTIFIER_VALUE(LensOverlayController,
                                      kOverlaySidePanelWebViewId);

LensOverlayController::SearchQuery::SearchQuery(std::string text_query,
                                                GURL url)
    : … { … }

LensOverlayController::SearchQuery::SearchQuery(const SearchQuery& other) { … }

LensOverlayController::SearchQuery&
LensOverlayController::SearchQuery::operator=(
    const LensOverlayController::SearchQuery& other) { … }

LensOverlayController::SearchQuery::~SearchQuery() = default;

void LensOverlayController::ShowUIWithPendingRegion(
    lens::LensOverlayInvocationSource invocation_source,
    const gfx::Rect& tab_bounds,
    const gfx::Rect& view_bounds,
    const gfx::Rect& image_bounds,
    const SkBitmap& region_bitmap) { … }

void LensOverlayController::ShowUIWithPendingRegion(
    lens::LensOverlayInvocationSource invocation_source,
    lens::mojom::CenterRotatedBoxPtr region,
    const SkBitmap& region_bitmap) { … }

void LensOverlayController::ShowUI(
    lens::LensOverlayInvocationSource invocation_source) { … }

void LensOverlayController::CloseUIAsync(
    lens::LensOverlayDismissalSource dismissal_source) { … }

void LensOverlayController::CloseUISync(
    lens::LensOverlayDismissalSource dismissal_source) { … }

// static
LensOverlayController* LensOverlayController::GetController(
    content::WebUI* web_ui) { … }

// static
LensOverlayController* LensOverlayController::GetController(
    content::WebContents* tab_contents) { … }

// static
LensOverlayController*
LensOverlayController::GetControllerFromWebViewWebContents(
    content::WebContents* contents) { … }

// static
const std::u16string LensOverlayController::GetFilenameForURL(const GURL& url) { … }

void LensOverlayController::BindOverlay(
    mojo::PendingReceiver<lens::mojom::LensPageHandler> receiver,
    mojo::PendingRemote<lens::mojom::LensPage> page) { … }

void LensOverlayController::BindSidePanel(
    mojo::PendingReceiver<lens::mojom::LensSidePanelPageHandler> receiver,
    mojo::PendingRemote<lens::mojom::LensSidePanelPage> page) { … }

void LensOverlayController::SetSearchboxHandler(
    std::unique_ptr<RealboxHandler> handler) { … }

void LensOverlayController::SetContextualSearchboxHandler(
    std::unique_ptr<RealboxHandler> handler) { … }

void LensOverlayController::ResetSearchboxHandler() { … }

uint64_t LensOverlayController::GetInvocationTimeSinceEpoch() { … }

views::View* LensOverlayController::GetOverlayViewForTesting() { … }

views::WebView* LensOverlayController::GetOverlayWebViewForTesting() { … }

void LensOverlayController::CreateGlueForWebView(views::WebView* web_view) { … }

void LensOverlayController::RemoveGlueForWebView(views::WebView* web_view) { … }

void LensOverlayController::SendText(lens::mojom::TextPtr text) { … }

lens::mojom::OverlayThemePtr LensOverlayController::CreateTheme(
    lens::PaletteId palette_id) { … }

void LensOverlayController::SendObjects(
    std::vector<lens::mojom::OverlayObjectPtr> objects) { … }

void LensOverlayController::NotifyResultsPanelOpened() { … }

void LensOverlayController::TriggerCopyText() { … }

bool LensOverlayController::IsOverlayShowing() { … }

bool LensOverlayController::IsOverlayClosing() { … }

void LensOverlayController::LoadURLInResultsFrame(const GURL& url) { … }

void LensOverlayController::SetSearchboxInputText(const std::string& text) { … }

void LensOverlayController::AddQueryToHistory(std::string query,
                                              GURL search_url) { … }

void LensOverlayController::PopAndLoadQueryFromHistory() { … }

void LensOverlayController::SetSidePanelIsLoadingResults(bool is_loading) { … }

void LensOverlayController::SetSidePanelShowErrorPage(
    bool should_show_error_page) { … }

void LensOverlayController::OnSidePanelWillHide(
    SidePanelEntryHideReason reason) { … }

void LensOverlayController::OnSidePanelHidden() { … }

tabs::TabInterface* LensOverlayController::GetTabInterface() { … }

void LensOverlayController::IssueLensRegionRequestForTesting(
    lens::mojom::CenterRotatedBoxPtr region,
    bool is_click) { … }

void LensOverlayController::IssueTextSelectionRequestForTesting(
    const std::string& text_query,
    int selection_start_index,
    int selection_end_index) { … }

void LensOverlayController::
    RecordUkmAndTaskCompletionForLensOverlayInteractionForTesting(
        lens::mojom::UserAction user_action) { … }

void LensOverlayController::IssueSearchBoxRequestForTesting(
    const std::string& search_box_text,
    AutocompleteMatchType::Type match_type,
    bool is_zero_prefix_suggestion,
    std::map<std::string, std::string> additional_query_params) { … }

void LensOverlayController::IssueTranslateSelectionRequestForTesting(
    const std::string& text_query,
    const std::string& content_language,
    int selection_start_index,
    int selection_end_index) { … }

void LensOverlayController::IssueTranslateFullPageRequestForTesting(
    const std::string& source_language,
    const std::string& target_language) { … }

void LensOverlayController::IssueTranslateFullPageRequest(
    const std::string& source_language,
    const std::string& target_language) { … }

void LensOverlayController::NotifyOverlayInitialized() { … }

void LensOverlayController::CopyText(const std::string& text) { … }

void LensOverlayController::CopyImage(lens::mojom::CenterRotatedBoxPtr region) { … }

void LensOverlayController::SaveAsImage(
    lens::mojom::CenterRotatedBoxPtr region) { … }

void LensOverlayController::RecordUkmAndTaskCompletionForLensOverlayInteraction(
    lens::mojom::UserAction user_action) { … }

std::string LensOverlayController::GetInvocationSourceString() { … }

content::WebContents*
LensOverlayController::GetSidePanelWebContentsForTesting() { … }

const GURL& LensOverlayController::GetPageURLForTesting() { … }

SessionID LensOverlayController::GetTabIdForTesting() { … }

metrics::OmniboxEventProto::PageClassification
LensOverlayController::GetPageClassificationForTesting() { … }

const std::string& LensOverlayController::GetThumbnailForTesting() { … }

void LensOverlayController::OnTextModifiedForTesting() { … }

void LensOverlayController::OnThumbnailRemovedForTesting() { … }

const lens::proto::LensOverlayInteractionResponse&
LensOverlayController::GetLensResponseForTesting() { … }

std::unique_ptr<lens::LensOverlayQueryController>
LensOverlayController::CreateLensQueryController(
    lens::LensOverlayFullImageResponseCallback full_image_callback,
    lens::LensOverlayUrlResponseCallback url_callback,
    lens::LensOverlayInteractionResponseCallback interaction_data_callback,
    lens::LensOverlayThumbnailCreatedCallback thumbnail_created_callback,
    variations::VariationsClient* variations_client,
    signin::IdentityManager* identity_manager,
    Profile* profile,
    lens::LensOverlayInvocationSource invocation_source,
    bool use_dark_mode) { … }

LensOverlayController::OverlayInitializationData::OverlayInitializationData(
    const SkBitmap& screenshot,
    SkBitmap rgb_screenshot,
    lens::PaletteId color_palette,
    std::optional<GURL> page_url,
    std::optional<std::string> page_title)
    : … { … }
LensOverlayController::OverlayInitializationData::~OverlayInitializationData() =
    default;

class LensOverlayController::UnderlyingWebContentsObserver
    : public content::WebContentsObserver { … };

void LensOverlayController::CaptureScreenshot() { … }

void LensOverlayController::FetchViewportImageBoundingBoxes(
    const SkBitmap& bitmap) { … }

void LensOverlayController::DidCaptureScreenshot(
    mojo::AssociatedRemote<chrome::mojom::ChromeRenderFrame>
        chrome_render_frame,
    int attempt_id,
    const SkBitmap& bitmap,
    const std::vector<gfx::Rect>& all_bounds) { … }

void LensOverlayController::CreateInitializationData(
    const SkBitmap& screenshot,
    const std::vector<gfx::Rect>& all_bounds) { … }

void LensOverlayController::ContinueCreateInitializationData(
    const SkBitmap& screenshot,
    const std::vector<gfx::Rect>& all_bounds,
    SkBitmap rgb_screenshot) { … }

void LensOverlayController::AddBoundingBoxesToInitializationData(
    const std::vector<gfx::Rect>& all_bounds) { … }

void LensOverlayController::ShowOverlay() { … }

void LensOverlayController::BackgroundUI() { … }

void LensOverlayController::CloseUIPart2(
    lens::LensOverlayDismissalSource dismissal_source) { … }

void LensOverlayController::InitializeOverlay() { … }

void LensOverlayController::InitializeOverlayUI(
    const OverlayInitializationData& init_data) { … }

std::unique_ptr<views::View> LensOverlayController::CreateViewForOverlay() { … }

bool LensOverlayController::HandleContextMenu(
    content::RenderFrameHost& render_frame_host,
    const content::ContextMenuParams& params) { … }

bool LensOverlayController::HandleKeyboardEvent(
    content::WebContents* source,
    const input::NativeWebKeyboardEvent& event) { … }

void LensOverlayController::OnFullscreenStateChanged() { … }

void LensOverlayController::OnViewBoundsChanged(views::View* observed_view) { … }

void LensOverlayController::OnWidgetDestroying(views::Widget* widget) { … }

void LensOverlayController::OnOmniboxFocusChanged(
    OmniboxFocusState state,
    OmniboxFocusChangeReason reason) { … }

void LensOverlayController::OnFindEmptyText(
    content::WebContents* web_contents) { … }

void LensOverlayController::OnFindResultAvailable(
    content::WebContents* web_contents) { … }

const GURL& LensOverlayController::GetPageURL() const { … }

SessionID LensOverlayController::GetTabId() const { … }

metrics::OmniboxEventProto::PageClassification
LensOverlayController::GetPageClassification() const { … }

std::string& LensOverlayController::GetThumbnail() { … }

const lens::proto::LensOverlayInteractionResponse&
LensOverlayController::GetLensResponse() const { … }

void LensOverlayController::OnTextModified() { … }

void LensOverlayController::OnThumbnailRemoved() { … }

void LensOverlayController::OnSuggestionAccepted(
    const GURL& destination_url,
    AutocompleteMatchType::Type match_type,
    bool is_zero_prefix_suggestion) { … }

void LensOverlayController::OnPageBound() { … }

void LensOverlayController::OnSidePanelDidOpen() { … }

void LensOverlayController::OnSidePanelCloseInterrupted() { … }

void LensOverlayController::OnSidePanelDidClose() { … }

void LensOverlayController::RenderProcessExited(
    content::RenderProcessHost* host,
    const content::ChildProcessTerminationInfo& info) { … }

void LensOverlayController::TabForegrounded(tabs::TabInterface* tab) { … }

void LensOverlayController::TabWillEnterBackground(tabs::TabInterface* tab) { … }

void LensOverlayController::WillDiscardContents(
    tabs::TabInterface* tab,
    content::WebContents* old_contents,
    content::WebContents* new_contents) { … }

void LensOverlayController::WillDetach(
    tabs::TabInterface* tab,
    tabs::TabInterface::DetachReason reason) { … }

void LensOverlayController::DoLensRequest(
    lens::mojom::CenterRotatedBoxPtr region,
    lens::LensOverlaySelectionType selection_type,
    std::optional<SkBitmap> region_bytes) { … }

void LensOverlayController::ActivityRequestedByOverlay(
    ui::mojom::ClickModifiersPtr click_modifiers) { … }

void LensOverlayController::ActivityRequestedByEvent(int event_flags) { … }

void LensOverlayController::AddBackgroundBlur() { … }

void LensOverlayController::CloseRequestedByOverlayCloseButton() { … }

void LensOverlayController::CloseRequestedByOverlayBackgroundClick() { … }

void LensOverlayController::FeedbackRequestedByOverlay() { … }

void LensOverlayController::FeedbackRequestedByEvent(int event_flags) { … }

void LensOverlayController::GetOverlayInvocationSource(
    GetOverlayInvocationSourceCallback callback) { … }

void LensOverlayController::InfoRequestedByOverlay(
    ui::mojom::ClickModifiersPtr click_modifiers) { … }

void LensOverlayController::InfoRequestedByEvent(int event_flags) { … }

void LensOverlayController::IssueLensRegionRequest(
    lens::mojom::CenterRotatedBoxPtr region,
    bool is_click) { … }

void LensOverlayController::IssueLensObjectRequest(
    lens::mojom::CenterRotatedBoxPtr region,
    bool is_mask_click) { … }

void LensOverlayController::IssueTextSelectionRequest(const std::string& query,
                                                      int selection_start_index,
                                                      int selection_end_index) { … }

void LensOverlayController::IssueTranslateSelectionRequest(
    const std::string& query,
    const std::string& content_language,
    int selection_start_index,
    int selection_end_index) { … }

void LensOverlayController::IssueTextSelectionRequestInner(
    const std::string& query,
    int selection_start_index,
    int selection_end_index) { … }

void LensOverlayController::CloseSearchBubble() { … }

void LensOverlayController::ClosePreselectionBubble() { … }

void LensOverlayController::ShowPreselectionBubble() { … }

void LensOverlayController::HidePreselectionBubble() { … }

void LensOverlayController::IssueSearchBoxRequest(
    const std::string& search_box_text,
    AutocompleteMatchType::Type match_type,
    bool is_zero_prefix_suggestion,
    std::map<std::string, std::string> additional_query_params) { … }

void LensOverlayController::HandleStartQueryResponse(
    std::vector<lens::mojom::OverlayObjectPtr> objects,
    lens::mojom::TextPtr text,
    bool is_error) { … }

void LensOverlayController::HandleInteractionURLResponse(
    lens::proto::LensOverlayUrlResponse response) { … }

void LensOverlayController::HandleInteractionDataResponse(
    lens::proto::LensOverlayInteractionResponse response) { … }

void LensOverlayController::HandleThumbnailCreated(
    const std::string& thumbnail_bytes) { … }

void LensOverlayController::SetSearchboxThumbnail(
    const std::string& thumbnail_uri) { … }

void LensOverlayController::RecordTimeToFirstInteraction() { … }

void LensOverlayController::RecordEndOfSessionMetrics(
    lens::LensOverlayDismissalSource dismissal_source) { … }
chromium/chrome/browser/ui/lens/lens_overlay_controller.cc
