lens_overlay_query_controller.cc | Explore in Territory

// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/ui/lens/lens_overlay_query_controller.h"

#include <optional>

#include "base/base64url.h"
#include "base/logging.h"
#include "base/rand_util.h"
#include "base/task/bind_post_task.h"
#include "base/task/thread_pool.h"
#include "base/time/time.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/lens/core/mojom/geometry.mojom.h"
#include "chrome/browser/lens/core/mojom/overlay_object.mojom-forward.h"
#include "chrome/browser/lens/core/mojom/text.mojom.h"
#include "chrome/browser/search_engines/template_url_service_factory.h"
#include "chrome/browser/ui/lens/lens_overlay_image_helper.h"
#include "chrome/browser/ui/lens/lens_overlay_proto_converter.h"
#include "chrome/browser/ui/lens/lens_overlay_url_builder.h"
#include "chrome/browser/ui/lens/ref_counted_lens_overlay_client_logs.h"
#include "chrome/common/channel_info.h"
#include "components/endpoint_fetcher/endpoint_fetcher.h"
#include "components/lens/lens_features.h"
#include "components/lens/proto/server/lens_overlay_response.pb.h"
#include "components/metrics_services_manager/metrics_services_manager.h"
#include "components/search_engines/template_url_service.h"
#include "components/signin/public/base/consent_level.h"
#include "components/signin/public/identity_manager/access_token_info.h"
#include "components/signin/public/identity_manager/identity_manager.h"
#include "components/variations/variations.mojom.h"
#include "components/variations/variations_client.h"
#include "components/variations/variations_ids_provider.h"
#include "components/version_info/channel.h"
#include "google_apis/common/api_error_codes.h"
#include "google_apis/gaia/gaia_constants.h"
#include "google_apis/gaia/gaia_urls.h"
#include "google_apis/google_api_keys.h"
#include "net/base/url_util.h"
#include "net/http/http_request_headers.h"
#include "net/traffic_annotation/network_traffic_annotation.h"
#include "services/network/public/cpp/resource_request.h"
#include "services/network/public/cpp/shared_url_loader_factory.h"
#include "services/network/public/cpp/simple_url_loader.h"
#include "third_party/icu/source/common/unicode/locid.h"
#include "third_party/icu/source/common/unicode/unistr.h"
#include "third_party/icu/source/i18n/unicode/timezone.h"
#include "third_party/lens_server_proto/lens_overlay_client_platform.pb.h"
#include "third_party/lens_server_proto/lens_overlay_filters.pb.h"
#include "third_party/lens_server_proto/lens_overlay_platform.pb.h"
#include "third_party/lens_server_proto/lens_overlay_polygon.pb.h"
#include "third_party/lens_server_proto/lens_overlay_request_id.pb.h"
#include "third_party/lens_server_proto/lens_overlay_surface.pb.h"
#include "third_party/lens_server_proto/lens_overlay_visual_search_interaction_data.pb.h"
#include "ui/gfx/geometry/rect.h"

namespace lens {

namespace {

const int kMaxDownloadBytes = …;
const int kTranslateTaskCompletionID = …;
const int kCopyTextTaskCompletionID = …;
const int kSelectTextTaskCompletionID = …;

// The name string for the header for variations information.
constexpr char kClientDataHeader[] = …;
constexpr char kHttpMethod[] = …;
constexpr char kContentType[] = …;
constexpr char kDeveloperKey[] = …;
constexpr char kSessionIdQueryParameterKey[] = …;
constexpr char kOAuthConsumerName[] = …;
constexpr char kStartTimeQueryParameter[] = …;
constexpr char kGen204IdentifierQueryParameter[] = …;
constexpr char kVisualSearchInteractionDataQueryParameterKey[] = …;

constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotationTag = …;

lens::CoordinateType ConvertToServerCoordinateType(
    lens::mojom::CenterRotatedBox_CoordinateType type) { … }

lens::CenterRotatedBox ConvertToServerCenterRotatedBox(
    lens::mojom::CenterRotatedBoxPtr box) { … }

std::vector<std::string> CreateOAuthHeader(
    GoogleServiceAuthError error,
    signin::AccessTokenInfo access_token_info) { … }

std::map<std::string, std::string> AddStartTimeQueryParam(
    std::map<std::string, std::string> additional_search_query_params) { … }

lens::LensOverlayClientLogs::LensOverlayEntryPoint
LenOverlayEntryPointFromInvocationSource(
    lens::LensOverlayInvocationSource invocation_source) { … }

}  // namespace

LensOverlayQueryController::LensOverlayQueryController(
    LensOverlayFullImageResponseCallback full_image_callback,
    LensOverlayUrlResponseCallback url_callback,
    LensOverlayInteractionResponseCallback interaction_data_callback,
    LensOverlayThumbnailCreatedCallback thumbnail_created_callback,
    variations::VariationsClient* variations_client,
    signin::IdentityManager* identity_manager,
    Profile* profile,
    lens::LensOverlayInvocationSource invocation_source,
    bool use_dark_mode)
    : … { … }

LensOverlayQueryController::~LensOverlayQueryController() = default;

void LensOverlayQueryController::StartQueryFlow(
    const SkBitmap& screenshot,
    std::optional<GURL> page_url,
    std::optional<std::string> page_title,
    std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes,
    float ui_scale_factor) { … }

void LensOverlayQueryController::PrepareAndFetchFullImageRequest() { … }

void LensOverlayQueryController::OnImageDataReady(
    scoped_refptr<lens::RefCountedLensOverlayClientLogs> ref_counted_logs,
    lens::ImageData image_data) { … }

lens::LensOverlayClientContext
LensOverlayQueryController::CreateClientContext() { … }

std::map<std::string, std::string>
LensOverlayQueryController::AddVisualSearchInteractionLogData(
    std::map<std::string, std::string> additional_search_query_params,
    lens::LensOverlaySelectionType selection_type) { … }

void LensOverlayQueryController::FetchFullImageRequest(
    std::unique_ptr<lens::LensOverlayRequestId> request_id,
    lens::ImageData image_data,
    lens::LensOverlayClientLogs client_logs) { … }

void LensOverlayQueryController::FullImageFetchResponseHandler(
    int64_t query_start_time_ms,
    int request_sequence_id,
    std::unique_ptr<EndpointResponse> response) { … }

void LensOverlayQueryController::SendLatencyGen204IfEnabled(
    int64_t latency_ms) { … }

void LensOverlayQueryController::OnLatencyGen204LoaderComplete(
    std::unique_ptr<std::string> response_body) { … }

void LensOverlayQueryController::SendTaskCompletionGen204IfEnabled(
    lens::mojom::UserAction user_action) { … }

void LensOverlayQueryController::OnTaskCompletionGen204LoaderComplete(
    std::unique_ptr<std::string> response_body) { … }

void LensOverlayQueryController::RunFullImageCallbackForError() { … }

void LensOverlayQueryController::EndQuery() { … }

void LensOverlayQueryController::SendFullPageTranslateQuery(
    const std::string& source_language,
    const std::string& target_language) { … }

void LensOverlayQueryController::SendRegionSearch(
    lens::mojom::CenterRotatedBoxPtr region,
    lens::LensOverlaySelectionType lens_selection_type,
    std::map<std::string, std::string> additional_search_query_params,
    std::optional<SkBitmap> region_bytes) { … }

void LensOverlayQueryController::SendMultimodalRequest(
    lens::mojom::CenterRotatedBoxPtr region,
    const std::string& query_text,
    lens::LensOverlaySelectionType multimodal_selection_type,
    std::map<std::string, std::string> additional_search_query_params,
    std::optional<SkBitmap> region_bytes) { … }

void LensOverlayQueryController::SendTextOnlyQuery(
    const std::string& query_text,
    TextOnlyQueryType text_only_query_type,
    std::map<std::string, std::string> additional_search_query_params) { … }

void LensOverlayQueryController::SendInteraction(
    lens::mojom::CenterRotatedBoxPtr region,
    std::optional<std::string> query_text,
    std::optional<std::string> object_id,
    lens::LensOverlaySelectionType selection_type,
    std::map<std::string, std::string> additional_search_query_params,
    std::optional<SkBitmap> region_bytes) { … }

void LensOverlayQueryController::OnImageCropReady(
    int request_index,
    lens::mojom::CenterRotatedBoxPtr region,
    std::optional<std::string> query_text,
    std::optional<std::string> object_id,
    lens::LensOverlaySelectionType selection_type,
    std::map<std::string, std::string> additional_search_query_params,
    scoped_refptr<lens::RefCountedLensOverlayClientLogs> ref_counted_logs,
    std::optional<lens::ImageCrop> image_crop) { … }

void LensOverlayQueryController::
    FetchInteractionRequestAndGenerateUrlIfClusterInfoReady(
        int request_index,
        lens::mojom::CenterRotatedBoxPtr region,
        std::optional<std::string> query_text,
        std::optional<std::string> object_id,
        lens::LensOverlaySelectionType selection_type,
        std::map<std::string, std::string> additional_search_query_params,
        std::optional<lens::ImageCrop> image_crop,
        lens::LensOverlayClientLogs client_logs) { … }

lens::LensOverlayServerRequest
LensOverlayQueryController::CreateInteractionRequest(
    lens::mojom::CenterRotatedBoxPtr region,
    std::optional<std::string> query_text,
    std::optional<std::string> object_id,
    std::optional<lens::ImageCrop> image_crop,
    lens::LensOverlayClientLogs client_logs,
    std::unique_ptr<lens::LensOverlayRequestId> request_id) { … }

void LensOverlayQueryController::
    FetchInteractionRequestAndGenerateLensSearchUrl(
        int request_index,
        lens::mojom::CenterRotatedBoxPtr region,
        std::optional<std::string> query_text,
        std::optional<std::string> object_id,
        lens::LensOverlaySelectionType selection_type,
        std::map<std::string, std::string> additional_search_query_params,
        std::optional<lens::ImageCrop> image_crop,
        lens::LensOverlayClientLogs client_logs,
        lens::LensOverlayClusterInfo cluster_info) { … }

void LensOverlayQueryController::InteractionFetchResponseHandler(
    std::unique_ptr<EndpointResponse> response) { … }

void LensOverlayQueryController::RunInteractionCallbackForError() { … }

void LensOverlayQueryController::ResetRequestClusterInfoState() { … }

void LensOverlayQueryController::CreateAndFetchEndpointFetcher(
    lens::LensOverlayServerRequest request_data,
    base::OnceCallback<void(std::unique_ptr<EndpointFetcher>)>
        fetcher_created_callback,
    EndpointFetcherCallback fetched_response_callback) { … }

void LensOverlayQueryController::OnFullImageEndpointFetcherCreated(
    std::unique_ptr<EndpointFetcher> endpoint_fetcher) { … }

void LensOverlayQueryController::OnInteractionEndpointFetcherCreated(
    std::unique_ptr<EndpointFetcher> endpoint_fetcher) { … }

void LensOverlayQueryController::FetchEndpoint(
    lens::LensOverlayServerRequest request_data,
    base::OnceCallback<void(std::unique_ptr<EndpointFetcher>)>
        fetcher_created_callback,
    EndpointFetcherCallback fetched_response_callback,
    std::vector<std::string> headers) { … }

}  // namespace lens
chromium/chrome/browser/ui/lens/lens_overlay_query_controller.cc
