visual_query_classifier_agent.cc | Explore in Territory

// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/renderer/companion/visual_query/visual_query_classifier_agent.h"

#include "base/feature_list.h"
#include "base/files/file.h"
#include "base/files/memory_mapped_file.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/metrics/histogram_macros_local.h"
#include "base/strings/string_number_conversions.h"
#include "base/task/task_traits.h"
#include "base/task/thread_pool.h"
#include "chrome/common/companion/visual_query.mojom.h"
#include "chrome/common/companion/visual_query/features.h"
#include "chrome/renderer/companion/visual_query/visual_query_classification_and_eligibility.h"
#include "components/optimization_guide/proto/visual_search_model_metadata.pb.h"
#include "content/public/renderer/render_frame.h"
#include "content/public/renderer/render_frame_observer.h"
#include "mojo/public/mojom/base/proto_wrapper.mojom.h"
#include "third_party/blink/public/common/associated_interfaces/associated_interface_registry.h"
#include "third_party/blink/public/platform/browser_interface_broker_proxy.h"
#include "third_party/blink/public/web/web_element.h"
#include "third_party/blink/public/web/web_local_frame.h"
#include "third_party/blink/public/web/web_view.h"
#include "third_party/skia/include/core/SkBitmap.h"

namespace companion::visual_query {

namespace {

EligibilitySpec;
FeatureLibrary;
OrOfThresholdingRules;
ThresholdingRule;

DOMImageList;

EligibilitySpec CreateEligibilitySpec(
    std::optional<mojo_base::ProtoWrapper> config_proto) { … }

// Depth-first search for recursively traversing DOM elements and pulling out
// references for images (SkBitmap).
void FindImageElements(blink::WebElement element,
                       std::vector<blink::WebElement>& images) { … }

// Top-level wrapper call to trigger DOM traversal to find images.
DOMImageList FindImagesOnPage(content::RenderFrame* render_frame) { … }

ClassificationResultsAndStats ClassifyImagesOnBackground(
    DOMImageList images,
    std::string model_data,
    std::optional<mojo_base::ProtoWrapper> config_proto,
    gfx::SizeF viewport_size) { … }

}  // namespace

VisualQueryClassifierAgent::VisualQueryClassifierAgent(
    content::RenderFrame* render_frame)
    : … { … }

VisualQueryClassifierAgent::~VisualQueryClassifierAgent() = default;

// static
VisualQueryClassifierAgent* VisualQueryClassifierAgent::Create(
    content::RenderFrame* render_frame) { … }

void VisualQueryClassifierAgent::StartVisualClassification(
    base::File visual_model,
    std::optional<mojo_base::ProtoWrapper> config_proto,
    mojo::PendingRemote<mojom::VisualSuggestionsResultHandler> result_handler) { … }

void VisualQueryClassifierAgent::OnClassificationDone(
    ClassificationResultsAndStats results) { … }

void VisualQueryClassifierAgent::OnRendererAssociatedRequest(
    mojo::PendingAssociatedReceiver<mojom::VisualSuggestionsRequestHandler>
        receiver) { … }

void VisualQueryClassifierAgent::DidFinishLoad() { … }

void VisualQueryClassifierAgent::HandleGetModelCallback(
    base::File file,
    std::optional<mojo_base::ProtoWrapper> config) { … }

void VisualQueryClassifierAgent::OnDestruct() { … }

}  // namespace companion::visual_query
chromium/chrome/renderer/companion/visual_query/visual_query_classifier_agent.cc
