visual_query.mojom | Explore in Territory

// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

module companion.visual_query.mojom;

import "mojo/public/mojom/base/proto_wrapper.mojom";
import "mojo/public/mojom/base/read_only_file.mojom";
import "skia/public/mojom/bitmap.mojom";

// Structure used to capture metrics for classification in renderer.
struct ClassificationStats {
  // Stat for the number of eligible images on page.
  uint32 eligible_count;

  // Stat for the number of sensitive images on page.
  uint32 sensitive_count;

  // Stat for the number of shoppy images on page.
  uint32 shoppy_count;

  // Stat for the number of shoppy/nonsensitive images on page.
  uint32 shoppy_nonsensitive_count;

  // Stat for number of visual suggestions on page.
  uint32 results_count;
};

// Encapsulates a visual query suggestion result.
struct VisualQuerySuggestion {
  // raw skbitmap version of image.
  skia.mojom.BitmapN32 image;

  // alt-text for the image.
  string alt_text;
};

// Browser-side interface handling responses for visual query suggestions.
interface VisualSuggestionsResultHandler {
  // Called when the renderer is ready to send visual query suggestions back
  // to the browser.
  // `results` is a list of visual query suggestions.
  // `stats` contains statistics about the classification (typically used for
  // logging).
  HandleClassification(array<VisualQuerySuggestion> results,
    ClassificationStats stats);
};

// Renderer-side interface listening for visual query suggestions requests.
// We make this IPC call once per page load to the main render frame.
interface VisualSuggestionsRequestHandler {
  // Called to kick off the visual classification request and passes all of
  // the necessary dependencies needed on the renderer side to complete task.
  // `visual_model_file` is the file pointer to the model used for
  // classification.
  // `config_proto` configures params and thresholds for the classifier,
  //                wraps the model's EligibilitySpec.
  // `result_handler` is a mojom interface necessary to send back the results.
  StartVisualClassification(
    mojo_base.mojom.ReadOnlyFile visual_model_file,
    mojo_base.mojom.ProtoWrapper? config,
    pending_remote<VisualSuggestionsResultHandler> result_handler);
};

// Interface used by renderer to request model and metadata from the
// VisualQuerySuggestionsService running in the browser process.
// This interface is used in cases where we want image classification to
// be triggered on the renderer process rather than browser process.
interface VisualSuggestionsModelProvider {
  // Requests the model bytes with metadata.
  // `visual_model_file` is the file pointer to the model used for
  // classification.
  // `config_proto` wraps the model's EligibilitySpec.
  GetModelWithMetadata() => (
    mojo_base.mojom.ReadOnlyFile visual_model_file,
    mojo_base.mojom.ProtoWrapper? config);
};
chromium/chrome/common/companion/visual_query.mojom
