chromium/services/image_annotation/public/mojom/image_annotation.mojom

// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

module image_annotation.mojom;

interface ImageProcessor {
  // Returns the (potentially resized and compressed) pixels for the image,
  // along with its new width and height.
  //
  // TODO(crbug.com/41432508): expand this signature to include arguments when we
  //                         require more sophisticated render-side processing.
  GetJpgImageData() => (array<uint8> bytes, int32 width, int32 height);
};

// The ways in which an annotation request can fail.
enum AnnotateImageError {
  kCanceled,
  kFailure,
  kAdult,
};

// The types of annotations that can be returned.
//
// Logged in metrics - do not reuse or reassign values.
enum AnnotationType {
  kOcr = 1,
  kLabel = 2,
  kCaption = 3,
  kIcon = 4,
};

// One annotation for an image.
struct Annotation {
  AnnotationType type;
  double score;
  string text;
};

union AnnotateImageResult {
  AnnotateImageError error_code;

  // If the union is of this type, |annotations| will be non-empty.
  array<Annotation> annotations;
};

// This interface is used inside Blink renderers and received within the
// ImageAnnotationService to return image annotations.
interface Annotator {
  // Requests a11y annotations (i.e. OCR, labels) for the given image.
  //
  // |source_id| is either the URL for an image, or some non-URL string that
  // uniquely identifies an image (e.g. a hash of image content for a data
  // URI). Source IDs are used to query local and remote caches.
  //
  // |description_language_tag| is the string representation of the BCP-47 code
  // for the language in which descriptions should be generated (or empty for
  // the default language).
  //
  // |result| will contain either the error code specifying how annotation
  // failed, or the annotations extracted from the image.
  //
  // TODO(crbug.com/41432508): expand this signature to include a request
  //                         argument when we support more than one type of
  //                         annotation.
  AnnotateImage(string source_id, string description_language_tag,
                pending_remote<ImageProcessor> image_processor)
                    => (AnnotateImageResult result);
};

// The main interface to the Image Annotation service.
interface ImageAnnotationService {
  // Binds an Annotator endpoint which can be used to annotate images.
  BindAnnotator(pending_receiver<Annotator> receiver);
};