handwriting_recognizer.mojom | Explore in Territory

// Copyright 2020 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Next MinVersion: 3

// Datatypes and interfaces of handwriting recognition API.

// NOTE: This mojom exists in two places and must be kept in sync:
//       Chromium:  //chromeos/services/machine_learning/public/mojom/
//       Chrome OS: src/platform2/ml/mojom/
//       Note: Other repos downstream of Chromium might also use this mojom.
// Example: A backwards-compatible mojom change (and corresponding
// implementation change) can be made in Chrome OS first, then replicated to the
// clients (Chromium, other downstream repos) later.
// Use //chromeos/services/machine_learning/public/mojom/roll_mojoms.sh to help
// replicate Chrome OS-side changes over to Chromium.

module chromeos.machine_learning.mojom;

import "mojo/public/mojom/base/time.mojom";

// A single point in an ink stroke.
[Stable]
struct InkPoint {
  // (x, y) coordinates of the point. The upper-left corner of the writing area
  // is (0, 0). The positive x-axis points to the right. The positive y-axis
  // points down.
  float x;
  float y;
  // Optional TimeDelta from the point was captured relative to the beginning of
  // the ink. So first point of a request should have t=0.
  mojo_base.mojom.TimeDelta? t;
};

// A single ink stroke.
[Stable]
struct InkStroke {
  // The set of points representing the stroke.
  array<InkPoint> points;
};

// The writing guide shown to the user during input.
[Stable]
struct WritingGuide {
   // Size of the writing area. The writing area must have (0, 0) at the top
   // left.
   // width is the max value of x-axis (to the right) and height is the max
   // value of y-axis (to the bottom).
   float width;
   float height;
};

// Options that will influence the recognition output.
[Stable]
struct RecognitionContext {
  // The writing guide shown to the user when the ink was captured.
  WritingGuide? writing_guide;
  // The context before the written text.
  string? pre_context;
};

// Defines a handwriting recognition query.
[Stable]
struct HandwritingRecognitionQuery {
  // The set of ink strokes to be recognized.
  array<InkStroke> ink;
  // The optional context in which the ink was generated.
  RecognitionContext? context;
  // Maximum number of results to be returned.
  uint32 max_num_results;
  // If true, returns the segmentation results. This will make the response
  // struct much larger.
  bool return_segmentation;
};

// Represents a contiguous range of ink. Start and end strokes / points are
// inclusive.
[Stable]
struct HandwritingRecognizerInkRange {
  // Zero-based start- and end-stroke indices in the ink strokes.
  uint32 start_stroke;
  uint32 end_stroke;
  // Zero-based start- and end-point indices with the start / end strokes.
  uint32 start_point;
  uint32 end_point;
};

// A segment defines the substrokes that are associated with a substring of the
// recognized text.
[Stable]
struct HandwritingRecognizerSegment {
  // The substring of the recognized text represented by this segment.
  string sublabel;
  // The ink ranges represented by this segment.
  array<HandwritingRecognizerInkRange> ink_ranges;
};

// The ink segmentation information.
[Stable]
struct HandwritingRecognizerSegmentation {
  // The grouping of the cut strokes into characters.
  array<HandwritingRecognizerSegment> segments;
};

// One possible candidate from the handwriting recognition.
[Stable]
struct HandwritingRecognizerCandidate {
  // The recognized text.
  string text;
  // Scores will most often correspond to -log(p(c)) or a scaling thereof, so
  // lower scores represent higher confidence.
  float score;
  // The stroke segmentation that was used to generate the result.
  HandwritingRecognizerSegmentation? segmentation;
};

// The handwriting recognition response.
[Stable]
struct HandwritingRecognizerResult {
  // Status of the recognition response.
  [Stable, Extensible]
  enum Status {
    OK = 0,
    ERROR = 1,
  };
  Status status;

  // The recognition candidates with additional alternatives, sorted by lower
  // score first (lower score present higher confidence).
  array<HandwritingRecognizerCandidate> candidates;
};

// The specification of a handwriting recognizer.
[Stable]
struct HandwritingRecognizerSpec {
  // The language the recognizer will handle. Only "en" (for english) and
  // "gesture_in_context" (for gesture) are supported.
  string language;

  // Path to the language pack to use (downloaded by Chrome via DLC).
  // If unset, ml-service will try to find the model from rootfs based on the
  // `language` field above.
  [MinVersion=1] string? language_pack_path;

  // Path to already-installed handwriting recognition library (downloaded by
  // chrome via DLC).
  // If unset, ml-service will try to find the library from rootfs.
  [MinVersion=2] string? library_dlc_path;
};

// The mojom interface for performing the recognition of handwritten text.
// Next ordinal: 1
[Stable]
interface HandwritingRecognizer {
  // Performs handwriting recognition on a set of ink strokes, and returns a set
  // of alternative recognition results.
  Recognize@0(HandwritingRecognitionQuery query) =>
      (HandwritingRecognizerResult result);
};

// Enum indicates the result of LoadHandwritingModel in mlservice.
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
// Keep this enum in sync with MachineLearningServiceHandwritingModelResultEvent
// in tools/metrics/histograms/metadata/cros_ml/enums.xml.
[Stable, Extensible]
enum LoadHandwritingModelResult {
  OK = 0,

  // Deprecated enum values.
  DEPRECATED_MODEL_SPEC_ERROR = 1,
  LOAD_MODEL_ERROR = 2,

  // If both USE.ondevice_handwriting and USE.ondevice_handwriting_dlc
  // are not defined. (checked in chrome)
  FEATURE_NOT_SUPPORTED_ERROR = 3,

  // If the required language is not in an allowlist, "en" and
  // "gesture_in_context" for now. (checked in chrome)
  LANGUAGE_NOT_SUPPORTED_ERROR = 4,

  // If the user goes to chrome://flag page and manually disable
  // handwriting or handwriting_dlc. (checked in chrome)
  FEATURE_DISABLED_BY_USER = 5,

  // if the USE.ondevice_handwriting_dlc is enabled, but dlc is not
  // on the device. (checked in chrome)
  DLC_DOES_NOT_EXIST = 6,

  // If the dlc is on the device, but Install returns an error.
  // (checked in chrome).
  DLC_INSTALL_ERROR = 7,

  // If dlc is installed successfully, but GetDlcState returns an error.
  // (checked inside MachineLearningServiceImpl)
  DLC_GET_PATH_ERROR = 8,

  // if loading libhandwriting.so fails. (checked inside HandWritingLibrary)
  LOAD_NATIVE_LIB_ERROR = 9,

  // if getting the function pointers fails. (checked inside HandWritingLibrary)
  LOAD_FUNC_PTR_ERROR = 10,

  // If loading model files fails. (checked inside HandWritingLibrary)
  LOAD_MODEL_FILES_ERROR = 11,
};
chromium/chromeos/services/machine_learning/public/mojom/handwriting_recognizer.mojom
