// Copyright 2021 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
module handwriting.mojom;
// https://github.com/WICG/handwriting-recognition/blob/main/explainer.md
// CrOS's handwriting service's API, defined in
// //chromeos/services/machine_leaning/public/mojom and mirrored into platform2,
// is largely derived from the types and interfaces defined below.
//
// The feature query APIs are not plumbed through, but otherwise, new
// fields/methods likely need to be plumbed through the corresponding CrOS APIs
// as well.
//
// Any changes in overlapping part may need to be kept synced in all these
// three places,
// 1. "chromeos/services/machine_learning/public/mojom/web_platform_handwriting.mojom"
// in chrome repo.
// 2. "platform2/ml/mojom/web_platform_handwriting.mojom" in CrOS repo.
// 3. This file.
import "mojo/public/mojom/base/time.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";
// Represents a single point in a handwriting stroke.
// Corresponds to handwriting_point.idl.
struct HandwritingPoint {
// Represent the horizontal (location.x) and vertical (location.y) location
// of the point.
// The top-left corner coordinate is (location.x=0, location.y=0).
gfx.mojom.PointF location;
// The time elapsed since the starting time (e.g. when the first ink point
// of the drawing is captured).
mojo_base.mojom.TimeDelta? t;
};
// Represents a stroke which is just a series of points.
// Corresponds to handwriting_stroke.idl.
struct HandwritingStroke {
array<HandwritingPoint> points;
};
// Represents a segment of a handwriting stroke in the grapheme detected.
// One `HandwritingDrawingSegment` can only refer one stroke, denoted by
// `stroke_index` which is the index of the stroke in the input stroke arrays
// (i.e., the first parameter of the `HandwritingRecognizer::GetPrediction`
// function).
// The reason we need this struct is that different parts of one single stroke
// can belong to different grapheme detected.
// Corresponds to handwriting_drawing_segment.idl.
struct HandwritingDrawingSegment {
// The index of the corresponding stroke in the input stroke array.
uint32 stroke_index;
// The index of the first point in the stroke that belongs to this drawing
// segment.
uint32 begin_point_index;
// The index of the last point in the stroke that belongs to this drawing
// segment.
uint32 end_point_index;
};
// Represents a segment detected.
// Corresponds to handwriting_segment.idl.
struct HandwritingSegment {
// The string representation of this grapheme.
string grapheme;
// HandwritingPrediction.text.slice(begin_index, end_index) === grapheme
// If the grapheme spans multiple Unicode code points,
// `end_index - begin_index` is greater than 1.
uint32 begin_index;
uint32 end_index;
array<HandwritingDrawingSegment> drawing_segments;
};
// Represents one single prediction result.
// The final prediction output is an array of these.
// Corresponds to handwriting_prediction.idl.
struct HandwritingPrediction {
string text;
array<HandwritingSegment> segmentation_result;
};
// Represents the hints provided to the recognizer for better performance.
// Corresponds to handwriting_hints.idl.
struct HandwritingHints {
// The type of content to be recognized. The recognizer may use these to
// better rank the recognition results. (e.g. "text", "email", "number",
// "per-character").
string recognition_type;
// Identifies how the strokes are captured. (e.g. "touch", "mouse", "pen")
string input_type;
// The text that comes before the handwriting. This can be texts that were
// previously recognized, or were given as the writing context (e.g.
// "Write your name here:"). This is the linguistic context to help
// disambiguate the handwriting (e.g. “Hello world” vs. “Hello word”).
// This could be null if the application doesn't or can't provide context. It
// could be the empty string if a text field is empty, e.g. the beginning of
// input.
string? text_context;
// The maximum number of alternative predictions to generate.
uint32 alternatives;
};
enum HandwritingRecognitionType {
kText,
};
enum HandwritingInputType {
kMouse,
kStylus,
kTouch,
};
// The description of supported hints.
// Corresponds to handwriting_recognizer_query_result.idl
struct HandwritingHintsQueryResult {
array<HandwritingRecognitionType> recognition_type;
array<HandwritingInputType> input_type;
bool text_context;
bool alternatives;
};
// The recognizer query result.
// Corresponds to handwriting_recognizer_query_result.idl
struct QueryHandwritingRecognizerResult {
bool text_alternatives;
bool text_segmentation;
HandwritingHintsQueryResult hints;
};
// Used in querying and creating recognizer.
// Corresponds to handwriting_model_constraint.idl.
struct HandwritingModelConstraint {
// Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
array<string> languages;
};
// Interface for a renderer to use a specific handwriting recognition backend.
// The browser handles the requests and forwards them to the appropriate
// backend.
interface HandwritingRecognizer {
// Does the recognition and outputs the prediction result.
// This is used by IDL API `blink::HandwritingDrawing::getPrediction`.
// The input `strokes` and `hints` should both come from
// `blink::HandwritingDrawing`.
// If the returned `Optional` has no value, it means there is some error in
// recognition. If the returned `Optional` has value but the array is empty,
// it means the recognizer can not recognize anything from the input.
GetPrediction(array<HandwritingStroke> strokes, HandwritingHints hints)
=> (array<HandwritingPrediction>? prediction);
};
// Per-document interface that allows the renderer to ask the browser for
// a specific handwriting recognizer backend, query capabilities, et cetera.
// Corresponds to `navigator_handwriting_recognition_service.idl`.
interface HandwritingRecognitionService {
// Creates a recognizer.
// This is used by IDL API
// `blink::HandwritingRecognitionService::createHandwritingRecognizer`.
CreateHandwritingRecognizer(HandwritingModelConstraint constraint)
=> (CreateHandwritingRecognizerResult result,
pending_remote<HandwritingRecognizer>? handwriting_recognizer);
// Queries the features of the best recognizer satisfying `constraint`.
// `blink::HandwritingRecognitionService::queryHandwritingRecognizer`.
QueryHandwritingRecognizer(HandwritingModelConstraint? constraint)
=> (QueryHandwritingRecognizerResult? result);
};
enum CreateHandwritingRecognizerResult {
kOk,
kError,
// The provided model constraint can't be satisfied (e.g. the language is
// not supported).
kNotSupported,
};
Codebase Browser
chromium