syntax = "proto2";
option optimize_for = LITE_RUNTIME;
package mri;
// The output of the media analytics process. Implicitly tied to the
// MediaPerception dictionary defined in Chromium source at
// src/extensions/common/api/media_perception_private.idl for the
// Chromium mediaPerceptionPrivate API. This .proto needs to be compatible
// with the version used in the binary checked into the Chromebox For
// Meetings private overlay.
//
// This message is packaged by the graph runner when a PerceptionSample
// or array of PerceptionSamples comes out of the graph.
message MediaPerception {
// The timestamp attached when this data originated from the analysis process.
optional uint64 timestamp = 1; // In milliseconds since Epoch.
// A single FramePerception message or array of perceptions (if reporting the
// results from multiple frames).
repeated FramePerception frame_perception = 2;
// A single AudioPerception message or array of audio perceptions (if
// reporting the results from multiple audio frames).
repeated AudioPerception audio_perception = 3;
// A single AudioVisualPerception message or array of audio-visual
// perceptions.
repeated AudioVisualPerception audio_visual_perception = 4;
// Stores metadata such as version of media perception features.
optional Metadata metadata = 5;
}
// Stores metadata such as version of media perception features.
message Metadata {
optional string visual_experience_controller_version = 1;
}
// Used to transmit a history of image frames and their associated annotations.
// This is accumulated over time by the graph runner.
message Diagnostics {
repeated PerceptionSample perception_sample = 1;
}
message VideoStreamParam {
// Identifies the video stream described by these parameters.
optional string id = 1;
// Frame width in pixels.
optional int32 width = 2;
// Frame height in pixels.
optional int32 height = 3;
// The frame rate at which this video stream would be processed.
optional float frame_rate = 4;
}
message State {
enum Status {
STATUS_UNSPECIFIED = 0; // Unused required default value for Proto enums.
UNINITIALIZED = 1; // Media analytics working on loading configuration.
STARTED = 2; // Analysis process running but not recieving frames.
RUNNING = 3; // Analysis process running and injesting frames.
SUSPENDED = 4; // Media analytics process waiting to be started.
RESTARTING = 5; // Media analytics process should be restarted.
STOPPED = 6; // Media analytics process should be stopped.
}
// Note: RUNNING and SUSPENDED are the only two states which should be sent to
// SetState.
optional Status status = 1;
// Device context so that the media analytics process can better select the
// right video device to open.
optional string device_context = 2;
// A list of video streams processed by the analytics process.
repeated VideoStreamParam video_stream_param = 3;
// Media analytics configuration. It can only be used when setting state to
// RUNNING.
optional string configuration = 4;
// Corners and aspect ratio of the whiteboard in the image frame. Should only
// be set when setting state to running and configuration to whiteboard.
optional Whiteboard whiteboard = 5;
enum Feature {
// UNSET is not a real feature value.
FEATURE_UNSET = 0;
FEATURE_AUTOZOOM = 1;
FEATURE_HOTWORD_DETECTION = 2;
FEATURE_OCCUPANCY_DETECTION = 3;
FEATURE_EDGE_EMBEDDINGS = 4;
FEATURE_SOFTWARE_CROPPING = 5;
}
// A list of enabled media perception features.
repeated Feature features = 6;
message NamedTemplateArgument {
optional string name = 1;
oneof value {
string str = 2;
double num = 3;
}
}
// An optional list of template arguments to be substituted at
// run-time. Each argument present in this list will be set to the
// specified values, others will not be changed. Furthermore, nested
// arguments (dictionaries) are not supported.
// CAUTION: These template args can be used to overwrite the
// Feature flags defined above since feature flags are implemented
// as numeric template args.
repeated NamedTemplateArgument named_template_arguments = 7;
}
// This is the output of the MediaPerceptionSinkCalculator.
message PerceptionSample {
optional FramePerception frame_perception = 1;
// The image frame data associated with the frame perception.
optional ImageFrame image_frame = 2;
optional AudioPerception audio_perception = 3;
optional AudioVisualPerception audio_visual_perception = 4;
// Stores metadata such as version of media perception features.
optional Metadata metadata = 5;
}
// Perception results based on both audio and video inputs.
message AudioVisualPerception {
// A timestamp in microseconds attached when this message was generated.
optional uint64 timestamp_us = 1;
// Human presence detection results.
optional AudioVisualHumanPresenceDetection
audio_visual_human_presence_detection = 2;
}
// Detection of human presence based on both audio and video inputs.
message AudioVisualHumanPresenceDetection {
// Indicates a probability in [0, 1] interval that a human is present.
optional double human_presence_likelihood = 1;
}
// Audio perception results for an audio frame.
message AudioPerception {
// A timestamp in microseconds attached when this message was generated.
optional uint64 timestamp_us = 1;
// Audio localization results for an audio frame.
optional AudioLocalization audio_localization = 2;
// Human presence detection results for an audio frame.
optional AudioHumanPresenceDetection audio_human_presence_detection = 3;
// Hotword detection results.
optional HotwordDetection hotword_detection = 4;
}
// Detection of hotword in the audio stream.
message HotwordDetection {
enum Type {
UNKNOWN_TYPE = 0;
OK_GOOGLE = 1;
}
message Hotword {
// Unique identifier for the hotword instance. Note that a single hotword
// instance can span more than one audio frame. In that case a single
// hotword instance can be reported in multiple Hotword or HotwordDetection
// messages. Hotword messages associated with the same hotword instance will
// have the same id.
optional uint64 id = 1;
// Indicates the type of this hotword.
optional Type type = 2;
// Id of the audio frame in which the hotword was detected.
optional uint64 frame_id = 3;
// Indicates the start time of this hotword in the audio frame.
optional uint64 start_timestamp_ms = 4;
// Indicates the end time of this hotword in the audio frame.
optional uint64 end_timestamp_ms = 5;
// Indicates a probability in [0, 1] interval that this hotword is present
// in the audio frame.
optional float confidence = 6;
}
repeated Hotword hotwords = 1;
}
// An estimate of the direction that the sound is coming from.
message AudioLocalization {
// An angle in radians in the horizontal plane. It roughly points to the peak
// in the probability distribution of azimuth defined below.
optional double azimuth_radians = 1;
// A probability distribution for the current snapshot in time that shows the
// likelihood of a sound source being at a particular azimuth. For example,
// azimuthScores = [0.1, 0.2, 0.3, 0.4] means that the probability that the
// sound is coming from an azimuth of 0, pi/2, pi, 3*pi/2 is 0.1, 0.2, 0.3 and
// 0.4, respectively.
repeated double azimuth_scores = 2;
}
// Detection of human presence close to the microphone.
message AudioHumanPresenceDetection {
// Indicates a probability in [0, 1] interval that a human has caused a sound
// close to the microphone.
optional double human_presence_likelihood = 1;
// Estimate of the noise spectrogram.
optional AudioSpectrogram noise_spectrogram = 2;
// Spectrogram of an audio frame.
optional AudioSpectrogram frame_spectrogram = 3;
}
// Spectrogram of an audio frame.
message AudioSpectrogram {
repeated double values = 1;
}
// This message stores the image frame along with the meta data.
message ImageFrame {
enum Format {
FORMAT_UNSPECIFIED = 0; // Unused required default value for Proto enums.
RGB = 1; // Raw rgb image.
PNG = 2; // PNG image.
JPEG = 3; // JPEG image.
}
optional int32 width = 1;
optional int32 height = 2;
optional Format format = 3;
optional int32 data_length = 4;
optional bytes pixel_data = 5;
}
// The set of computer vision metadata for an image frame.
message FramePerception {
optional uint64 frame_id = 1;
optional uint32 frame_width_in_px = 2;
optional uint32 frame_height_in_px = 3;
// The timestamp associated with the frame (when it enters the graph).
optional uint64 timestamp = 4;
// The list of entities detected for this frame.
repeated Entity entity = 5;
// Latency measurement for a list of packet streams in drishti graph.
repeated PacketLatency packet_latency = 6;
// Human presence detection results for a video frame.
optional VideoHumanPresenceDetection video_human_presence_detection = 7;
enum PerceptionType {
UNKNOWN_TYPE = 0;
FACE_DETECTION = 1;
MOTION_DETECTION = 2;
PERSON_DETECTION = 3;
}
// Indicates what types of frame perception were run.
repeated PerceptionType perception_types = 8;
}
// Detection of human presence close to the camera.
message VideoHumanPresenceDetection {
// Indicates a probability in [0, 1] interval that a human is present in the
// video frame.
optional double human_presence_likelihood = 1;
// Indicates a probability in [0, 1] interval that motion has been detected
// in the video frame.
optional double motion_detected_likelihood = 2;
// Type of lighting conditions.
enum LightCondition {
UNSPECIFIED = 0;
// No noticeable change occurred.
NO_CHANGE = 1;
// Light was switched on in the room.
TURNED_ON = 2;
// Light was switched off in the room.
TURNED_OFF = 3;
// Light gradually got dimmer (for example, due to a sunset).
DIMMER = 4;
// Light gradually got brighter (for example, due to a sunrise).
BRIGHTER = 5;
// Black frame detected - the current frame contains only noise.
BLACK_FRAME = 6;
}
// Indicates lighting condition in the video frame.
optional LightCondition light_condition = 3;
// Indicates a probability in [0, 1] interval that light condition value is
// correct.
optional double light_condition_likelihood = 4;
}
message Entity {
// A unique id associated with the detected entity, which can be used to track
// the entity over time.
optional uint32 id = 1;
enum EntityType {
UNSPECIFIED = 0;
FACE = 1;
PERSON = 2;
MOTION_REGION = 3;
LABELED_REGION = 4;
}
optional EntityType type = 2;
// Minimum box, which captures entire detected entity.
optional BoundingBox bounding_box = 3;
// A value for the quality of this detection.
optional float confidence = 4;
// Perpendicular distance (depth) from the camera plane to the entity.
optional Distance depth = 5;
// String label for this entity.
optional string label = 6;
}
message BoundingBox {
// The points that define the corners of a bounding box.
optional Point top_left = 1;
optional Point bottom_right = 2;
// Indicates whether or not these coordinates are normalized to values between
// 0 and 1.
optional bool normalized = 3 [default = false];
}
message PacketLatency {
// An identifier label for the packet.
optional string label = 1;
// Delay in microseconds with respect to a reference packet.
optional uint64 latency_usec = 2;
}
message Point {
// x represents the horizontal distance from the top left corner of the image
// to the point.
optional float x = 1;
// y represents the vertical distance from the top left corner of the image to
// the point.
optional float y = 2;
}
// Generic message object to encapsulate a distance magnitude and units.
message Distance {
enum DistanceUnits {
UNITS_UNSPECIFIED = 0;
METERS = 1;
PIXELS = 2;
}
optional DistanceUnits units = 1;
optional float magnitude = 2;
}
// The parameters for a whiteboard in the image frame.
message Whiteboard {
// The top left corner of the whiteboard in the image frame.
optional Point top_left = 1;
// The top right corner of the whiteboard in the image frame.
optional Point top_right = 2;
// The bottom left corner of the whiteboard in the image frame.
optional Point bottom_left = 3;
// The bottom right corner of the whiteboard in the image frame.
optional Point bottom_right = 4;
// The physical aspect ratio of the whiteboard.
optional float aspect_ratio = 5;
}
Codebase Browser
chromium