# MediaPipe graph that performs box tracking with TensorFlow Lite on GPU.
# Images coming into and out of the graph.
input_stream: "input_video"
input_stream: "WIDTH:input_width"
input_stream: "HEIGHT:input_height"
input_side_packet: "LABELS_CSV:allowed_labels"
input_side_packet: "MODEL_SCALE:model_scale"
input_side_packet: "MODEL_TRANSFORMATION:model_transformation"
input_side_packet: "TEXTURE:box_texture"
input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
input_side_packet: "ANIMATION_ASSET:box_asset_name"
input_side_packet: "MASK_TEXTURE:obj_texture"
input_side_packet: "MASK_ASSET:obj_asset_name"
output_stream: "output_video"
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:output_video"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Crops the image from the center to the size WIDTHxHEIGHT.
node: {
calculator: "ImageCroppingCalculator"
input_stream: "IMAGE_GPU:throttled_input_video"
output_stream: "IMAGE_GPU:throttled_input_video_3x4"
input_stream: "WIDTH:input_width"
input_stream: "HEIGHT:input_height"
node_options: {
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
border_mode: BORDER_REPLICATE
}
}
}
node {
calculator: "ObjectronGpuSubgraph"
input_stream: "IMAGE_GPU:throttled_input_video_3x4"
input_side_packet: "LABELS_CSV:allowed_labels"
input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
output_stream: "FRAME_ANNOTATION:lifted_objects"
}
# The rendering nodes:
# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly
# on the texture, and 2) a virtual object, which we use as an occlusion mask.
# These models are designed using different tools, so we supply a transformation
# to bring both of them to the Objectron's coordinate system.
# Creates a model matrices for the tracked object given the lifted 3D points.
# This calculator does two things: 1) Estimates object's pose (orientation,
# translation, and scale) from the 3D vertices, and
# 2) bring the object from the objectron's coordinate system to the renderer
# (OpenGL) coordinate system. Since the final goal is to render a mesh file on
# top of the object, we also supply a transformation to bring the mesh to the
# objectron's coordinate system, and rescale mesh to the unit size.
node {
calculator: "AnnotationsToModelMatricesCalculator"
input_stream: "ANNOTATIONS:lifted_objects"
output_stream: "MODEL_MATRICES:model_matrices"
node_options: {
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
# Re-scale the CAD model to the size of a unit box
model_scale: [0.04, 0.04, 0.04]
# Bring the box CAD model to objectron's coordinate system. This
# is equivalent of -pi/2 rotation along the y-axis (right-hand rule):
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY())
model_transformation: [0.0, 0.0, -1.0, 0.0]
model_transformation: [0.0, 1.0, 0.0, 0.0]
model_transformation: [1.0, 0.0, 0.0, 0.0]
model_transformation: [0.0, 0.0, 0.0, 1.0]
}
}
}
# Compute the model matrices for the CAD model of the virtual object, to be used
# as an occlusion mask. The model will be rendered at the exact same location as
# the bounding box.
node {
calculator: "AnnotationsToModelMatricesCalculator"
input_stream: "ANNOTATIONS:lifted_objects"
input_side_packet: "MODEL_SCALE:model_scale"
input_side_packet: "MODEL_TRANSFORMATION:model_transformation"
output_stream: "MODEL_MATRICES:mask_model_matrices"
}
# Render everything together. First we render the 3D bounding box animation,
# then we render the occlusion mask.
node: {
calculator: "GlAnimationOverlayCalculator"
input_stream: "VIDEO:throttled_input_video_3x4"
input_stream: "MODEL_MATRICES:model_matrices"
input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
output_stream: "output_video"
input_side_packet: "TEXTURE:box_texture"
input_side_packet: "ANIMATION_ASSET:box_asset_name"
input_side_packet: "MASK_TEXTURE:obj_texture"
input_side_packet: "MASK_ASSET:obj_asset_name"
node_options: {
[type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] {
aspect_ratio: 0.75
vertical_fov_degrees: 70.
animation_speed_fps: 25
}
}
}
Codebase Browser
chromium