chromium/third_party/mediapipe/src/mediapipe/graphs/media_sequence/tvl1_flow_and_rgb_from_file.pbtxt

# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Convert the string input into a decoded SequenceExample.
node {
  calculator: "StringToSequenceExampleCalculator"
  input_side_packet: "STRING:input_sequence_example"
  output_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
}

# Unpack the data path and clip timing from the SequenceExample.
node {
  calculator: "UnpackMediaSequenceCalculator"
  input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
  output_side_packet: "DATA_PATH:input_video_path"
  output_side_packet: "RESAMPLER_OPTIONS:packet_resampler_options"
  node_options: {
    [type.googleapis.com/mediapipe.UnpackMediaSequenceCalculatorOptions]: {
      base_packet_resampler_options: {
        frame_rate: 25.0
        base_timestamp: 0
      }
    }
  }
}

# Decode the entire video.
node {
  calculator: "OpenCvVideoDecoderCalculator"
  input_side_packet: "INPUT_FILE_PATH:input_video_path"
  output_stream: "VIDEO:decoded_frames"
}

# Extract the subset of frames we want to keep.
node {
  calculator: "PacketResamplerCalculator"
  input_stream: "decoded_frames"
  output_stream: "sampled_frames"
  input_side_packet: "OPTIONS:packet_resampler_options"
}

# Fit the images into the target size.
node: {
  calculator: "ScaleImageCalculator"
  input_stream: "sampled_frames"
  output_stream: "scaled_frames"
  node_options: {
    [type.googleapis.com/mediapipe.ScaleImageCalculatorOptions]: {
      target_height: 256
      preserve_aspect_ratio: true
    }
  }
}

# Shift the the timestamps of packets along a stream.
# With a packet_offset of -1, the first packet will be dropped, the second will
# be output with the timestamp of the first, the third with the timestamp of
# the second, and so on.
node: {
  calculator: "SequenceShiftCalculator"
  input_stream: "scaled_frames"
  output_stream: "shifted_scaled_frames"
  node_options: {
    [type.googleapis.com/mediapipe.SequenceShiftCalculatorOptions]: {
      packet_offset: -1
    }
  }
}

# Join the original input stream and the one that is shifted by one packet.
node: {
  calculator: "PacketInnerJoinCalculator"
  input_stream: "scaled_frames"
  input_stream: "shifted_scaled_frames"
  output_stream: "first_frames"
  output_stream: "second_frames"
}

# Compute the forward optical flow.
node {
  calculator: "Tvl1OpticalFlowCalculator"
  input_stream: "FIRST_FRAME:first_frames"
  input_stream: "SECOND_FRAME:second_frames"
  output_stream: "FORWARD_FLOW:forward_flow"
  max_in_flight: 32
}

# Convert an optical flow to be an image frame with 2 channels (v_x and v_y),
# each channel is quantized to 0-255.
node: {
  calculator: "FlowToImageCalculator"
  input_stream: "forward_flow"
  output_stream: "flow_frames"
  node_options: {
    [type.googleapis.com/mediapipe.FlowToImageCalculatorOptions]: {
      min_value: -20.0
      max_value: 20.0
    }
  }
}

# Encode the optical flow images to store in the SequenceExample.
node {
  calculator: "OpenCvImageEncoderCalculator"
  input_stream: "flow_frames"
  output_stream: "encoded_flow_frames"
  node_options: {
    [type.googleapis.com/mediapipe.OpenCvImageEncoderCalculatorOptions]: {
      quality: 100
    }
  }
}

# Encode the rgb images to store in the SequenceExample.
node {
  calculator: "OpenCvImageEncoderCalculator"
  input_stream: "scaled_frames"
  output_stream: "encoded_frames"
  node_options: {
    [type.googleapis.com/mediapipe.OpenCvImageEncoderCalculatorOptions]: {
      quality: 100
    }
  }
}

# Store the images in the SequenceExample.
node {
  calculator: "PackMediaSequenceCalculator"
  input_stream: "IMAGE:encoded_frames"
  input_stream: "FORWARD_FLOW_ENCODED:encoded_flow_frames"
  input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
  output_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
}

# Serialize the SequenceExample to a string for storage.
node {
  calculator: "StringToSequenceExampleCalculator"
  input_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
  output_side_packet: "STRING:output_sequence_example"
}

num_threads: 32