video_track_adapter.cc | Explore in Territory

// Copyright 2014 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/351564777): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "third_party/blink/renderer/modules/mediastream/video_track_adapter.h"

#include <algorithm>
#include <cmath>
#include <limits>
#include <memory>
#include <optional>
#include <string>
#include <utility>

#include "base/containers/flat_map.h"
#include "base/functional/bind.h"
#include "base/location.h"
#include "base/metrics/histogram_macros.h"
#include "base/sequence_checker.h"
#include "base/strings/string_number_conversions.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/task/single_thread_task_runner.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
#include "media/base/limits.h"
#include "media/base/video_util.h"
#include "third_party/blink/public/common/features.h"
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/renderer/modules/mediastream/video_track_adapter_settings.h"
#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_gfx.h"
#include "third_party/blink/renderer/platform/wtf/functional.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"

namespace WTF {

// Template specializations of [1], needed to be able to pass WTF callbacks
// that have VideoTrackAdapterSettings or gfx::Size parameters across threads.
//
// [1] third_party/blink/renderer/platform/wtf/cross_thread_copier.h.
template <>
struct CrossThreadCopier<blink::VideoTrackAdapterSettings>
    : public CrossThreadCopierPassThrough<blink::VideoTrackAdapterSettings> { … };

}  // namespace WTF

namespace blink {

namespace {

// Amount of frame intervals to wait before considering the source as muted, for
// the first frame and under normal conditions, respectively. First frame might
// take longer to arrive due to source startup.
const float kFirstFrameTimeoutInFrameIntervals = …;
const float kNormalFrameTimeoutInFrameIntervals = …;

// |kMaxDeltaDeviationFactor| is used to determine |max_delta_deviation_| which
// specifies the allowed deviation from |target_delta_| before dropping a frame.
// It's set to 20% to be aligned with the previous logic in this file.
constexpr float kMaxDeltaDeviationFactor = …;

// If the delta between two frames is bigger than this, we will consider it to
// be invalid and reset the fps calculation.
constexpr base::TimeDelta kMaxTimeBetweenFrames = …;

constexpr base::TimeDelta kFrameRateChangeInterval = …;
const double kFrameRateChangeRate = …;
constexpr base::TimeDelta kFrameRateUpdateInterval = …;

struct ComputedSettings { … };

int ClampToValidDimension(int dimension) { … }

void ComputeFrameRate(const base::TimeDelta& frame_timestamp,
                      double* frame_rate,
                      base::TimeDelta* prev_frame_timestamp) { … }

// Controls the frequency of settings updates based on frame rate changes.
// Returns |true| if over the last second the computed frame rate is
// consistently kFrameRateChangeRate different than the last reported value,
// or if there hasn't been any update in the last
// kFrameRateUpdateIntervalInSeconds seconds.
bool MaybeUpdateFrameRate(ComputedSettings* settings) { … }

VideoTrackAdapterSettings ReturnSettingsMaybeOverrideMaxFps(
    const VideoTrackAdapterSettings& settings) { … }

}  // anonymous namespace

// VideoFrameResolutionAdapter is created on and lives on the video task runner.
// It does the resolution adaptation and delivers frames to all registered
// tracks on the video task runner. All method calls must be on the video task
// runner.
class VideoTrackAdapter::VideoFrameResolutionAdapter
    : public WTF::ThreadSafeRefCounted<VideoFrameResolutionAdapter> { … };

VideoTrackAdapter::VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
    scoped_refptr<base::SingleThreadTaskRunner> reader_task_runner,
    const VideoTrackAdapterSettings& settings,
    base::WeakPtr<MediaStreamVideoSource> media_stream_video_source)
    : … { … }

VideoTrackAdapter::VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallbacks(
    const MediaStreamVideoTrack* track,
    VideoCaptureDeliverFrameInternalCallback frame_callback,
    VideoCaptureNotifyFrameDroppedInternalCallback
        notify_frame_dropped_callback,
    DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
    VideoCaptureSubCaptureTargetVersionInternalCallback
        sub_capture_target_version_callback,
    VideoTrackSettingsInternalCallback settings_callback,
    VideoTrackFormatInternalCallback format_callback) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallbacks(
    const MediaStreamVideoTrack* track) { … }

VideoTrackAdapter::VideoFrameResolutionAdapter::VideoTrackCallbacks
VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveAndGetCallbacks(
    const MediaStreamVideoTrack* track) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
    scoped_refptr<media::VideoFrame> video_frame,
    const base::TimeTicks& estimated_capture_time,
    bool is_device_rotated) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverEncodedVideoFrame(
    scoped_refptr<EncodedVideoFrame> frame,
    base::TimeTicks estimated_capture_time) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::
    NewSubCaptureTargetVersionOnVideoTaskRunner(
        uint32_t sub_capture_target_version) { … }

bool VideoTrackAdapter::VideoFrameResolutionAdapter::SettingsMatch(
    const VideoTrackAdapterSettings& settings) const { … }

bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame(
    scoped_refptr<media::VideoFrame> video_frame,
    const base::TimeTicks& estimated_capture_time) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::OnFrameDropped(
    media::VideoCaptureFrameDropReason reason) { … }

bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
    const media::VideoFrame& frame,
    float source_frame_rate,
    media::VideoCaptureFrameDropReason* reason) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTrackSettings(
    const VideoTrackSettingsInternalCallback& settings_callback,
    const media::VideoFrame& frame) { … }
void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTracksFormat(
    const media::VideoFrame& frame) { … }

void VideoTrackAdapter::VideoFrameResolutionAdapter::ResetFrameRate() { … }

VideoTrackAdapter::VideoTrackAdapter(
    scoped_refptr<base::SequencedTaskRunner> video_task_runner,
    base::WeakPtr<MediaStreamVideoSource> media_stream_video_source)
    : … { … }

VideoTrackAdapter::~VideoTrackAdapter() { … }

void VideoTrackAdapter::AddTrack(
    const MediaStreamVideoTrack* track,
    VideoCaptureDeliverFrameCB frame_callback,
    VideoCaptureNotifyFrameDroppedCB notify_frame_dropped_callback,
    EncodedVideoFrameCB encoded_frame_callback,
    VideoCaptureSubCaptureTargetVersionCB sub_capture_target_version_callback,
    VideoTrackSettingsCallback settings_callback,
    VideoTrackFormatCallback format_callback,
    const VideoTrackAdapterSettings& settings) { … }

void VideoTrackAdapter::AddTrackOnVideoTaskRunner(
    const MediaStreamVideoTrack* track,
    VideoCaptureDeliverFrameInternalCallback frame_callback,
    VideoCaptureNotifyFrameDroppedInternalCallback
        notify_frame_dropped_callback,
    DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
    VideoCaptureSubCaptureTargetVersionInternalCallback
        sub_capture_target_version_callback,
    VideoTrackSettingsInternalCallback settings_callback,
    VideoTrackFormatInternalCallback format_callback,
    const VideoTrackAdapterSettings& settings) { … }

void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) { … }

void VideoTrackAdapter::ReconfigureTrack(
    const MediaStreamVideoTrack* track,
    const VideoTrackAdapterSettings& settings) { … }

void VideoTrackAdapter::StartFrameMonitoring(
    double source_frame_rate,
    const OnMutedCallback& on_muted_callback) { … }

void VideoTrackAdapter::StopFrameMonitoring() { … }

void VideoTrackAdapter::SetSourceFrameSize(const gfx::Size& source_frame_size) { … }

bool VideoTrackAdapter::CalculateDesiredSize(
    bool is_rotated,
    const gfx::Size& original_input_size,
    const VideoTrackAdapterSettings& settings,
    gfx::Size* desired_size) { … }

void VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner(
    OnMutedInternalCallback on_muted_callback,
    double source_frame_rate) { … }

void VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner() { … }

void VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner(
    const gfx::Size& source_frame_size) { … }

void VideoTrackAdapter::RemoveTrackOnVideoTaskRunner(
    const MediaStreamVideoTrack* track) { … }

void VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner(
    const MediaStreamVideoTrack* track,
    const VideoTrackAdapterSettings& settings) { … }

void VideoTrackAdapter::DeliverFrameOnVideoTaskRunner(
    scoped_refptr<media::VideoFrame> video_frame,
    base::TimeTicks estimated_capture_time) { … }

void VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner(
    scoped_refptr<EncodedVideoFrame> frame,
    base::TimeTicks estimated_capture_time) { … }

void VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner(
    media::VideoCaptureFrameDropReason reason) { … }

void VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner(
    uint32_t sub_capture_target_version) { … }

void VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner() { … }

}  // namespace blink
chromium/third_party/blink/renderer/modules/mediastream/video_track_adapter.cc
