chromium/third_party/blink/renderer/platform/peerconnection/rtc_video_decoder_stream_adapter.cc

// Copyright 2020 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "third_party/blink/renderer/platform/peerconnection/rtc_video_decoder_stream_adapter.h"

#include <algorithm>
#include <functional>
#include <utility>

#include "base/atomic_ref_count.h"
#include "base/containers/circular_deque.h"
#include "base/feature_list.h"
#include "base/functional/callback_helpers.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/synchronization/waitable_event.h"
#include "base/task/sequenced_task_runner.h"
#include "base/threading/thread_restrictions.h"
#include "base/time/time.h"
#include "base/trace_event/base_tracing.h"
#include "build/build_config.h"
#include "build/chromeos_buildflags.h"
#include "media/base/media_log.h"
#include "media/base/media_switches.h"
#include "media/base/media_util.h"
#include "media/base/overlay_info.h"
#include "media/base/platform_features.h"
#include "media/base/video_types.h"
#include "media/renderers/default_decoder_factory.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "media/video/video_decode_accelerator.h"
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/renderer/platform/peerconnection/rtc_video_decoder_adapter.h"
#include "third_party/blink/renderer/platform/peerconnection/rtc_video_decoder_fallback_recorder.h"
#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
#include "third_party/blink/renderer/platform/webrtc/webrtc_video_frame_adapter.h"
#include "third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_std.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
#include "third_party/webrtc/api/video/video_frame.h"
#include "third_party/webrtc/api/video_codecs/vp9_profile.h"
#include "third_party/webrtc/modules/video_coding/codecs/h264/include/h264.h"
#include "third_party/webrtc/rtc_base/ref_count.h"
#include "third_party/webrtc/rtc_base/ref_counted_object.h"
#include "ui/gfx/color_space.h"

namespace WTF {

template <>
struct CrossThreadCopier<media::VideoDecoderConfig>
    : public CrossThreadCopierPassThrough<media::VideoDecoderConfig> { … };

}  // namespace WTF

namespace blink {

namespace {

// Any reasonable size, will be overridden by the decoder anyway.
constexpr gfx::Size kDefaultSize(640, 480);

// How many buffers are we willing to receive while decoding is paused, before
// we give up and fall back to software?  This is not used during normal
// decoding, only when we believe that the decoder is in a state where it is not
// ready to decode.  Our goal during this state is to keep the queue of pending
// buffers very small by requesting a keyframe.
//
// Also note that this value still counts buffers that were dropped in favor of
// a more recent keyframe; it represents the maximum number of calls to
// Decode before the decoder is ready to do work.
constexpr int32_t kMaxFramesWhilePausedBeforeFallback = …;

// Max number of non-keyframes we'll queue without a decoder before we request a
// keyframe to replace them while the decoder is paused.  Previously queued
// frames will be discarded, to prevent a lot of old frames from being dumped
// onto the newly-unpaused decoder, that are probably stale anyway.
constexpr int32_t kMaxKeyFrameIntervalWhilePaused = …;

// Maximum number of buffers that we will queue in the decoder stream during
// normal operation.  It includes all buffers that we have not gotten an output
// for.  "Normal operation" means that we believe that the decoder is trying to
// drain the queue.  During init and reset, for example, we don't expect it.
// See above for constants used while the decoder is in one of those states.
//
// If we go over this value, then we'll reset the decoder and request a keyframe
// to try to catch up.
//
// Note: This value is chosen to be Ludicrously High(tm), so that we can see
// where reasonable limits should be via UMA.
constexpr int32_t kMaxPendingBuffers = …;

// Absolute maximum number of pending buffers.  If, at any time while we have
// an unpaused decoder, we believe that there are this many decodes in-flight
// when a new decode request arrives, we will fall back to software decoding.
//
// This value is not used while decoding is paused.  Instead, we use
// `kMaxFramesWhilePausedBeforeFallback`, since we might have different
// tolerances for "during init/reset" and "during normal decode".
//
// Note: This value is chosen to be Ludicrously High(tm), so that we can see
// where reasonable limits should be via UMA.  Changing this changes UMA, so
// probably don't.
constexpr int32_t kAbsoluteMaxPendingBuffers = …;

// Name we'll report for hardware decoders.
constexpr const char* kExternalDecoderName = …;

// Number of RTCVideoDecoder instances right now that have started decoding.
class DecoderCounter { … };

DecoderCounter* GetDecoderCounter() { … }

void RecordInitializationLatency(base::TimeDelta latency) { … }

}  // namespace

// static
constexpr gfx::Size RTCVideoDecoderStreamAdapter::kMinResolution;

// DemuxerStream implementation that forwards DecoderBuffer from some other
// source (i.e., VideoDecoder::Decode).
class RTCVideoDecoderStreamAdapter::InternalDemuxerStream
    : public media::DemuxerStream { … };

// static
std::unique_ptr<RTCVideoDecoderStreamAdapter>
RTCVideoDecoderStreamAdapter::Create(
    media::GpuVideoAcceleratorFactories* gpu_factories,
    base::WeakPtr<media::DecoderFactory> decoder_factory,
    scoped_refptr<base::SequencedTaskRunner> media_task_runner,
    const gfx::ColorSpace& render_color_space,
    const webrtc::SdpVideoFormat& format) { … }

RTCVideoDecoderStreamAdapter::RTCVideoDecoderStreamAdapter(
    media::GpuVideoAcceleratorFactories* gpu_factories,
    base::WeakPtr<media::DecoderFactory> decoder_factory,
    scoped_refptr<base::SequencedTaskRunner> media_task_runner,
    const gfx::ColorSpace& render_color_space,
    const media::VideoDecoderConfig& config,
    const webrtc::SdpVideoFormat& format)
    : … { … }

RTCVideoDecoderStreamAdapter::~RTCVideoDecoderStreamAdapter() { … }

void RTCVideoDecoderStreamAdapter::InitializeOrReinitializeSync() { … }

bool RTCVideoDecoderStreamAdapter::Configure(const Settings& settings) { … }

void RTCVideoDecoderStreamAdapter::AttemptLogInitializationState_Locked() { … }

int32_t RTCVideoDecoderStreamAdapter::Decode(
    const webrtc::EncodedImage& input_image,
    bool missing_frames,
    int64_t render_time_ms) { … }

int32_t RTCVideoDecoderStreamAdapter::RegisterDecodeCompleteCallback(
    webrtc::DecodedImageCallback* callback) { … }

int32_t RTCVideoDecoderStreamAdapter::Release() { … }

webrtc::VideoDecoder::DecoderInfo RTCVideoDecoderStreamAdapter::GetDecoderInfo()
    const { … }

void RTCVideoDecoderStreamAdapter::InitializeOnMediaThread(
    const media::VideoDecoderConfig& config,
    InitCB init_cb) { … }

void RTCVideoDecoderStreamAdapter::OnInitializeDone(base::TimeTicks start_time,
                                                    bool success) { … }

void RTCVideoDecoderStreamAdapter::DecodeOnMediaThread(
    std::unique_ptr<PendingBuffer> pending_buffer) { … }

void RTCVideoDecoderStreamAdapter::OnFrameReady(
    media::VideoDecoderStream::ReadResult result) { … }

void RTCVideoDecoderStreamAdapter::AttemptRead() { … }

bool RTCVideoDecoderStreamAdapter::ShouldReinitializeForSettingHDRColorSpace(
    const webrtc::EncodedImage& input_image) const { … }

void RTCVideoDecoderStreamAdapter::ResetOnMediaThread() { … }

void RTCVideoDecoderStreamAdapter::OnResetCompleteOnMediaThread() { … }

void RTCVideoDecoderStreamAdapter::AdjustQueueLength_Locked() { … }

void RTCVideoDecoderStreamAdapter::ShutdownOnMediaThread() { … }

void RTCVideoDecoderStreamAdapter::OnDecoderChanged(
    media::VideoDecoder* decoder) { … }

void RTCVideoDecoderStreamAdapter::RecordMaxInFlightDecodesLockedOnMedia() { … }

void RTCVideoDecoderStreamAdapter::RestartDecoderStreamOnMedia() { … }

int32_t RTCVideoDecoderStreamAdapter::FallBackToSoftware_Locked() { … }

bool RTCVideoDecoderStreamAdapter::IsDecodingPaused_Locked() const { … }

}  // namespace blink