rtc_video_encoder.cc | Explore in Territory

// Copyright 2013 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/351564777): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.h"

#include <memory>
#include <numeric>
#include <vector>

#include "base/command_line.h"
#include "base/containers/contains.h"
#include "base/feature_list.h"
#include "base/functional/callback_helpers.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/memory/raw_ptr.h"
#include "base/memory/unsafe_shared_memory_region.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/numerics/safe_conversions.h"
#include "base/strings/strcat.h"
#include "base/strings/stringprintf.h"
#include "base/synchronization/waitable_event.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/thread_annotations.h"
#include "base/threading/thread_restrictions.h"
#include "base/time/time.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
#include "build/chromeos_buildflags.h"
#include "components/viz/common/resources/shared_image_format_utils.h"
#include "media/base/bitrate.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/media_switches.h"
#include "media/base/media_util.h"
#include "media/base/platform_features.h"
#include "media/base/svc_scalability_mode.h"
#include "media/base/video_bitrate_allocation.h"
#include "media/base/video_frame.h"
#include "media/base/video_util.h"
#include "media/capture/capture_switches.h"
#include "media/media_buildflags.h"
#include "media/mojo/clients/mojo_video_encoder_metrics_provider.h"
#include "media/parsers/h264_parser.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "media/video/video_encode_accelerator.h"
#include "third_party/blink/public/common/buildflags.h"
#include "third_party/blink/public/common/features.h"
#include "third_party/blink/renderer/platform/allow_discouraged_type.h"
#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
#include "third_party/blink/renderer/platform/webrtc/convert_to_webrtc_video_frame_buffer.h"
#include "third_party/blink/renderer/platform/webrtc/webrtc_video_frame_adapter.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_gfx.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_std.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
#include "third_party/blink/renderer/platform/wtf/deque.h"
#include "third_party/blink/renderer/platform/wtf/functional.h"
#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
#include "third_party/blink/renderer/platform/wtf/vector.h"
#include "third_party/libyuv/include/libyuv.h"
#include "third_party/webrtc/modules/video_coding/codecs/h264/include/h264.h"
#include "third_party/webrtc/modules/video_coding/include/video_error_codes.h"
#include "third_party/webrtc/rtc_base/time_utils.h"

namespace {

// Allow MappableSI to be used for RTCVideoEncoder.
BASE_FEATURE(…);

media::SVCScalabilityMode ToSVCScalabilityMode(
    const std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>&
        spatial_layers,
    media::SVCInterLayerPredMode inter_layer_pred) { … }

class SignaledValue { … };

class ScopedSignaledValue { … };

// TODO(https://crbug.com/1448809): Move to base/memory/ref_counted_memory.h
class RefCountedWritableSharedMemoryMapping
    : public ThreadSafeRefCounted<RefCountedWritableSharedMemoryMapping> { … };

class EncodedDataWrapper : public webrtc::EncodedImageBufferInterface { … };

struct FrameChunk { … };

bool ConvertKbpsToBps(uint32_t bitrate_kbps, uint32_t* bitrate_bps) { … }

uint8_t GetDropFrameThreshold(const webrtc::VideoCodec& codec_settings) { … }

webrtc::VideoBitrateAllocation AllocateBitrateForVEAConfig(
    const media::VideoEncodeAccelerator::Config& config) { … }

}  // namespace

namespace WTF {

template <>
struct CrossThreadCopier<webrtc::VideoEncoder::RateControlParameters>
    : public CrossThreadCopierPassThrough<
          webrtc::VideoEncoder::RateControlParameters> { … };

template <>
struct CrossThreadCopier<
    std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>>
    : public CrossThreadCopierPassThrough<
          std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>> { … };

template <>
struct CrossThreadCopier<FrameChunk>
    : public CrossThreadCopierPassThrough<FrameChunk> { … };

template <>
struct CrossThreadCopier<media::VideoEncodeAccelerator::Config>
    : public CrossThreadCopierPassThrough<
          media::VideoEncodeAccelerator::Config> { … };

template <>
struct CrossThreadCopier<SignaledValue> { … };
}  // namespace WTF

namespace blink {

namespace features {

// Enabled-by-default, except for Android where SW encoder for H264 is not
// available. The existence of this flag remains only for testing purposes.
BASE_FEATURE(…);
#else
             base::FEATURE_DISABLED_BY_DEFAULT);
#endif

// When disabled, SW is forced at <360p. When enabled, SW is forced at <=360p.
// Only applicable when `kForceSoftwareForLowResolutions` is enabled.
BASE_FEATURE(…);

// Avoids large latencies to build up by dropping frames when the number of
// frames that are sent to a hardware video encoder reaches a certain limit.
// See b/298660336 for details.
BASE_FEATURE(…);

// When enabled, the encoder instance is preserved on Release() call.
// Reinitialization of the encoder will reuse the instance with the new
// resolution. See b/1466102 for details.
BASE_FEATURE(…);

}  // namespace features

namespace {
media::SVCInterLayerPredMode CopyFromWebRtcInterLayerPredMode(
    const webrtc::InterLayerPredMode inter_layer_pred) { … }

// Create VEA::Config::SpatialLayer from |codec_settings|. If some config of
// |codec_settings| is not supported, returns false.
bool CreateSpatialLayersConfig(
    const webrtc::VideoCodec& codec_settings,
    std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>*
        spatial_layers,
    media::SVCInterLayerPredMode* inter_layer_pred,
    gfx::Size* highest_active_resolution) { … }

struct ActiveSpatialLayers { … };

struct FrameInfo { … };

webrtc::VideoCodecType ProfileToWebRtcVideoCodecType(
    media::VideoCodecProfile profile) { … }

void RecordInitEncodeUMA(int32_t init_retval,
                         media::VideoCodecProfile profile) { … }

void RecordEncoderStatusUMA(const media::EncoderStatus& status,
                            webrtc::VideoCodecType type) { … }

bool IsZeroCopyEnabled(webrtc::VideoContentType content_type) { … }

}  // namespace

namespace features {
// Fallback from hardware encoder (if available) to software, for WebRTC
// screensharing that uses temporal scalability.
BASE_FEATURE(…);
}  // namespace features

// This private class of RTCVideoEncoder does the actual work of communicating
// with a media::VideoEncodeAccelerator for handling video encoding. It can
// be created on any thread, but should subsequently be executed on
// |gpu_task_runner| including destructor.
//
// This class separates state related to the thread that RTCVideoEncoder
// operates on from the thread that |gpu_factories_| provides for accelerator
// operations (presently the media thread).
class RTCVideoEncoder::Impl : public media::VideoEncodeAccelerator::Client { … };

RTCVideoEncoder::Impl::Impl(
    media::GpuVideoAcceleratorFactories* gpu_factories,
    scoped_refptr<media::MojoVideoEncoderMetricsProviderFactory>
        encoder_metrics_provider_factory,
    webrtc::VideoCodecType video_codec_type,
    std::optional<webrtc::ScalabilityMode> scalability_mode,
    webrtc::VideoContentType video_content_type,
    UpdateEncoderInfoCallback update_encoder_info_callback,
    base::RepeatingClosure execute_software_fallback,
    base::WeakPtr<Impl>& weak_this_for_client)
    : … { … }

void RTCVideoEncoder::Impl::CreateAndInitializeVEA(
    const media::VideoEncodeAccelerator::Config& vea_config,
    SignaledValue init_event) { … }

void RTCVideoEncoder::Impl::NotifyEncoderInfoChange(
    const media::VideoEncoderInfo& info) { … }

void RTCVideoEncoder::Impl::Enqueue(FrameChunk frame_chunk) { … }

void RTCVideoEncoder::Impl::BitstreamBufferAvailable(
    int32_t bitstream_buffer_id) { … }

void RTCVideoEncoder::Impl::Suspend(SignaledValue event) { … }

void RTCVideoEncoder::Impl::Drain(SignaledValue event) { … }

void RTCVideoEncoder::Impl::DrainCompleted(bool success) { … }

void RTCVideoEncoder::Impl::UseOutputBitstreamBuffer(
    int32_t bitstream_buffer_id) { … }

void RTCVideoEncoder::Impl::RequestEncodingParametersChange(
    const webrtc::VideoEncoder::RateControlParameters& parameters) { … }

void RTCVideoEncoder::Impl::RequestEncodingParametersChangeInternal(
    const webrtc::VideoEncoder::RateControlParameters& parameters,
    const std::optional<gfx::Size>& input_visible_size) { … }

void RTCVideoEncoder::Impl::RequestEncodingParametersChangeWithSizeChange(
    const webrtc::VideoEncoder::RateControlParameters& parameters,
    const gfx::Size& input_visible_size,
    const media::VideoCodecProfile& profile,
    const media::SVCInterLayerPredMode& inter_layer_pred,
    const std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>&
        spatial_layers,
    SignaledValue event) { … }

void RTCVideoEncoder::Impl::RecordTimestampMatchUMA() const { … }

ActiveSpatialLayers RTCVideoEncoder::Impl::GetActiveSpatialLayers() const { … }

void RTCVideoEncoder::Impl::RequireBitstreamBuffers(
    unsigned int input_count,
    const gfx::Size& input_coded_size,
    size_t output_buffer_size) { … }

void RTCVideoEncoder::Impl::BitstreamBufferReady(
    int32_t bitstream_buffer_id,
    const media::BitstreamBufferMetadata& metadata) { … }

void RTCVideoEncoder::Impl::NotifyErrorStatus(
    const media::EncoderStatus& status) { … }

RTCVideoEncoder::Impl::~Impl() { … }

void RTCVideoEncoder::Impl::EncodeOneFrame(FrameChunk frame_chunk) { … }

void RTCVideoEncoder::Impl::EncodeOneFrameWithNativeInput(
    FrameChunk frame_chunk) { … }

bool RTCVideoEncoder::Impl::CreateBlackGpuMemoryBufferFrame(
    const gfx::Size& natural_size) { … }

void RTCVideoEncoder::Impl::InputBufferReleased(int index) { … }

bool RTCVideoEncoder::Impl::RequiresSizeChange(
    const media::VideoFrame& frame) const { … }

void RTCVideoEncoder::Impl::RegisterEncodeCompleteCallback(
    webrtc::EncodedImageCallback* callback) { … }

#if BUILDFLAG(RTC_USE_H265)
void RTCVideoEncoder::Impl::SetH265ParameterSetsTrackerForTesting(
    std::unique_ptr<H265ParameterSetsTracker> tracker) {
  ps_tracker_ = std::move(tracker);
}
#endif

RTCVideoEncoder::RTCVideoEncoder(
    media::VideoCodecProfile profile,
    bool is_constrained_h264,
    media::GpuVideoAcceleratorFactories* gpu_factories,
    scoped_refptr<media::MojoVideoEncoderMetricsProviderFactory>
        encoder_metrics_provider_factory)
    : … { … }

RTCVideoEncoder::~RTCVideoEncoder() { … }

int32_t RTCVideoEncoder::DrainEncoderAndUpdateFrameSize(
    const gfx::Size& input_visible_size,
    const webrtc::VideoEncoder::RateControlParameters& params,
    const media::SVCInterLayerPredMode& inter_layer_pred,
    const std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>&
        spatial_layers) { … }

int32_t RTCVideoEncoder::InitializeEncoder(
    const media::VideoEncodeAccelerator::Config& vea_config) { … }

bool RTCVideoEncoder::CodecSettingsUsableForFrameSizeChange(
    const webrtc::VideoCodec& codec_settings) const { … }

int32_t RTCVideoEncoder::InitEncode(
    const webrtc::VideoCodec* codec_settings,
    const webrtc::VideoEncoder::Settings& settings) { … }

int32_t RTCVideoEncoder::Encode(
    const webrtc::VideoFrame& input_image,
    const std::vector<webrtc::VideoFrameType>* frame_types) { … }

int32_t RTCVideoEncoder::RegisterEncodeCompleteCallback(
    webrtc::EncodedImageCallback* callback) { … }

int32_t RTCVideoEncoder::Release() { … }

void RTCVideoEncoder::ReleaseImpl() { … }

void RTCVideoEncoder::SetRates(
    const webrtc::VideoEncoder::RateControlParameters& parameters) { … }

webrtc::VideoEncoder::EncoderInfo RTCVideoEncoder::GetEncoderInfo() const { … }

void RTCVideoEncoder::UpdateEncoderInfo(
    media::VideoEncoderInfo media_enc_info,
    std::vector<webrtc::VideoFrameBuffer::Type> preferred_pixel_formats) { … }

void RTCVideoEncoder::SetError(uint32_t impl_id) { … }

#if BUILDFLAG(RTC_USE_H265)
void RTCVideoEncoder::SetH265ParameterSetsTrackerForTesting(
    std::unique_ptr<H265ParameterSetsTracker> tracker) {
  if (!impl_) {
    DVLOG(1) << "Encoder is not initialized";
    return;
  }
  impl_->SetH265ParameterSetsTrackerForTesting(std::move(tracker));
}
#endif

}  // namespace blink
chromium/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
