video_track_recorder.cc | Explore in Territory

// Copyright 2015 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/351564777): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "third_party/blink/renderer/modules/mediarecorder/video_track_recorder.h"

#include <memory>

#include "base/functional/bind.h"
#include "base/functional/overloaded.h"
#include "base/logging.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/metrics/histogram_macros.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/task/task_traits.h"
#include "base/task/thread_pool.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "cc/paint/skia_paint_canvas.h"
#include "media/base/async_destroy_video_encoder.h"
#include "media/base/media_util.h"
#include "media/base/video_codecs.h"
#include "media/base/video_encoder_metrics_provider.h"
#include "media/base/video_frame.h"
#include "media/base/video_util.h"
#include "media/media_buildflags.h"
#include "media/muxers/webm_muxer.h"
#include "media/renderers/paint_canvas_video_renderer.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "media/video/video_encode_accelerator_adapter.h"
#include "media/video/vpx_video_encoder.h"
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/public/platform/web_graphics_context_3d_provider.h"
#include "third_party/blink/renderer/modules/mediarecorder/media_recorder_encoder_wrapper.h"
#include "third_party/blink/renderer/modules/mediarecorder/vea_encoder.h"
#include "third_party/blink/renderer/modules/mediarecorder/vpx_encoder.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
#include "third_party/blink/renderer/platform/graphics/web_graphics_context_3d_provider_util.h"
#include "third_party/blink/renderer/platform/heap/persistent.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_component.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
#include "third_party/blink/renderer/platform/runtime_enabled_features.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_std.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
#include "third_party/blink/renderer/platform/wtf/functional.h"
#include "third_party/blink/renderer/platform/wtf/sequence_bound.h"
#include "third_party/libyuv/include/libyuv.h"
#include "ui/gfx/geometry/size.h"

#if BUILDFLAG(ENABLE_OPENH264)
#include "media/video/openh264_video_encoder.h"
#include "third_party/blink/renderer/modules/mediarecorder/h264_encoder.h"
#endif  // #if BUILDFLAG(ENABLE_OPENH264)

#if BUILDFLAG(ENABLE_LIBAOM)
#include "media/video/av1_video_encoder.h"
#endif  // BUILDFLAG(ENABLE_LIBAOM)

kVEAEncoderMinResolutionHeight;
kVEAEncoderMinResolutionWidth;

namespace WTF {
template <>
struct CrossThreadCopier<std::vector<scoped_refptr<media::VideoFrame>>>
    : public CrossThreadCopierPassThrough<
          std::vector<scoped_refptr<media::VideoFrame>>> { … };

template <>
struct CrossThreadCopier<blink::KeyFrameRequestProcessor::Configuration>
    : public CrossThreadCopierPassThrough<
          blink::KeyFrameRequestProcessor::Configuration> { … };
}  // namespace WTF

namespace blink {

// Helper class used to bless annotation of our calls to
// CreateOffscreenGraphicsContext3DProvider using ScopedAllowBaseSyncPrimitives.
class VideoTrackRecorderImplContextProvider { … };

CodecId;

libyuv::RotationMode MediaVideoRotationToRotationMode(
    media::VideoRotation rotation) { … }

namespace {

static const struct { … } kPreferredCodecIdAndVEAProfiles[] = …;

static_assert …;

// The maximum number of frames which we'll keep frame references alive for
// encode. The number of frames in flight is further restricted by the device
// video capture max buffer pool size if it is smaller. This guarantees that
// there is limit on the number of frames in a FIFO queue that are being encoded
// and frames coming after this limit is reached are dropped.
// TODO(emircan): Make this a LIFO queue that has different sizes for each
// encoder implementation.
const size_t kMaxNumberOfFramesInEncode = …;

void NotifyEncoderSupportKnown(base::OnceClosure callback) { … }

// Obtains video encode accelerator's supported profiles.
media::VideoEncodeAccelerator::SupportedProfiles GetVEASupportedProfiles() { … }

VideoTrackRecorderImpl::CodecEnumerator* GetCodecEnumerator() { … }

void UmaHistogramForCodec(bool uses_acceleration, CodecId codec_id) { … }

// Returns the default codec profile for |codec_id|.
std::optional<media::VideoCodecProfile> GetMediaVideoCodecProfileForSwEncoder(
    VideoTrackRecorder::CodecId codec_id) { … }

bool IsSoftwareEncoderAvailable(CodecId codec_id) { … }

std::optional<media::VideoCodecProfile> GetMediaVideoCodecProfile(
    VideoTrackRecorder::CodecProfile codec_profile,
    const gfx::Size& input_size,
    bool allow_vea_encoder) { … }

MediaRecorderEncoderWrapper::CreateEncoderCB
GetCreateHardwareVideoEncoderCallback() { … }

MediaRecorderEncoderWrapper::CreateEncoderCB
GetCreateSoftwareVideoEncoderCallback(CodecId codec_id) { … }
}  // anonymous namespace

VideoTrackRecorder::VideoTrackRecorder(
    scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner,
    WeakCell<CallbackInterface>* callback_interface)
    : … { … }

VideoTrackRecorderImpl::CodecProfile::CodecProfile(CodecId codec_id)
    : … { … }

VideoTrackRecorderImpl::CodecProfile::CodecProfile(
    CodecId codec_id,
    std::optional<media::VideoCodecProfile> opt_profile,
    std::optional<media::VideoCodecLevel> opt_level)
    : … { … }

VideoTrackRecorderImpl::CodecProfile::CodecProfile(
    CodecId codec_id,
    media::VideoCodecProfile profile,
    media::VideoCodecLevel level)
    : … { … }

VideoTrackRecorderImpl::CodecEnumerator::CodecEnumerator(
    const media::VideoEncodeAccelerator::SupportedProfiles&
        vea_supported_profiles) { … }

VideoTrackRecorderImpl::CodecEnumerator::~CodecEnumerator() = default;

std::pair<media::VideoCodecProfile, bool>
VideoTrackRecorderImpl::CodecEnumerator::FindSupportedVideoCodecProfile(
    CodecId codec,
    media::VideoCodecProfile profile) const { … }

VideoTrackRecorderImpl::CodecId
VideoTrackRecorderImpl::CodecEnumerator::GetPreferredCodecId(
    MediaTrackContainerType type) const { … }

std::pair<media::VideoCodecProfile, bool>
VideoTrackRecorderImpl::CodecEnumerator::GetFirstSupportedVideoCodecProfile(
    CodecId codec) const { … }

media::VideoEncodeAccelerator::SupportedProfiles
VideoTrackRecorderImpl::CodecEnumerator::GetSupportedProfiles(
    CodecId codec) const { … }

VideoTrackRecorderImpl::Counter::Counter() : … { … }

VideoTrackRecorderImpl::Counter::~Counter() = default;

void VideoTrackRecorderImpl::Counter::IncreaseCount() { … }

void VideoTrackRecorderImpl::Counter::DecreaseCount() { … }

base::WeakPtr<VideoTrackRecorderImpl::Counter>
VideoTrackRecorderImpl::Counter::GetWeakPtr() { … }

VideoTrackRecorderImpl::Encoder::Encoder(
    scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
    const OnEncodedVideoCB& on_encoded_video_cb,
    uint32_t bits_per_second)
    : … { … }

VideoTrackRecorderImpl::Encoder::~Encoder() = default;

void VideoTrackRecorderImpl::Encoder::InitializeEncoder(
    KeyFrameRequestProcessor::Configuration key_frame_config,
    std::unique_ptr<media::VideoEncoderMetricsProvider> metrics_provider,
    size_t frame_buffer_pool_limit) { … }

void VideoTrackRecorderImpl::Encoder::Initialize() { … }

void VideoTrackRecorderImpl::Encoder::StartFrameEncode(
    scoped_refptr<media::VideoFrame> video_frame,
    base::TimeTicks capture_timestamp) { … }

scoped_refptr<media::VideoFrame>
VideoTrackRecorderImpl::Encoder::MaybeProvideEncodableFrame(
    scoped_refptr<media::VideoFrame> video_frame) { … }

void VideoTrackRecorderImpl::Encoder::SetPaused(bool paused) { … }

bool VideoTrackRecorderImpl::Encoder::CanEncodeAlphaChannel() const { … }

scoped_refptr<media::VideoFrame>
VideoTrackRecorderImpl::Encoder::ConvertToI420ForSoftwareEncoder(
    scoped_refptr<media::VideoFrame> frame) { … }

// static
VideoTrackRecorderImpl::CodecId VideoTrackRecorderImpl::GetPreferredCodecId(
    MediaTrackContainerType type) { … }

// static
bool VideoTrackRecorderImpl::CanUseAcceleratedEncoder(
    CodecProfile& codec_profile,
    size_t width,
    size_t height,
    double framerate) { … }

VideoTrackRecorderImpl::VideoTrackRecorderImpl(
    scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner,
    CodecProfile codec_profile,
    MediaStreamComponent* track,
    WeakCell<CallbackInterface>* callback_interface,
    uint32_t bits_per_second,
    KeyFrameRequestProcessor::Configuration key_frame_config,
    size_t frame_buffer_pool_limit)
    : … { … }

VideoTrackRecorderImpl::~VideoTrackRecorderImpl() { … }

void VideoTrackRecorderImpl::OnEncoderSupportKnown() { … }

void VideoTrackRecorderImpl::OnVideoFrame(
    bool allow_vea_encoder,
    scoped_refptr<media::VideoFrame> video_frame,
    base::TimeTicks capture_time) { … }

void VideoTrackRecorderImpl::ProcessOneVideoFrame(
    bool allow_vea_encoder,
    scoped_refptr<media::VideoFrame> video_frame,
    base::TimeTicks capture_time) { … }

void VideoTrackRecorderImpl::Pause() { … }

void VideoTrackRecorderImpl::Resume() { … }

void VideoTrackRecorderImpl::OnVideoFrameForTesting(
    scoped_refptr<media::VideoFrame> frame,
    base::TimeTicks timestamp,
    bool allow_vea_encoder) { … }

void VideoTrackRecorderImpl::ForceKeyFrameForNextFrameForTesting() { … }

std::unique_ptr<VideoTrackRecorder::Encoder>
VideoTrackRecorderImpl::CreateMediaVideoEncoder(
    scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
    CodecProfile codec_profile,
    bool is_screencast,
    bool create_vea_encoder) { … }

std::unique_ptr<VideoTrackRecorder::Encoder>
VideoTrackRecorderImpl::CreateSoftwareVideoEncoder(
    scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
    CodecProfile codec_profile,
    bool is_screencast) { … }

std::unique_ptr<VideoTrackRecorder::Encoder>
VideoTrackRecorderImpl::CreateHardwareVideoEncoder(
    scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
    CodecProfile codec_profile,
    const gfx::Size& input_size,
    bool use_import_mode,
    bool is_screencast) { … }

void VideoTrackRecorderImpl::InitializeEncoder(
    uint32_t bits_per_second,
    bool allow_vea_encoder,
    media::VideoFrame::StorageType frame_storage_type,
    gfx::Size input_size) { … }

void VideoTrackRecorderImpl::OnHardwareEncoderError() { … }

void VideoTrackRecorderImpl::ConnectToTrack(
    const VideoCaptureDeliverFrameCB& callback) { … }

void VideoTrackRecorderImpl::DisconnectFromTrack() { … }

VideoTrackRecorderPassthrough::VideoTrackRecorderPassthrough(
    scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner,
    MediaStreamComponent* track,
    WeakCell<CallbackInterface>* callback_interface,
    KeyFrameRequestProcessor::Configuration key_frame_config)
    : … { … }

VideoTrackRecorderPassthrough::~VideoTrackRecorderPassthrough() { … }

void VideoTrackRecorderPassthrough::Pause() { … }

void VideoTrackRecorderPassthrough::Resume() { … }

void VideoTrackRecorderPassthrough::OnEncodedVideoFrameForTesting(
    base::TimeTicks now,
    scoped_refptr<EncodedVideoFrame> frame,
    base::TimeTicks capture_time) { … }

void VideoTrackRecorderPassthrough::RequestKeyFrame() { … }

void VideoTrackRecorderPassthrough::DisconnectFromTrack() { … }

void VideoTrackRecorderPassthrough::HandleEncodedVideoFrame(
    base::RepeatingCallback<base::TimeTicks()> time_now_callback,
    scoped_refptr<EncodedVideoFrame> encoded_frame,
    base::TimeTicks estimated_capture_time) { … }

}  // namespace blink
chromium/third_party/blink/renderer/modules/mediarecorder/video_track_recorder.cc
