user_media_processor.cc | Explore in Territory

// Copyright 2017 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifdef UNSAFE_BUFFERS_BUILD
// TODO(crbug.com/351564777): Remove this and convert code to safer constructs.
#pragma allow_unsafe_buffers
#endif

#include "third_party/blink/renderer/modules/mediastream/user_media_processor.h"

#include <stddef.h>

#include <utility>
#include <vector>

#include "base/containers/contains.h"
#include "base/functional/callback_helpers.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/not_fatal_until.h"
#include "base/ranges/algorithm.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "base/task/single_thread_task_runner.h"
#include "base/types/optional_util.h"
#include "build/build_config.h"
#include "media/base/audio_parameters.h"
#include "media/capture/video_capture_types.h"
#include "media/webrtc/constants.h"
#include "third_party/blink/public/common/features.h"
#include "third_party/blink/public/common/mediastream/media_stream_controls.h"
#include "third_party/blink/public/common/mediastream/media_stream_request.h"
#include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h"
#include "third_party/blink/public/platform/browser_interface_broker_proxy.h"
#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_source.h"
#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_track.h"
#include "third_party/blink/public/platform/modules/webrtc/webrtc_logging.h"
#include "third_party/blink/public/platform/web_string.h"
#include "third_party/blink/public/platform/web_vector.h"
#include "third_party/blink/public/web/modules/mediastream/web_media_stream_device_observer.h"
#include "third_party/blink/public/web/web_local_frame.h"
#include "third_party/blink/public/web/web_local_frame_client.h"
#include "third_party/blink/renderer/core/frame/local_dom_window.h"
#include "third_party/blink/renderer/core/frame/local_frame.h"
#include "third_party/blink/renderer/modules/mediastream/local_media_stream_audio_source.h"
#include "third_party/blink/renderer/modules/mediastream/local_video_capturer_source.h"
#include "third_party/blink/renderer/modules/mediastream/media_constraints.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_audio_processor.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_video_content.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_video_device.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_utils.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_capturer_source.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
#include "third_party/blink/renderer/modules/mediastream/processed_local_audio_source.h"
#include "third_party/blink/renderer/modules/mediastream/user_media_client.h"
#include "third_party/blink/renderer/platform/mediastream/media_constraints_consts.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_source.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_component_impl.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_descriptor.h"
#include "third_party/blink/renderer/platform/mediastream/webrtc_uma_histograms.h"
#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
#include "third_party/blink/renderer/platform/wtf/text/string_builder.h"
#include "third_party/blink/renderer/platform/wtf/wtf_size_t.h"
#include "ui/gfx/geometry/size.h"

namespace blink {

MediaStreamRequestResult;
MediaStreamType;
EchoCancellationType;
AudioSourceErrorCode;

namespace {

const char* MediaStreamRequestResultToString(MediaStreamRequestResult value) { … }

void SendLogMessage(const std::string& message) { … }

void MaybeLogStreamDevice(const int32_t& request_id,
                          const String& label,
                          const std::optional<MediaStreamDevice>& device) { … }

std::string GetTrackLogString(MediaStreamComponent* component,
                              bool is_pending) { … }

std::string GetTrackSourceLogString(blink::MediaStreamAudioSource* source) { … }

std::string GetOnTrackStartedLogString(
    blink::WebPlatformMediaStreamSource* source,
    MediaStreamRequestResult result) { … }

bool IsSameDevice(const MediaStreamDevice& device,
                  const MediaStreamDevice& other_device) { … }

bool IsSameSource(MediaStreamSource* source, MediaStreamSource* other_source) { … }

void SurfaceAudioProcessingSettings(MediaStreamSource* source) { … }

// TODO(crbug.com/704136): Check all places where this helper is used.
// Change their types from using std::vector to WTF::Vector, so this
// extra conversion round is not needed.
template <typename T>
std::vector<T> ToStdVector(const Vector<T>& format_vector) { … }

Vector<blink::VideoInputDeviceCapabilities> ToVideoInputDeviceCapabilities(
    const Vector<mojom::blink::VideoInputDeviceCapabilitiesPtr>&
        input_capabilities) { … }

String ErrorCodeToString(MediaStreamRequestResult result) { … }

#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_CHROMEOS) && !BUILDFLAG(IS_FUCHSIA)

// Returns true if `kGetUserMediaDeferredDeviceSettingsSelection` is enabled,
// but gates it on `kCameraMicPreview` also being being enabled. This only
// applies to user media requests.
bool ShouldDeferDeviceSettingsSelection(
    UserMediaRequestType request_type,
    mojom::blink::MediaStreamType media_stream_type) { … }
#else
bool ShouldDeferDeviceSettingsSelection(
    UserMediaRequestType request_type,
    mojom::blink::MediaStreamType media_stream_type) {
  return false;
}
#endif

}  // namespace

// Class for storing state of the the processing of getUserMedia requests.
class UserMediaProcessor::RequestInfo final
    : public GarbageCollected<UserMediaProcessor::RequestInfo> { … };

// TODO(guidou): Initialize request_result_name_ as a null WTF::String.
// https://crbug.com/764293
UserMediaProcessor::RequestInfo::RequestInfo(UserMediaRequest* request)
    : … { … }

void UserMediaProcessor::RequestInfo::StartAudioTrack(
    MediaStreamComponent* component,
    bool is_pending) { … }

MediaStreamComponent* UserMediaProcessor::RequestInfo::CreateAndStartVideoTrack(
    MediaStreamSource* source) { … }

void UserMediaProcessor::RequestInfo::CallbackOnTracksStarted(
    ResourcesReady callback) { … }

void UserMediaProcessor::RequestInfo::OnTrackStarted(
    blink::WebPlatformMediaStreamSource* source,
    MediaStreamRequestResult result,
    const blink::WebString& result_name) { … }

void UserMediaProcessor::RequestInfo::CheckAllTracksStarted() { … }

size_t UserMediaProcessor::RequestInfo::count_video_devices() const { … }

void UserMediaProcessor::RequestInfo::OnAudioSourceStarted(
    blink::WebPlatformMediaStreamSource* source,
    MediaStreamRequestResult result,
    const String& result_name) { … }

UserMediaProcessor::UserMediaProcessor(
    LocalFrame* frame,
    MediaDevicesDispatcherCallback media_devices_dispatcher_cb,
    scoped_refptr<base::SingleThreadTaskRunner> task_runner)
    : … { … }

UserMediaProcessor::~UserMediaProcessor() { … }

UserMediaRequest* UserMediaProcessor::CurrentRequest() { … }

void UserMediaProcessor::ProcessRequest(UserMediaRequest* request,
                                        base::OnceClosure callback) { … }

void UserMediaProcessor::SetupAudioInput() { … }

void UserMediaProcessor::SelectAudioDeviceSettings(
    UserMediaRequest* user_media_request,
    Vector<mojom::blink::AudioInputDeviceCapabilitiesPtr>
        audio_input_capabilities) { … }

void UserMediaProcessor::SelectAudioSettings(
    UserMediaRequest* user_media_request,
    const blink::AudioDeviceCaptureCapabilities& capabilities) { … }

std::optional<base::UnguessableToken>
UserMediaProcessor::DetermineExistingAudioSessionId(
    const blink::AudioCaptureSettings& settings) { … }

WTF::HashMap<String, base::UnguessableToken>
UserMediaProcessor::DetermineExistingAudioSessionIds() { … }

void UserMediaProcessor::SetupVideoInput() { … }

// static
bool UserMediaProcessor::IsPanTiltZoomPermissionRequested(
    const MediaConstraints& constraints) { … }

void UserMediaProcessor::SelectVideoDeviceSettings(
    UserMediaRequest* user_media_request,
    Vector<mojom::blink::VideoInputDeviceCapabilitiesPtr>
        video_input_capabilities) { … }

void UserMediaProcessor::SelectVideoContentSettings() { … }

void UserMediaProcessor::GenerateStreamForCurrentRequestInfo(
    WTF::HashMap<String, base::UnguessableToken>
        requested_audio_capture_session_ids) { … }

WebMediaStreamDeviceObserver*
UserMediaProcessor::GetMediaStreamDeviceObserver() { … }

void UserMediaProcessor::GotOpenDevice(
    int32_t request_id,
    mojom::blink::MediaStreamRequestResult result,
    mojom::blink::GetOpenDeviceResponsePtr response) { … }

void UserMediaProcessor::OnStreamsGenerated(
    int32_t request_id,
    MediaStreamRequestResult result,
    const String& label,
    mojom::blink::StreamDevicesSetPtr stream_devices_set,
    bool pan_tilt_zoom_allowed) { … }

void UserMediaProcessor::GotAllVideoInputFormatsForDevice(
    UserMediaRequest* user_media_request,
    const String& label,
    const Vector<String>& device_ids,
    const Vector<media::VideoCaptureFormat>& formats) { … }

void UserMediaProcessor::OnStreamGeneratedForCancelledRequest(
    const mojom::blink::StreamDevices& stream_devices) { … }

// static
void UserMediaProcessor::OnAudioSourceStartedOnAudioThread(
    scoped_refptr<base::SingleThreadTaskRunner> task_runner,
    UserMediaProcessor* weak_ptr,
    blink::WebPlatformMediaStreamSource* source,
    MediaStreamRequestResult result,
    const blink::WebString& result_name) { … }

void UserMediaProcessor::OnAudioSourceStarted(
    blink::WebPlatformMediaStreamSource* source,
    MediaStreamRequestResult result,
    const String& result_name) { … }

void UserMediaProcessor::NotifyCurrentRequestInfoOfAudioSourceStarted(
    blink::WebPlatformMediaStreamSource* source,
    MediaStreamRequestResult result,
    const String& result_name) { … }

void UserMediaProcessor::OnStreamGenerationFailed(
    int32_t request_id,
    MediaStreamRequestResult result) { … }

void UserMediaProcessor::OnDeviceStopped(const MediaStreamDevice& device) { … }

void UserMediaProcessor::OnDeviceChanged(const MediaStreamDevice& old_device,
                                         const MediaStreamDevice& new_device) { … }

void UserMediaProcessor::OnDeviceRequestStateChange(
    const MediaStreamDevice& device,
    const mojom::blink::MediaStreamStateChange new_state) { … }

void UserMediaProcessor::OnDeviceCaptureConfigurationChange(
    const MediaStreamDevice& device) { … }

void UserMediaProcessor::OnDeviceCaptureHandleChange(
    const MediaStreamDevice& device) { … }

void UserMediaProcessor::OnZoomLevelChange(const MediaStreamDevice& device,
                                           int zoom_level) { … }

void UserMediaProcessor::Trace(Visitor* visitor) const { … }

MediaStreamSource* UserMediaProcessor::InitializeVideoSourceObject(
    const MediaStreamDevice& device) { … }

MediaStreamSource* UserMediaProcessor::InitializeAudioSourceObject(
    const MediaStreamDevice& device,
    bool* is_pending) { … }

std::unique_ptr<blink::MediaStreamAudioSource>
UserMediaProcessor::CreateAudioSource(
    const MediaStreamDevice& device,
    blink::WebPlatformMediaStreamSource::ConstraintsRepeatingCallback
        source_ready) { … }

std::unique_ptr<blink::MediaStreamVideoSource>
UserMediaProcessor::CreateVideoSource(
    const MediaStreamDevice& device,
    blink::WebPlatformMediaStreamSource::SourceStoppedCallback stop_callback) { … }

void UserMediaProcessor::StartTracks(const String& label) { … }

MediaStreamComponent* UserMediaProcessor::CreateVideoTrack(
    const std::optional<MediaStreamDevice>& device) { … }

MediaStreamComponent* UserMediaProcessor::CreateAudioTrack(
    const std::optional<MediaStreamDevice>& device) { … }

void UserMediaProcessor::OnCreateNativeTracksCompleted(
    const String& label,
    RequestInfo* request_info,
    MediaStreamRequestResult result,
    const String& constraint_name) { … }

void UserMediaProcessor::GetUserMediaRequestSucceeded(
    MediaStreamDescriptorVector* descriptors,
    UserMediaRequest* user_media_request) { … }

void UserMediaProcessor::DelayedGetUserMediaRequestSucceeded(
    int32_t request_id,
    MediaStreamDescriptorVector* components,
    UserMediaRequest* user_media_request) { … }

void UserMediaProcessor::GetUserMediaRequestFailed(
    MediaStreamRequestResult result,
    const String& constraint_name) { … }

void UserMediaProcessor::DelayedGetUserMediaRequestFailed(
    int32_t request_id,
    UserMediaRequest* user_media_request,
    MediaStreamRequestResult result,
    const String& constraint_name) { … }

MediaStreamSource* UserMediaProcessor::FindLocalSource(
    const LocalStreamSources& sources,
    const MediaStreamDevice& device) const { … }

MediaStreamSource* UserMediaProcessor::InitializeSourceObject(
    const MediaStreamDevice& device,
    std::unique_ptr<WebPlatformMediaStreamSource> platform_source) { … }

bool UserMediaProcessor::RemoveLocalSource(MediaStreamSource* source) { … }

bool UserMediaProcessor::IsCurrentRequestInfo(int32_t request_id) const { … }

bool UserMediaProcessor::IsCurrentRequestInfo(
    UserMediaRequest* user_media_request) const { … }

bool UserMediaProcessor::DeleteUserMediaRequest(
    UserMediaRequest* user_media_request) { … }

void UserMediaProcessor::StopAllProcessing() { … }

void UserMediaProcessor::OnLocalSourceStopped(
    const blink::WebMediaStreamSource& source) { … }

void UserMediaProcessor::StopLocalSource(MediaStreamSource* source,
                                         bool notify_dispatcher) { … }

bool UserMediaProcessor::HasActiveSources() const { … }

#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
void UserMediaProcessor::FocusCapturedSurface(const String& label, bool focus) { … }
#endif

mojom::blink::MediaStreamDispatcherHost*
UserMediaProcessor::GetMediaStreamDispatcherHost() { … }

mojom::blink::MediaDevicesDispatcherHost*
UserMediaProcessor::GetMediaDevicesDispatcher() { … }

const blink::AudioCaptureSettings&
UserMediaProcessor::AudioCaptureSettingsForTesting() const { … }

const Vector<blink::AudioCaptureSettings>&
UserMediaProcessor::EligibleAudioCaptureSettingsForTesting() const { … }

const blink::VideoCaptureSettings&
UserMediaProcessor::VideoCaptureSettingsForTesting() const { … }

const Vector<blink::VideoCaptureSettings>&
UserMediaProcessor::EligibleVideoCaptureSettingsForTesting() const { … }

void UserMediaProcessor::SetMediaStreamDeviceObserverForTesting(
    WebMediaStreamDeviceObserver* media_stream_device_observer) { … }

void UserMediaProcessor::KeepDeviceAliveForTransfer(
    base::UnguessableToken session_id,
    base::UnguessableToken transfer_id,
    KeepDeviceAliveForTransferCallback keep_alive_cb) { … }

}  // namespace blink
chromium/third_party/blink/renderer/modules/mediastream/user_media_processor.cc
