webrtc_voice_engine.cc | Explore in Territory

/*
 *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "media/engine/webrtc_voice_engine.h"

#include <algorithm>
#include <atomic>
#include <cstdint>
#include <functional>
#include <initializer_list>
#include <iterator>
#include <memory>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>

#include "absl/algorithm/algorithm.h"
#include "absl/algorithm/container.h"
#include "absl/functional/bind_front.h"
#include "absl/strings/match.h"
#include "absl/types/optional.h"
#include "api/audio/audio_frame.h"
#include "api/audio/audio_frame_processor.h"
#include "api/audio/audio_processing.h"
#include "api/audio/audio_processing_statistics.h"
#include "api/audio_codecs/audio_codec_pair_id.h"
#include "api/audio_codecs/audio_encoder.h"
#include "api/call/audio_sink.h"
#include "api/field_trials_view.h"
#include "api/make_ref_counted.h"
#include "api/media_types.h"
#include "api/priority.h"
#include "api/rtp_headers.h"
#include "api/rtp_parameters.h"
#include "api/rtp_transceiver_direction.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/transport/bitrate_settings.h"
#include "api/units/data_rate.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "call/audio_receive_stream.h"
#include "call/packet_receiver.h"
#include "call/rtp_config.h"
#include "call/rtp_transport_controller_send_interface.h"
#include "media/base/audio_source.h"
#include "media/base/codec.h"
#include "media/base/media_constants.h"
#include "media/base/stream_params.h"
#include "media/engine/adm_helpers.h"
#include "media/engine/payload_type_mapper.h"
#include "media/engine/webrtc_media_engine.h"
#include "modules/async_audio_processing/async_audio_processing.h"
#include "modules/audio_mixer/audio_mixer_impl.h"
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_util.h"
#include "rtc_base/checks.h"
#include "rtc_base/dscp.h"
#include "rtc_base/experiments/struct_parameters_parser.h"
#include "rtc_base/logging.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/string_encode.h"
#include "rtc_base/strings/audio_format_to_string.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/strings/string_format.h"
#include "rtc_base/thread_annotations.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/metrics.h"

#if WEBRTC_ENABLE_PROTOBUF
#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
#include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h"
#else
#include "modules/audio_coding/audio_network_adaptor/config.pb.h"
#endif

#endif

namespace cricket {
namespace {

ParseRtpSsrc;

constexpr size_t kMaxUnsignaledRecvStreams = …;

constexpr int kNackRtpHistoryMs = …;

const int kMinTelephoneEventCode = …;  // RFC4733 (Section 2.3.1)
const int kMaxTelephoneEventCode = …;

const int kMinPayloadType = …;
const int kMaxPayloadType = …;

class ProxySink : public webrtc::AudioSinkInterface { … };

bool ValidateStreamParams(const StreamParams& sp) { … }

// Dumps an AudioCodec in RFC 2327-ish format.
std::string ToString(const Codec& codec) { … }

bool IsCodec(const Codec& codec, const char* ref_name) { … }

absl::optional<Codec> FindCodec(const std::vector<Codec>& codecs,
                                const Codec& codec) { … }

bool VerifyUniquePayloadTypes(const std::vector<Codec>& codecs) { … }

absl::optional<std::string> GetAudioNetworkAdaptorConfig(
    const AudioOptions& options) { … }

// Returns its smallest positive argument. If neither argument is positive,
// returns an arbitrary nonpositive value.
int MinPositive(int a, int b) { … }

// `max_send_bitrate_bps` is the bitrate from "b=" in SDP.
// `rtp_max_bitrate_bps` is the bitrate from RtpSender::SetParameters.
absl::optional<int> ComputeSendBitrate(int max_send_bitrate_bps,
                                       absl::optional<int> rtp_max_bitrate_bps,
                                       const webrtc::AudioCodecSpec& spec) { … }

bool IsEnabled(const webrtc::FieldTrialsView& config, absl::string_view trial) { … }

struct AdaptivePtimeConfig { … };

// TODO(tommi): Constructing a receive stream could be made simpler.
// Move some of this boiler plate code into the config structs themselves.
webrtc::AudioReceiveStreamInterface::Config BuildReceiveStreamConfig(
    uint32_t remote_ssrc,
    uint32_t local_ssrc,
    bool use_nack,
    bool enable_non_sender_rtt,
    const std::vector<std::string>& stream_ids,
    const std::vector<webrtc::RtpExtension>& extensions,
    webrtc::Transport* rtcp_send_transport,
    const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
    const std::map<int, webrtc::SdpAudioFormat>& decoder_map,
    absl::optional<webrtc::AudioCodecPairId> codec_pair_id,
    size_t jitter_buffer_max_packets,
    bool jitter_buffer_fast_accelerate,
    int jitter_buffer_min_delay_ms,
    rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor,
    const webrtc::CryptoOptions& crypto_options,
    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { … }

// Utility function to check if RED codec and its parameters match a codec spec.
bool CheckRedParameters(
    const Codec& red_codec,
    const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) { … }

}  // namespace

WebRtcVoiceEngine::WebRtcVoiceEngine(
    webrtc::TaskQueueFactory* task_queue_factory,
    webrtc::AudioDeviceModule* adm,
    const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory,
    const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
    rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
    rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing,
    std::unique_ptr<webrtc::AudioFrameProcessor> audio_frame_processor,
    const webrtc::FieldTrialsView& trials)
    : … { … }

WebRtcVoiceEngine::~WebRtcVoiceEngine() { … }

void WebRtcVoiceEngine::Init() { … }

rtc::scoped_refptr<webrtc::AudioState> WebRtcVoiceEngine::GetAudioState()
    const { … }

std::unique_ptr<VoiceMediaSendChannelInterface>
WebRtcVoiceEngine::CreateSendChannel(
    webrtc::Call* call,
    const MediaConfig& config,
    const AudioOptions& options,
    const webrtc::CryptoOptions& crypto_options,
    webrtc::AudioCodecPairId codec_pair_id) { … }

std::unique_ptr<VoiceMediaReceiveChannelInterface>
WebRtcVoiceEngine::CreateReceiveChannel(
    webrtc::Call* call,
    const MediaConfig& config,
    const AudioOptions& options,
    const webrtc::CryptoOptions& crypto_options,
    webrtc::AudioCodecPairId codec_pair_id) { … }

void WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) { … }

const std::vector<Codec>& WebRtcVoiceEngine::send_codecs() const { … }

const std::vector<Codec>& WebRtcVoiceEngine::recv_codecs() const { … }

std::vector<webrtc::RtpHeaderExtensionCapability>
WebRtcVoiceEngine::GetRtpHeaderExtensions() const { … }

bool WebRtcVoiceEngine::StartAecDump(webrtc::FileWrapper file,
                                     int64_t max_size_bytes) { … }

void WebRtcVoiceEngine::StopAecDump() { … }

absl::optional<webrtc::AudioDeviceModule::Stats>
WebRtcVoiceEngine::GetAudioDeviceStats() { … }

webrtc::AudioDeviceModule* WebRtcVoiceEngine::adm() { … }

webrtc::AudioProcessing* WebRtcVoiceEngine::apm() const { … }

webrtc::AudioState* WebRtcVoiceEngine::audio_state() { … }

std::vector<Codec> WebRtcVoiceEngine::CollectCodecs(
    const std::vector<webrtc::AudioCodecSpec>& specs) const { … }

// --------------------------------- WebRtcVoiceSendChannel ------------------

class WebRtcVoiceSendChannel::WebRtcAudioSendStream : public AudioSource::Sink { … };

WebRtcVoiceSendChannel::WebRtcVoiceSendChannel(
    WebRtcVoiceEngine* engine,
    const MediaConfig& config,
    const AudioOptions& options,
    const webrtc::CryptoOptions& crypto_options,
    webrtc::Call* call,
    webrtc::AudioCodecPairId codec_pair_id)
    : … { … }

WebRtcVoiceSendChannel::~WebRtcVoiceSendChannel() { … }

bool WebRtcVoiceSendChannel::SetOptions(const AudioOptions& options) { … }

bool WebRtcVoiceSendChannel::SetSenderParameters(
    const AudioSenderParameter& params) { … }

absl::optional<Codec> WebRtcVoiceSendChannel::GetSendCodec() const { … }

// Utility function called from SetSenderParameters() to extract current send
// codec settings from the given list of codecs (originally from SDP). Both send
// and receive streams may be reconfigured based on the new settings.
bool WebRtcVoiceSendChannel::SetSendCodecs(
    const std::vector<Codec>& codecs,
    absl::optional<Codec> preferred_codec) { … }

void WebRtcVoiceSendChannel::SetSend(bool send) { … }

bool WebRtcVoiceSendChannel::SetAudioSend(uint32_t ssrc,
                                          bool enable,
                                          const AudioOptions* options,
                                          AudioSource* source) { … }

bool WebRtcVoiceSendChannel::AddSendStream(const StreamParams& sp) { … }

bool WebRtcVoiceSendChannel::RemoveSendStream(uint32_t ssrc) { … }

void WebRtcVoiceSendChannel::SetSsrcListChangedCallback(
    absl::AnyInvocable<void(const std::set<uint32_t>&)> callback) { … }

bool WebRtcVoiceSendChannel::SetLocalSource(uint32_t ssrc,
                                            AudioSource* source) { … }

bool WebRtcVoiceSendChannel::CanInsertDtmf() { … }

void WebRtcVoiceSendChannel::SetFrameEncryptor(
    uint32_t ssrc,
    rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) { … }

bool WebRtcVoiceSendChannel::InsertDtmf(uint32_t ssrc,
                                        int event,
                                        int duration) { … }

void WebRtcVoiceSendChannel::OnPacketSent(const rtc::SentPacket& sent_packet) { … }

void WebRtcVoiceSendChannel::OnNetworkRouteChanged(
    absl::string_view transport_name,
    const rtc::NetworkRoute& network_route) { … }

bool WebRtcVoiceSendChannel::MuteStream(uint32_t ssrc, bool muted) { … }

bool WebRtcVoiceSendChannel::SetMaxSendBitrate(int bps) { … }

void WebRtcVoiceSendChannel::OnReadyToSend(bool ready) { … }

bool WebRtcVoiceSendChannel::GetStats(VoiceMediaSendInfo* info) { … }

void WebRtcVoiceSendChannel::FillSendCodecStats(
    VoiceMediaSendInfo* voice_media_info) { … }

void WebRtcVoiceSendChannel::SetEncoderToPacketizerFrameTransformer(
    uint32_t ssrc,
    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { … }

webrtc::RtpParameters WebRtcVoiceSendChannel::GetRtpSendParameters(
    uint32_t ssrc) const { … }

webrtc::RTCError WebRtcVoiceSendChannel::SetRtpSendParameters(
    uint32_t ssrc,
    const webrtc::RtpParameters& parameters,
    webrtc::SetParametersCallback callback) { … }

// -------------------------- WebRtcVoiceReceiveChannel ----------------------

class WebRtcVoiceReceiveChannel::WebRtcAudioReceiveStream { … };

WebRtcVoiceReceiveChannel::WebRtcVoiceReceiveChannel(
    WebRtcVoiceEngine* engine,
    const MediaConfig& config,
    const AudioOptions& options,
    const webrtc::CryptoOptions& crypto_options,
    webrtc::Call* call,
    webrtc::AudioCodecPairId codec_pair_id)
    : … { … }

WebRtcVoiceReceiveChannel::~WebRtcVoiceReceiveChannel() { … }

bool WebRtcVoiceReceiveChannel::SetReceiverParameters(
    const AudioReceiverParameters& params) { … }

webrtc::RtpParameters WebRtcVoiceReceiveChannel::GetRtpReceiverParameters(
    uint32_t ssrc) const { … }

webrtc::RtpParameters
WebRtcVoiceReceiveChannel::GetDefaultRtpReceiveParameters() const { … }

bool WebRtcVoiceReceiveChannel::SetOptions(const AudioOptions& options) { … }

bool WebRtcVoiceReceiveChannel::SetRecvCodecs(
    const std::vector<Codec>& codecs) { … }

void WebRtcVoiceReceiveChannel::SetReceiveNackEnabled(bool enabled) { … }

void WebRtcVoiceReceiveChannel::SetRtcpMode(webrtc::RtcpMode mode) { … }

void WebRtcVoiceReceiveChannel::SetReceiveNonSenderRttEnabled(bool enabled) { … }

void WebRtcVoiceReceiveChannel::SetPlayout(bool playout) { … }

bool WebRtcVoiceReceiveChannel::AddRecvStream(const StreamParams& sp) { … }

bool WebRtcVoiceReceiveChannel::RemoveRecvStream(uint32_t ssrc) { … }

void WebRtcVoiceReceiveChannel::ResetUnsignaledRecvStream() { … }

absl::optional<uint32_t> WebRtcVoiceReceiveChannel::GetUnsignaledSsrc() const { … }

void WebRtcVoiceReceiveChannel::ChooseReceiverReportSsrc(
    const std::set<uint32_t>& choices) { … }

// Not implemented.
// TODO(https://crbug.com/webrtc/12676): Implement a fix for the unsignalled
// SSRC race that can happen when an m= section goes from receiving to not
// receiving.
void WebRtcVoiceReceiveChannel::OnDemuxerCriteriaUpdatePending() { … }
void WebRtcVoiceReceiveChannel::OnDemuxerCriteriaUpdateComplete() { … }

bool WebRtcVoiceReceiveChannel::SetOutputVolume(uint32_t ssrc, double volume) { … }

bool WebRtcVoiceReceiveChannel::SetDefaultOutputVolume(double volume) { … }

bool WebRtcVoiceReceiveChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc,
                                                             int delay_ms) { … }

absl::optional<int> WebRtcVoiceReceiveChannel::GetBaseMinimumPlayoutDelayMs(
    uint32_t ssrc) const { … }

void WebRtcVoiceReceiveChannel::SetFrameDecryptor(
    uint32_t ssrc,
    rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) { … }

void WebRtcVoiceReceiveChannel::OnPacketReceived(
    const webrtc::RtpPacketReceived& packet) { … }

bool WebRtcVoiceReceiveChannel::MaybeCreateDefaultReceiveStream(
    const webrtc::RtpPacketReceived& packet) { … }

bool WebRtcVoiceReceiveChannel::GetStats(VoiceMediaReceiveInfo* info,
                                         bool get_and_clear_legacy_stats) { … }

void WebRtcVoiceReceiveChannel::FillReceiveCodecStats(
    VoiceMediaReceiveInfo* voice_media_info) { … }

void WebRtcVoiceReceiveChannel::SetRawAudioSink(
    uint32_t ssrc,
    std::unique_ptr<webrtc::AudioSinkInterface> sink) { … }

void WebRtcVoiceReceiveChannel::SetDefaultRawAudioSink(
    std::unique_ptr<webrtc::AudioSinkInterface> sink) { … }

std::vector<webrtc::RtpSource> WebRtcVoiceReceiveChannel::GetSources(
    uint32_t ssrc) const { … }

void WebRtcVoiceReceiveChannel::SetDepacketizerToDecoderFrameTransformer(
    uint32_t ssrc,
    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { … }

bool WebRtcVoiceReceiveChannel::MaybeDeregisterUnsignaledRecvStream(
    uint32_t ssrc) { … }
}  // namespace cricket
chromium/third_party/webrtc/media/engine/webrtc_voice_engine.cc
