TeamPGM/Nagram
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
d30f796d8c
Nagram/TMessagesProj/jni/voip/webrtc/audio/audio_send_stream.cc
xaxtix d30f796d8c Update to 8.6.0
2022-03-11 19:49:54 +03:00

935 lines
35 KiB
C++
Raw Blame History

/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "audio/audio_send_stream.h"
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "api/audio_codecs/audio_encoder.h"
#include "api/audio_codecs/audio_encoder_factory.h"
#include "api/audio_codecs/audio_format.h"
#include "api/call/transport.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/function_view.h"
#include "api/rtc_event_log/rtc_event_log.h"
#include "audio/audio_state.h"
#include "audio/channel_send.h"
#include "audio/conversion.h"
#include "call/rtp_config.h"
#include "call/rtp_transport_controller_send_interface.h"
#include "common_audio/vad/include/vad.h"
#include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h"
#include "logging/rtc_event_log/rtc_stream_config.h"
#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
#include "modules/audio_coding/codecs/red/audio_encoder_copy_red.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/audio_format_to_string.h"
#include "rtc_base/task_queue.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
void UpdateEventLogStreamConfig(RtcEventLog* event_log,
const AudioSendStream::Config& config,
const AudioSendStream::Config* old_config) {
using SendCodecSpec = AudioSendStream::Config::SendCodecSpec;
// Only update if any of the things we log have changed.
auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a,
const absl::optional<SendCodecSpec>& b) {
if (a.has_value() && b.has_value()) {
return a->format.name == b->format.name &&
a->payload_type == b->payload_type;
}
return !a.has_value() && !b.has_value();
};
if (old_config && config.rtp.ssrc == old_config->rtp.ssrc &&
config.rtp.extensions == old_config->rtp.extensions &&
payload_types_equal(config.send_codec_spec,
old_config->send_codec_spec)) {
return;
}
auto rtclog_config = std::make_unique<rtclog::StreamConfig>();
rtclog_config->local_ssrc = config.rtp.ssrc;
rtclog_config->rtp_extensions = config.rtp.extensions;
if (config.send_codec_spec) {
rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name,
config.send_codec_spec->payload_type, 0);
}
event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>(
std::move(rtclog_config)));
}
} // namespace
constexpr char AudioAllocationConfig::kKey[];
std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() {
return StructParametersParser::Create( //
"min", &min_bitrate, //
"max", &max_bitrate, //
"prio_rate", &priority_bitrate, //
"prio_rate_raw", &priority_bitrate_raw, //
"rate_prio", &bitrate_priority);
}
AudioAllocationConfig::AudioAllocationConfig() {
Parser()->Parse(field_trial::FindFullName(kKey));
if (priority_bitrate_raw && !priority_bitrate.IsZero()) {
RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually "
"exclusive but both were configured.";
}
}
namespace internal {
AudioSendStream::AudioSendStream(
Clock* clock,
const webrtc::AudioSendStream::Config& config,
const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
TaskQueueFactory* task_queue_factory,
RtpTransportControllerSendInterface* rtp_transport,
BitrateAllocatorInterface* bitrate_allocator,
RtcEventLog* event_log,
RtcpRttStats* rtcp_rtt_stats,
const absl::optional<RtpState>& suspended_rtp_state)
: AudioSendStream(clock,
config,
audio_state,
task_queue_factory,
rtp_transport,
bitrate_allocator,
event_log,
suspended_rtp_state,
voe::CreateChannelSend(
clock,
task_queue_factory,
config.send_transport,
rtcp_rtt_stats,
event_log,
config.frame_encryptor,
config.crypto_options,
config.rtp.extmap_allow_mixed,
config.rtcp_report_interval_ms,
config.rtp.ssrc,
config.frame_transformer,
rtp_transport->transport_feedback_observer())) {}
AudioSendStream::AudioSendStream(
Clock* clock,
const webrtc::AudioSendStream::Config& config,
const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
TaskQueueFactory* task_queue_factory,
RtpTransportControllerSendInterface* rtp_transport,
BitrateAllocatorInterface* bitrate_allocator,
RtcEventLog* event_log,
const absl::optional<RtpState>& suspended_rtp_state,
std::unique_ptr<voe::ChannelSendInterface> channel_send)
: clock_(clock),
rtp_transport_queue_(rtp_transport->GetWorkerQueue()),
allocate_audio_without_feedback_(
field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")),
enable_audio_alr_probing_(
!field_trial::IsDisabled("WebRTC-Audio-AlrProbing")),
send_side_bwe_with_overhead_(
!field_trial::IsDisabled("WebRTC-SendSideBwe-WithOverhead")),
config_(Config(/*send_transport=*/nullptr)),
audio_state_(audio_state),
channel_send_(std::move(channel_send)),
event_log_(event_log),
use_legacy_overhead_calculation_(
field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")),
bitrate_allocator_(bitrate_allocator),
rtp_transport_(rtp_transport),
rtp_rtcp_module_(channel_send_->GetRtpRtcp()),
suspended_rtp_state_(suspended_rtp_state) {
RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc;
RTC_DCHECK(rtp_transport_queue_);
RTC_DCHECK(audio_state_);
RTC_DCHECK(channel_send_);
RTC_DCHECK(bitrate_allocator_);
RTC_DCHECK(rtp_transport);
RTC_DCHECK(rtp_rtcp_module_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
ConfigureStream(config, true);
UpdateCachedTargetAudioBitrateConstraints();
pacer_thread_checker_.Detach();
}
AudioSendStream::~AudioSendStream() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc;
RTC_DCHECK(!sending_);
channel_send_->ResetSenderCongestionControlObjects();
// Blocking call to synchronize state with worker queue to ensure that there
// are no pending tasks left that keeps references to audio.
rtc::Event thread_sync_event;
rtp_transport_queue_->PostTask([&] { thread_sync_event.Set(); });
thread_sync_event.Wait(rtc::Event::kForever);
}
const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return config_;
}
void AudioSendStream::Reconfigure(
const webrtc::AudioSendStream::Config& new_config) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
ConfigureStream(new_config, false);
}
AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds(
const std::vector<RtpExtension>& extensions) {
ExtensionIds ids;
for (const auto& extension : extensions) {
if (extension.uri == RtpExtension::kAudioLevelUri) {
ids.audio_level = extension.id;
} else if (extension.uri == RtpExtension::kAbsSendTimeUri) {
ids.abs_send_time = extension.id;
} else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
ids.transport_sequence_number = extension.id;
} else if (extension.uri == RtpExtension::kMidUri) {
ids.mid = extension.id;
} else if (extension.uri == RtpExtension::kRidUri) {
ids.rid = extension.id;
} else if (extension.uri == RtpExtension::kRepairedRidUri) {
ids.repaired_rid = extension.id;
} else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) {
ids.abs_capture_time = extension.id;
}
}
return ids;
}
int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) {
return FindExtensionIds(config.rtp.extensions).transport_sequence_number;
}
void AudioSendStream::ConfigureStream(
const webrtc::AudioSendStream::Config& new_config,
bool first_time) {
RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: "
<< new_config.ToString();
UpdateEventLogStreamConfig(event_log_, new_config,
first_time ? nullptr : &config_);
const auto& old_config = config_;
// Configuration parameters which cannot be changed.
RTC_DCHECK(first_time ||
old_config.send_transport == new_config.send_transport);
RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc);
if (suspended_rtp_state_ && first_time) {
rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_);
}
if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) {
channel_send_->SetRTCP_CNAME(new_config.rtp.c_name);
}
// Enable the frame encryptor if a new frame encryptor has been provided.
if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) {
channel_send_->SetFrameEncryptor(new_config.frame_encryptor);
}
if (first_time ||
new_config.frame_transformer != old_config.frame_transformer) {
channel_send_->SetEncoderToPacketizerFrameTransformer(
new_config.frame_transformer);
}
if (first_time ||
new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) {
rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed);
}
const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions);
const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions);
// Audio level indication
if (first_time || new_ids.audio_level != old_ids.audio_level) {
channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0,
new_ids.audio_level);
}
if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) {
absl::string_view uri = AbsoluteSendTime::Uri();
rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri);
if (new_ids.abs_send_time) {
rtp_rtcp_module_->RegisterRtpHeaderExtension(uri, new_ids.abs_send_time);
}
}
bool transport_seq_num_id_changed =
new_ids.transport_sequence_number != old_ids.transport_sequence_number;
if (first_time ||
(transport_seq_num_id_changed && !allocate_audio_without_feedback_)) {
if (!first_time) {
channel_send_->ResetSenderCongestionControlObjects();
}
RtcpBandwidthObserver* bandwidth_observer = nullptr;
if (!allocate_audio_without_feedback_ &&
new_ids.transport_sequence_number != 0) {
rtp_rtcp_module_->RegisterRtpHeaderExtension(
TransportSequenceNumber::Uri(), new_ids.transport_sequence_number);
// Probing in application limited region is only used in combination with
// send side congestion control, wich depends on feedback packets which
// requires transport sequence numbers to be enabled.
// Optionally request ALR probing but do not override any existing
// request from other streams.
if (enable_audio_alr_probing_) {
rtp_transport_->EnablePeriodicAlrProbing(true);
}
bandwidth_observer = rtp_transport_->GetBandwidthObserver();
}
channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_,
bandwidth_observer);
}
// MID RTP header extension.
if ((first_time || new_ids.mid != old_ids.mid ||
new_config.rtp.mid != old_config.rtp.mid) &&
new_ids.mid != 0 && !new_config.rtp.mid.empty()) {
rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::Uri(), new_ids.mid);
rtp_rtcp_module_->SetMid(new_config.rtp.mid);
}
if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) {
absl::string_view uri = AbsoluteCaptureTimeExtension::Uri();
rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(uri);
if (new_ids.abs_capture_time) {
rtp_rtcp_module_->RegisterRtpHeaderExtension(uri,
new_ids.abs_capture_time);
}
}
if (!ReconfigureSendCodec(new_config)) {
RTC_LOG(LS_ERROR) << "Failed to set up send codec state.";
}
// Set currently known overhead (used in ANA, opus only).
{
MutexLock lock(&overhead_per_packet_lock_);
UpdateOverheadForEncoder();
}
channel_send_->CallEncoder([this](AudioEncoder* encoder) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!encoder) {
return;
}
frame_length_range_ = encoder->GetFrameLengthRange();
UpdateCachedTargetAudioBitrateConstraints();
});
if (sending_) {
ReconfigureBitrateObserver(new_config);
}
config_ = new_config;
if (!first_time) {
UpdateCachedTargetAudioBitrateConstraints();
}
}
void AudioSendStream::Start() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (sending_) {
return;
}
if (!config_.has_dscp && config_.min_bitrate_bps != -1 &&
config_.max_bitrate_bps != -1 &&
(allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) {
rtp_transport_->AccountForAudioPacketsInPacedSender(true);
if (send_side_bwe_with_overhead_)
rtp_transport_->IncludeOverheadInPacedSender();
rtp_rtcp_module_->SetAsPartOfAllocation(true);
ConfigureBitrateObserver();
} else {
rtp_rtcp_module_->SetAsPartOfAllocation(false);
}
channel_send_->StartSend();
sending_ = true;
audio_state()->AddSendingStream(this, encoder_sample_rate_hz_,
encoder_num_channels_);
}
void AudioSendStream::Stop() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!sending_) {
return;
}
RemoveBitrateObserver();
channel_send_->StopSend();
sending_ = false;
audio_state()->RemoveSendingStream(this);
}
void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) {
RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_);
RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0);
double duration = static_cast<double>(audio_frame->samples_per_channel_) /
audio_frame->sample_rate_hz_;
{
// Note: SendAudioData() passes the frame further down the pipeline and it
// may eventually get sent. But this method is invoked even if we are not
// connected, as long as we have an AudioSendStream (created as a result of
// an O/A exchange). This means that we are calculating audio levels whether
// or not we are sending samples.
// TODO(https://crbug.com/webrtc/10771): All "media-source" related stats
// should move from send-streams to the local audio sources or tracks; a
// send-stream should not be required to read the microphone audio levels.
MutexLock lock(&audio_level_lock_);
audio_level_.ComputeLevel(*audio_frame, duration);
}
channel_send_->ProcessAndEncodeAudio(std::move(audio_frame));
}
bool AudioSendStream::SendTelephoneEvent(int payload_type,
int payload_frequency,
int event,
int duration_ms) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_send_->SetSendTelephoneEventPayloadType(payload_type,
payload_frequency);
return channel_send_->SendTelephoneEventOutband(event, duration_ms);
}
void AudioSendStream::SetMuted(bool muted) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_send_->SetInputMute(muted);
}
webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
return GetStats(true);
}
webrtc::AudioSendStream::Stats AudioSendStream::GetStats(
bool has_remote_tracks) const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
webrtc::AudioSendStream::Stats stats;
stats.local_ssrc = config_.rtp.ssrc;
stats.target_bitrate_bps = channel_send_->GetTargetBitrate();
webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics();
stats.payload_bytes_sent = call_stats.payload_bytes_sent;
stats.header_and_padding_bytes_sent =
call_stats.header_and_padding_bytes_sent;
stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent;
stats.packets_sent = call_stats.packetsSent;
stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent;
// RTT isn't known until a RTCP report is received. Until then, VoiceEngine
// returns 0 to indicate an error value.
if (call_stats.rttMs > 0) {
stats.rtt_ms = call_stats.rttMs;
}
if (config_.send_codec_spec) {
const auto& spec = *config_.send_codec_spec;
stats.codec_name = spec.format.name;
stats.codec_payload_type = spec.payload_type;
// Get data from the last remote RTCP report.
for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) {
// Lookup report for send ssrc only.
if (block.source_SSRC == stats.local_ssrc) {
stats.packets_lost = block.cumulative_num_packets_lost;
stats.fraction_lost = Q8ToFloat(block.fraction_lost);
// Convert timestamps to milliseconds.
if (spec.format.clockrate_hz / 1000 > 0) {
stats.jitter_ms =
block.interarrival_jitter / (spec.format.clockrate_hz / 1000);
}
break;
}
}
}
{
MutexLock lock(&audio_level_lock_);
stats.audio_level = audio_level_.LevelFullRange();
stats.total_input_energy = audio_level_.TotalEnergy();
stats.total_input_duration = audio_level_.TotalDuration();
}
stats.typing_noise_detected = audio_state()->typing_noise_detected();
stats.ana_statistics = channel_send_->GetANAStatistics();
AudioProcessing* ap = audio_state_->audio_processing();
if (ap) {
stats.apm_statistics = ap->GetStatistics(has_remote_tracks);
}
stats.report_block_datas = std::move(call_stats.report_block_datas);
stats.nacks_rcvd = call_stats.nacks_rcvd;
return stats;
}
void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_send_->ReceivedRTCPPacket(packet, length);
{
// Poll if overhead has changed, which it can do if ack triggers us to stop
// sending mid/rid.
MutexLock lock(&overhead_per_packet_lock_);
UpdateOverheadForEncoder();
}
UpdateCachedTargetAudioBitrateConstraints();
}
uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
RTC_DCHECK_RUN_ON(rtp_transport_queue_);
// Pick a target bitrate between the constraints. Overrules the allocator if
// it 1) allocated a bitrate of zero to disable the stream or 2) allocated a
// higher than max to allow for e.g. extra FEC.
RTC_DCHECK(cached_constraints_.has_value());
update.target_bitrate.Clamp(cached_constraints_->min,
cached_constraints_->max);
update.stable_target_bitrate.Clamp(cached_constraints_->min,
cached_constraints_->max);
channel_send_->OnBitrateAllocation(update);
// The amount of audio protection is not exposed by the encoder, hence
// always returning 0.
return 0;
}
void AudioSendStream::SetTransportOverhead(
int transport_overhead_per_packet_bytes) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
{
MutexLock lock(&overhead_per_packet_lock_);
transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes;
UpdateOverheadForEncoder();
}
UpdateCachedTargetAudioBitrateConstraints();
}
void AudioSendStream::UpdateOverheadForEncoder() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes();
if (overhead_per_packet_ == overhead_per_packet_bytes) {
return;
}
overhead_per_packet_ = overhead_per_packet_bytes;
channel_send_->CallEncoder([&](AudioEncoder* encoder) {
encoder->OnReceivedOverhead(overhead_per_packet_bytes);
});
if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) {
total_packet_overhead_bytes_ = overhead_per_packet_bytes;
if (registered_with_allocator_) {
ConfigureBitrateObserver();
}
}
}
size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const {
MutexLock lock(&overhead_per_packet_lock_);
return GetPerPacketOverheadBytes();
}
size_t AudioSendStream::GetPerPacketOverheadBytes() const {
return transport_overhead_per_packet_bytes_ +
rtp_rtcp_module_->ExpectedPerPacketOverhead();
}
RtpState AudioSendStream::GetRtpState() const {
return rtp_rtcp_module_->GetRtpState();
}
const voe::ChannelSendInterface* AudioSendStream::GetChannel() const {
return channel_send_.get();
}
internal::AudioState* AudioSendStream::audio_state() {
internal::AudioState* audio_state =
static_cast<internal::AudioState*>(audio_state_.get());
RTC_DCHECK(audio_state);
return audio_state;
}
const internal::AudioState* AudioSendStream::audio_state() const {
internal::AudioState* audio_state =
static_cast<internal::AudioState*>(audio_state_.get());
RTC_DCHECK(audio_state);
return audio_state;
}
void AudioSendStream::StoreEncoderProperties(int sample_rate_hz,
size_t num_channels) {
encoder_sample_rate_hz_ = sample_rate_hz;
encoder_num_channels_ = num_channels;
if (sending_) {
// Update AudioState's information about the stream.
audio_state()->AddSendingStream(this, sample_rate_hz, num_channels);
}
}
// Apply current codec settings to a single voe::Channel used for sending.
bool AudioSendStream::SetupSendCodec(const Config& new_config) {
RTC_DCHECK(new_config.send_codec_spec);
const auto& spec = *new_config.send_codec_spec;
RTC_DCHECK(new_config.encoder_factory);
std::unique_ptr<AudioEncoder> encoder =
new_config.encoder_factory->MakeAudioEncoder(
spec.payload_type, spec.format, new_config.codec_pair_id);
if (!encoder) {
RTC_DLOG(LS_ERROR) << "Unable to create encoder for "
<< rtc::ToString(spec.format);
return false;
}
// If a bitrate has been specified for the codec, use it over the
// codec's default.
if (spec.target_bitrate_bps) {
encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
}
// Enable ANA if configured (currently only used by Opus).
if (new_config.audio_network_adaptor_config) {
if (encoder->EnableAudioNetworkAdaptor(
*new_config.audio_network_adaptor_config, event_log_)) {
RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
<< new_config.rtp.ssrc;
} else {
RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
<< new_config.rtp.ssrc;
}
}
// Wrap the encoder in an AudioEncoderCNG, if VAD is enabled.
if (spec.cng_payload_type) {
AudioEncoderCngConfig cng_config;
cng_config.num_channels = encoder->NumChannels();
cng_config.payload_type = *spec.cng_payload_type;
cng_config.speech_encoder = std::move(encoder);
cng_config.vad_mode = Vad::kVadNormal;
encoder = CreateComfortNoiseEncoder(std::move(cng_config));
RegisterCngPayloadType(*spec.cng_payload_type,
new_config.send_codec_spec->format.clockrate_hz);
}
// Wrap the encoder in a RED encoder, if RED is enabled.
if (spec.red_payload_type) {
AudioEncoderCopyRed::Config red_config;
red_config.payload_type = *spec.red_payload_type;
red_config.speech_encoder = std::move(encoder);
encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config));
}
// Set currently known overhead (used in ANA, opus only).
// If overhead changes later, it will be updated in UpdateOverheadForEncoder.
{
MutexLock lock(&overhead_per_packet_lock_);
size_t overhead = GetPerPacketOverheadBytes();
if (overhead > 0) {
encoder->OnReceivedOverhead(overhead);
}
}
StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
std::move(encoder));
return true;
}
bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) {
const auto& old_config = config_;
if (!new_config.send_codec_spec) {
// We cannot de-configure a send codec. So we will do nothing.
// By design, the send codec should have not been configured.
RTC_DCHECK(!old_config.send_codec_spec);
return true;
}
if (new_config.send_codec_spec == old_config.send_codec_spec &&
new_config.audio_network_adaptor_config ==
old_config.audio_network_adaptor_config) {
return true;
}
// If we have no encoder, or the format or payload type's changed, create a
// new encoder.
if (!old_config.send_codec_spec ||
new_config.send_codec_spec->format !=
old_config.send_codec_spec->format ||
new_config.send_codec_spec->payload_type !=
old_config.send_codec_spec->payload_type ||
new_config.send_codec_spec->red_payload_type !=
old_config.send_codec_spec->red_payload_type) {
return SetupSendCodec(new_config);
}
const absl::optional<int>& new_target_bitrate_bps =
new_config.send_codec_spec->target_bitrate_bps;
// If a bitrate has been specified for the codec, use it over the
// codec's default.
if (new_target_bitrate_bps &&
new_target_bitrate_bps !=
old_config.send_codec_spec->target_bitrate_bps) {
channel_send_->CallEncoder([&](AudioEncoder* encoder) {
encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps);
});
}
ReconfigureANA(new_config);
ReconfigureCNG(new_config);
return true;
}
void AudioSendStream::ReconfigureANA(const Config& new_config) {
if (new_config.audio_network_adaptor_config ==
config_.audio_network_adaptor_config) {
return;
}
if (new_config.audio_network_adaptor_config) {
// This lock needs to be acquired before CallEncoder, since it aquires
// another lock and we need to maintain the same order at all call sites to
// avoid deadlock.
MutexLock lock(&overhead_per_packet_lock_);
size_t overhead = GetPerPacketOverheadBytes();
channel_send_->CallEncoder([&](AudioEncoder* encoder) {
if (encoder->EnableAudioNetworkAdaptor(
*new_config.audio_network_adaptor_config, event_log_)) {
RTC_LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
<< new_config.rtp.ssrc;
if (overhead > 0) {
encoder->OnReceivedOverhead(overhead);
}
} else {
RTC_LOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC "
<< new_config.rtp.ssrc;
}
});
} else {
channel_send_->CallEncoder(
[&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); });
RTC_LOG(LS_INFO) << "Audio network adaptor disabled on SSRC "
<< new_config.rtp.ssrc;
}
}
void AudioSendStream::ReconfigureCNG(const Config& new_config) {
if (new_config.send_codec_spec->cng_payload_type ==
config_.send_codec_spec->cng_payload_type) {
return;
}
// Register the CNG payload type if it's been added, don't do anything if CNG
// is removed. Payload types must not be redefined.
if (new_config.send_codec_spec->cng_payload_type) {
RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type,
new_config.send_codec_spec->format.clockrate_hz);
}
// Wrap or unwrap the encoder in an AudioEncoderCNG.
channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr));
auto sub_encoders = old_encoder->ReclaimContainedEncoders();
if (!sub_encoders.empty()) {
// Replace enc with its sub encoder. We need to put the sub
// encoder in a temporary first, since otherwise the old value
// of enc would be destroyed before the new value got assigned,
// which would be bad since the new value is a part of the old
// value.
auto tmp = std::move(sub_encoders[0]);
old_encoder = std::move(tmp);
}
if (new_config.send_codec_spec->cng_payload_type) {
AudioEncoderCngConfig config;
config.speech_encoder = std::move(old_encoder);
config.num_channels = config.speech_encoder->NumChannels();
config.payload_type = *new_config.send_codec_spec->cng_payload_type;
config.vad_mode = Vad::kVadNormal;
*encoder_ptr = CreateComfortNoiseEncoder(std::move(config));
} else {
*encoder_ptr = std::move(old_encoder);
}
});
}
void AudioSendStream::ReconfigureBitrateObserver(
const webrtc::AudioSendStream::Config& new_config) {
// Since the Config's default is for both of these to be -1, this test will
// allow us to configure the bitrate observer if the new config has bitrate
// limits set, but would only have us call RemoveBitrateObserver if we were
// previously configured with bitrate limits.
if (config_.min_bitrate_bps == new_config.min_bitrate_bps &&
config_.max_bitrate_bps == new_config.max_bitrate_bps &&
config_.bitrate_priority == new_config.bitrate_priority &&
TransportSeqNumId(config_) == TransportSeqNumId(new_config) &&
config_.audio_network_adaptor_config ==
new_config.audio_network_adaptor_config) {
return;
}
if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 &&
new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) {
rtp_transport_->AccountForAudioPacketsInPacedSender(true);
if (send_side_bwe_with_overhead_)
rtp_transport_->IncludeOverheadInPacedSender();
// We may get a callback immediately as the observer is registered, so
// make sure the bitrate limits in config_ are up-to-date.
config_.min_bitrate_bps = new_config.min_bitrate_bps;
config_.max_bitrate_bps = new_config.max_bitrate_bps;
config_.bitrate_priority = new_config.bitrate_priority;
ConfigureBitrateObserver();
rtp_rtcp_module_->SetAsPartOfAllocation(true);
} else {
rtp_transport_->AccountForAudioPacketsInPacedSender(false);
RemoveBitrateObserver();
rtp_rtcp_module_->SetAsPartOfAllocation(false);
}
}
void AudioSendStream::ConfigureBitrateObserver() {
// This either updates the current observer or adds a new observer.
// TODO(srte): Add overhead compensation here.
auto constraints = GetMinMaxBitrateConstraints();
RTC_DCHECK(constraints.has_value());
DataRate priority_bitrate = allocation_settings_.priority_bitrate;
if (send_side_bwe_with_overhead_) {
if (use_legacy_overhead_calculation_) {
// OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
const TimeDelta kMinPacketDuration = TimeDelta::Millis(20);
DataRate max_overhead =
DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration;
priority_bitrate += max_overhead;
} else {
RTC_DCHECK(frame_length_range_);
const DataSize overhead_per_packet =
DataSize::Bytes(total_packet_overhead_bytes_);
DataRate min_overhead = overhead_per_packet / frame_length_range_->second;
priority_bitrate += min_overhead;
}
}
if (allocation_settings_.priority_bitrate_raw)
priority_bitrate = *allocation_settings_.priority_bitrate_raw;
rtp_transport_queue_->PostTask([this, constraints, priority_bitrate,
config_bitrate_priority =
config_.bitrate_priority] {
RTC_DCHECK_RUN_ON(rtp_transport_queue_);
bitrate_allocator_->AddObserver(
this,
MediaStreamAllocationConfig{
constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(),
0, priority_bitrate.bps(), true,
allocation_settings_.bitrate_priority.value_or(
config_bitrate_priority)});
});
registered_with_allocator_ = true;
}
void AudioSendStream::RemoveBitrateObserver() {
registered_with_allocator_ = false;
rtc::Event thread_sync_event;
rtp_transport_queue_->PostTask([this, &thread_sync_event] {
RTC_DCHECK_RUN_ON(rtp_transport_queue_);
bitrate_allocator_->RemoveObserver(this);
thread_sync_event.Set();
});
thread_sync_event.Wait(rtc::Event::kForever);
}
absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
AudioSendStream::GetMinMaxBitrateConstraints() const {
if (config_.min_bitrate_bps < 0 || config_.max_bitrate_bps < 0) {
RTC_LOG(LS_WARNING) << "Config is invalid: min_bitrate_bps="
<< config_.min_bitrate_bps
<< "; max_bitrate_bps=" << config_.max_bitrate_bps
<< "; both expected greater or equal to 0";
return absl::nullopt;
}
TargetAudioBitrateConstraints constraints{
DataRate::BitsPerSec(config_.min_bitrate_bps),
DataRate::BitsPerSec(config_.max_bitrate_bps)};
// If bitrates were explicitly overriden via field trial, use those values.
if (allocation_settings_.min_bitrate)
constraints.min = *allocation_settings_.min_bitrate;
if (allocation_settings_.max_bitrate)
constraints.max = *allocation_settings_.max_bitrate;
RTC_DCHECK_GE(constraints.min, DataRate::Zero());
RTC_DCHECK_GE(constraints.max, DataRate::Zero());
if (constraints.max < constraints.min) {
RTC_LOG(LS_WARNING) << "TargetAudioBitrateConstraints::max is less than "
<< "TargetAudioBitrateConstraints::min";
return absl::nullopt;
}
if (send_side_bwe_with_overhead_) {
if (use_legacy_overhead_calculation_) {
// OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12);
const TimeDelta kMaxFrameLength =
TimeDelta::Millis(60); // Based on Opus spec
const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength;
constraints.min += kMinOverhead;
constraints.max += kMinOverhead;
} else {
if (!frame_length_range_.has_value()) {
RTC_LOG(LS_WARNING) << "frame_length_range_ is not set";
return absl::nullopt;
}
const DataSize kOverheadPerPacket =
DataSize::Bytes(total_packet_overhead_bytes_);
constraints.min += kOverheadPerPacket / frame_length_range_->second;
constraints.max += kOverheadPerPacket / frame_length_range_->first;
}
}
return constraints;
}
void AudioSendStream::RegisterCngPayloadType(int payload_type,
int clockrate_hz) {
channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz);
}
void AudioSendStream::UpdateCachedTargetAudioBitrateConstraints() {
absl::optional<AudioSendStream::TargetAudioBitrateConstraints>
new_constraints = GetMinMaxBitrateConstraints();
if (!new_constraints.has_value()) {
return;
}
rtp_transport_queue_->PostTask([this, new_constraints]() {
RTC_DCHECK_RUN_ON(rtp_transport_queue_);
cached_constraints_ = new_constraints;
});
}
} // namespace internal
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink