/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/audio_mixer/audio_mixer_impl.h" #include #include #include #include #include #include "modules/audio_mixer/audio_frame_manipulator.h" #include "modules/audio_mixer/default_output_rate_calculator.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" namespace webrtc { namespace { struct SourceFrame { SourceFrame(AudioMixerImpl::SourceStatus* source_status, AudioFrame* audio_frame, bool muted) : source_status(source_status), audio_frame(audio_frame), muted(muted) { RTC_DCHECK(source_status); RTC_DCHECK(audio_frame); if (!muted) { energy = AudioMixerCalculateEnergy(*audio_frame); } } SourceFrame(AudioMixerImpl::SourceStatus* source_status, AudioFrame* audio_frame, bool muted, uint32_t energy) : source_status(source_status), audio_frame(audio_frame), muted(muted), energy(energy) { RTC_DCHECK(source_status); RTC_DCHECK(audio_frame); } AudioMixerImpl::SourceStatus* source_status = nullptr; AudioFrame* audio_frame = nullptr; bool muted = true; uint32_t energy = 0; }; // ShouldMixBefore(a, b) is used to select mixer sources. bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) { if (a.muted != b.muted) { return b.muted; } const auto a_activity = a.audio_frame->vad_activity_; const auto b_activity = b.audio_frame->vad_activity_; if (a_activity != b_activity) { return a_activity == AudioFrame::kVadActive; } return a.energy > b.energy; } void RampAndUpdateGain( const std::vector& mixed_sources_and_frames) { for (const auto& source_frame : mixed_sources_and_frames) { float target_gain = source_frame.source_status->is_mixed ? 1.0f : 0.0f; Ramp(source_frame.source_status->gain, target_gain, source_frame.audio_frame); source_frame.source_status->gain = target_gain; } } AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList( AudioMixerImpl::Source const* audio_source, AudioMixerImpl::SourceStatusList const* audio_source_list) { return std::find_if( audio_source_list->begin(), audio_source_list->end(), [audio_source](const std::unique_ptr& p) { return p->audio_source == audio_source; }); } } // namespace AudioMixerImpl::AudioMixerImpl( std::unique_ptr output_rate_calculator, bool use_limiter) : output_rate_calculator_(std::move(output_rate_calculator)), output_frequency_(0), sample_size_(0), audio_source_list_(), frame_combiner_(use_limiter) {} AudioMixerImpl::~AudioMixerImpl() {} rtc::scoped_refptr AudioMixerImpl::Create() { return Create(std::unique_ptr( new DefaultOutputRateCalculator()), true); } rtc::scoped_refptr AudioMixerImpl::Create( std::unique_ptr output_rate_calculator, bool use_limiter) { return rtc::scoped_refptr( new rtc::RefCountedObject( std::move(output_rate_calculator), use_limiter)); } void AudioMixerImpl::Mix(size_t number_of_channels, AudioFrame* audio_frame_for_mixing) { RTC_DCHECK(number_of_channels >= 1); RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); CalculateOutputFrequency(); { MutexLock lock(&mutex_); const size_t number_of_streams = audio_source_list_.size(); frame_combiner_.Combine(GetAudioFromSources(), number_of_channels, OutputFrequency(), number_of_streams, audio_frame_for_mixing); } return; } void AudioMixerImpl::CalculateOutputFrequency() { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); MutexLock lock(&mutex_); std::vector preferred_rates; std::transform(audio_source_list_.begin(), audio_source_list_.end(), std::back_inserter(preferred_rates), [&](std::unique_ptr& a) { return a->audio_source->PreferredSampleRate(); }); output_frequency_ = output_rate_calculator_->CalculateOutputRate(preferred_rates); sample_size_ = (output_frequency_ * kFrameDurationInMs) / 1000; } int AudioMixerImpl::OutputFrequency() const { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); return output_frequency_; } bool AudioMixerImpl::AddSource(Source* audio_source) { RTC_DCHECK(audio_source); MutexLock lock(&mutex_); RTC_DCHECK(FindSourceInList(audio_source, &audio_source_list_) == audio_source_list_.end()) << "Source already added to mixer"; audio_source_list_.emplace_back(new SourceStatus(audio_source, false, 0)); return true; } void AudioMixerImpl::RemoveSource(Source* audio_source) { RTC_DCHECK(audio_source); MutexLock lock(&mutex_); const auto iter = FindSourceInList(audio_source, &audio_source_list_); RTC_DCHECK(iter != audio_source_list_.end()) << "Source not present in mixer"; audio_source_list_.erase(iter); } AudioFrameList AudioMixerImpl::GetAudioFromSources() { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); AudioFrameList result; std::vector audio_source_mixing_data_list; std::vector ramp_list; // Get audio from the audio sources and put it in the SourceFrame vector. for (auto& source_and_status : audio_source_list_) { const auto audio_frame_info = source_and_status->audio_source->GetAudioFrameWithInfo( OutputFrequency(), &source_and_status->audio_frame); if (audio_frame_info == Source::AudioFrameInfo::kError) { RTC_LOG_F(LS_WARNING) << "failed to GetAudioFrameWithInfo() from source"; continue; } audio_source_mixing_data_list.emplace_back( source_and_status.get(), &source_and_status->audio_frame, audio_frame_info == Source::AudioFrameInfo::kMuted); } // Sort frames by sorting function. std::sort(audio_source_mixing_data_list.begin(), audio_source_mixing_data_list.end(), ShouldMixBefore); int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources; // Go through list in order and put unmuted frames in result list. for (const auto& p : audio_source_mixing_data_list) { // Filter muted. if (p.muted) { p.source_status->is_mixed = false; continue; } // Add frame to result vector for mixing. bool is_mixed = false; if (max_audio_frame_counter > 0) { --max_audio_frame_counter; result.push_back(p.audio_frame); ramp_list.emplace_back(p.source_status, p.audio_frame, false, -1); is_mixed = true; } p.source_status->is_mixed = is_mixed; } RampAndUpdateGain(ramp_list); return result; } bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest( AudioMixerImpl::Source* audio_source) const { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); MutexLock lock(&mutex_); const auto iter = FindSourceInList(audio_source, &audio_source_list_); if (iter != audio_source_list_.end()) { return (*iter)->is_mixed; } RTC_LOG(LS_ERROR) << "Audio source unknown"; return false; } } // namespace webrtc