| // Copyright 2013 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "content/renderer/media/media_stream_audio_processor.h" |
| |
| #include "base/command_line.h" |
| #include "base/debug/trace_event.h" |
| #include "base/metrics/histogram.h" |
| #include "content/public/common/content_switches.h" |
| #include "content/renderer/media/media_stream_audio_processor_options.h" |
| #include "content/renderer/media/rtc_media_constraints.h" |
| #include "content/renderer/media/webrtc_audio_device_impl.h" |
| #include "media/audio/audio_parameters.h" |
| #include "media/base/audio_converter.h" |
| #include "media/base/audio_fifo.h" |
| #include "media/base/channel_layout.h" |
| #include "third_party/WebKit/public/platform/WebMediaConstraints.h" |
| #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface.h" |
| #include "third_party/webrtc/modules/audio_processing/typing_detection.h" |
| |
| namespace content { |
| |
| namespace { |
| |
| using webrtc::AudioProcessing; |
| |
| #if defined(OS_ANDROID) |
| const int kAudioProcessingSampleRate = 16000; |
| #else |
| const int kAudioProcessingSampleRate = 32000; |
| #endif |
| const int kAudioProcessingNumberOfChannels = 1; |
| const AudioProcessing::ChannelLayout kAudioProcessingChannelLayout = |
| AudioProcessing::kMono; |
| |
| const int kMaxNumberOfBuffersInFifo = 2; |
| |
| // Used by UMA histograms and entries shouldn't be re-ordered or removed. |
| enum AudioTrackProcessingStates { |
| AUDIO_PROCESSING_ENABLED = 0, |
| AUDIO_PROCESSING_DISABLED, |
| AUDIO_PROCESSING_IN_WEBRTC, |
| AUDIO_PROCESSING_MAX |
| }; |
| |
| void RecordProcessingState(AudioTrackProcessingStates state) { |
| UMA_HISTOGRAM_ENUMERATION("Media.AudioTrackProcessingStates", |
| state, AUDIO_PROCESSING_MAX); |
| } |
| |
| } // namespace |
| |
| class MediaStreamAudioProcessor::MediaStreamAudioConverter |
| : public media::AudioConverter::InputCallback { |
| public: |
| MediaStreamAudioConverter(const media::AudioParameters& source_params, |
| const media::AudioParameters& sink_params) |
| : source_params_(source_params), |
| sink_params_(sink_params), |
| audio_converter_(source_params, sink_params_, false) { |
| // An instance of MediaStreamAudioConverter may be created in the main |
| // render thread and used in the audio thread, for example, the |
| // |MediaStreamAudioProcessor::capture_converter_|. |
| thread_checker_.DetachFromThread(); |
| audio_converter_.AddInput(this); |
| |
| // Create and initialize audio fifo and audio bus wrapper. |
| // The size of the FIFO should be at least twice of the source buffer size |
| // or twice of the sink buffer size. Also, FIFO needs to have enough space |
| // to store pre-processed data before passing the data to |
| // webrtc::AudioProcessing, which requires 10ms as packet size. |
| int max_frame_size = std::max(source_params_.frames_per_buffer(), |
| sink_params_.frames_per_buffer()); |
| int buffer_size = std::max( |
| kMaxNumberOfBuffersInFifo * max_frame_size, |
| kMaxNumberOfBuffersInFifo * source_params_.sample_rate() / 100); |
| fifo_.reset(new media::AudioFifo(source_params_.channels(), buffer_size)); |
| |
| // TODO(xians): Use CreateWrapper to save one memcpy. |
| audio_wrapper_ = media::AudioBus::Create(sink_params_.channels(), |
| sink_params_.frames_per_buffer()); |
| } |
| |
| virtual ~MediaStreamAudioConverter() { |
| audio_converter_.RemoveInput(this); |
| } |
| |
| void Push(const media::AudioBus* audio_source) { |
| // Called on the audio thread, which is the capture audio thread for |
| // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread |
| // for |MediaStreamAudioProcessor::render_converter_|. |
| // And it must be the same thread as calling Convert(). |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| fifo_->Push(audio_source); |
| } |
| |
| bool Convert(webrtc::AudioFrame* out, bool audio_mirroring) { |
| // Called on the audio thread, which is the capture audio thread for |
| // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread |
| // for |MediaStreamAudioProcessor::render_converter_|. |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| // Return false if there is not enough data in the FIFO, this happens when |
| // fifo_->frames() / source_params_.sample_rate() is less than |
| // sink_params.frames_per_buffer() / sink_params.sample_rate(). |
| if (fifo_->frames() * sink_params_.sample_rate() < |
| sink_params_.frames_per_buffer() * source_params_.sample_rate()) { |
| return false; |
| } |
| |
| // Convert data to the output format, this will trigger ProvideInput(). |
| audio_converter_.Convert(audio_wrapper_.get()); |
| DCHECK_EQ(audio_wrapper_->frames(), sink_params_.frames_per_buffer()); |
| |
| // Swap channels before interleaving the data if |audio_mirroring| is |
| // set to true. |
| if (audio_mirroring && |
| sink_params_.channel_layout() == media::CHANNEL_LAYOUT_STEREO) { |
| // Swap the first and second channels. |
| audio_wrapper_->SwapChannels(0, 1); |
| } |
| |
| // TODO(xians): Figure out a better way to handle the interleaved and |
| // deinterleaved format switching. |
| audio_wrapper_->ToInterleaved(audio_wrapper_->frames(), |
| sink_params_.bits_per_sample() / 8, |
| out->data_); |
| |
| out->samples_per_channel_ = sink_params_.frames_per_buffer(); |
| out->sample_rate_hz_ = sink_params_.sample_rate(); |
| out->speech_type_ = webrtc::AudioFrame::kNormalSpeech; |
| out->vad_activity_ = webrtc::AudioFrame::kVadUnknown; |
| out->num_channels_ = sink_params_.channels(); |
| |
| return true; |
| } |
| |
| const media::AudioParameters& source_parameters() const { |
| return source_params_; |
| } |
| const media::AudioParameters& sink_parameters() const { |
| return sink_params_; |
| } |
| |
| private: |
| // AudioConverter::InputCallback implementation. |
| virtual double ProvideInput(media::AudioBus* audio_bus, |
| base::TimeDelta buffer_delay) OVERRIDE { |
| // Called on realtime audio thread. |
| // TODO(xians): Figure out why the first Convert() triggers ProvideInput |
| // two times. |
| if (fifo_->frames() < audio_bus->frames()) |
| return 0; |
| |
| fifo_->Consume(audio_bus, 0, audio_bus->frames()); |
| |
| // Return 1.0 to indicate no volume scaling on the data. |
| return 1.0; |
| } |
| |
| base::ThreadChecker thread_checker_; |
| const media::AudioParameters source_params_; |
| const media::AudioParameters sink_params_; |
| |
| // TODO(xians): consider using SincResampler to save some memcpy. |
| // Handles mixing and resampling between input and output parameters. |
| media::AudioConverter audio_converter_; |
| scoped_ptr<media::AudioBus> audio_wrapper_; |
| scoped_ptr<media::AudioFifo> fifo_; |
| }; |
| |
| bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { |
| return !CommandLine::ForCurrentProcess()->HasSwitch( |
| switches::kDisableAudioTrackProcessing); |
| } |
| |
| MediaStreamAudioProcessor::MediaStreamAudioProcessor( |
| const blink::WebMediaConstraints& constraints, |
| int effects, |
| WebRtcPlayoutDataSource* playout_data_source) |
| : render_delay_ms_(0), |
| playout_data_source_(playout_data_source), |
| audio_mirroring_(false), |
| typing_detected_(false), |
| stopped_(false) { |
| capture_thread_checker_.DetachFromThread(); |
| render_thread_checker_.DetachFromThread(); |
| InitializeAudioProcessingModule(constraints, effects); |
| if (IsAudioTrackProcessingEnabled()) { |
| aec_dump_message_filter_ = AecDumpMessageFilter::Get(); |
| // In unit tests not creating a message filter, |aec_dump_message_filter_| |
| // will be NULL. We can just ignore that. Other unit tests and browser tests |
| // ensure that we do get the filter when we should. |
| if (aec_dump_message_filter_) |
| aec_dump_message_filter_->AddDelegate(this); |
| } |
| } |
| |
| MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| Stop(); |
| } |
| |
| void MediaStreamAudioProcessor::OnCaptureFormatChanged( |
| const media::AudioParameters& source_params) { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| // There is no need to hold a lock here since the caller guarantees that |
| // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks |
| // on the capture thread. |
| InitializeCaptureConverter(source_params); |
| |
| // Reset the |capture_thread_checker_| since the capture data will come from |
| // a new capture thread. |
| capture_thread_checker_.DetachFromThread(); |
| } |
| |
| void MediaStreamAudioProcessor::PushCaptureData( |
| const media::AudioBus* audio_source) { |
| DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| DCHECK_EQ(audio_source->channels(), |
| capture_converter_->source_parameters().channels()); |
| DCHECK_EQ(audio_source->frames(), |
| capture_converter_->source_parameters().frames_per_buffer()); |
| |
| capture_converter_->Push(audio_source); |
| } |
| |
| bool MediaStreamAudioProcessor::ProcessAndConsumeData( |
| base::TimeDelta capture_delay, int volume, bool key_pressed, |
| int* new_volume, int16** out) { |
| DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData"); |
| |
| if (!capture_converter_->Convert(&capture_frame_, audio_mirroring_)) |
| return false; |
| |
| *new_volume = ProcessData(&capture_frame_, capture_delay, volume, |
| key_pressed); |
| *out = capture_frame_.data_; |
| |
| return true; |
| } |
| |
| void MediaStreamAudioProcessor::Stop() { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| if (stopped_) |
| return; |
| |
| stopped_ = true; |
| |
| if (aec_dump_message_filter_) { |
| aec_dump_message_filter_->RemoveDelegate(this); |
| aec_dump_message_filter_ = NULL; |
| } |
| |
| if (!audio_processing_.get()) |
| return; |
| |
| StopEchoCancellationDump(audio_processing_.get()); |
| |
| if (playout_data_source_) { |
| playout_data_source_->RemovePlayoutSink(this); |
| playout_data_source_ = NULL; |
| } |
| } |
| |
| const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { |
| return capture_converter_->source_parameters(); |
| } |
| |
| const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { |
| return capture_converter_->sink_parameters(); |
| } |
| |
| void MediaStreamAudioProcessor::OnAecDumpFile( |
| const IPC::PlatformFileForTransit& file_handle) { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| |
| base::File file = IPC::PlatformFileForTransitToFile(file_handle); |
| DCHECK(file.IsValid()); |
| |
| if (audio_processing_) |
| StartEchoCancellationDump(audio_processing_.get(), file.Pass()); |
| else |
| file.Close(); |
| } |
| |
| void MediaStreamAudioProcessor::OnDisableAecDump() { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| if (audio_processing_) |
| StopEchoCancellationDump(audio_processing_.get()); |
| } |
| |
| void MediaStreamAudioProcessor::OnIpcClosing() { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| aec_dump_message_filter_ = NULL; |
| } |
| |
| void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus, |
| int sample_rate, |
| int audio_delay_milliseconds) { |
| DCHECK(render_thread_checker_.CalledOnValidThread()); |
| DCHECK(audio_processing_->echo_control_mobile()->is_enabled() ^ |
| audio_processing_->echo_cancellation()->is_enabled()); |
| |
| TRACE_EVENT0("audio", "MediaStreamAudioProcessor::OnPlayoutData"); |
| DCHECK_LT(audio_delay_milliseconds, |
| std::numeric_limits<base::subtle::Atomic32>::max()); |
| base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds); |
| |
| InitializeRenderConverterIfNeeded(sample_rate, audio_bus->channels(), |
| audio_bus->frames()); |
| |
| render_converter_->Push(audio_bus); |
| while (render_converter_->Convert(&render_frame_, false)) |
| audio_processing_->AnalyzeReverseStream(&render_frame_); |
| } |
| |
| void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| // There is no need to hold a lock here since the caller guarantees that |
| // there is no more OnPlayoutData() callback on the render thread. |
| render_thread_checker_.DetachFromThread(); |
| render_converter_.reset(); |
| } |
| |
| void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { |
| stats->typing_noise_detected = |
| (base::subtle::Acquire_Load(&typing_detected_) != false); |
| GetAecStats(audio_processing_.get(), stats); |
| } |
| |
| void MediaStreamAudioProcessor::InitializeAudioProcessingModule( |
| const blink::WebMediaConstraints& constraints, int effects) { |
| DCHECK(!audio_processing_); |
| |
| MediaAudioConstraints audio_constraints(constraints, effects); |
| |
| // Audio mirroring can be enabled even though audio processing is otherwise |
| // disabled. |
| audio_mirroring_ = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogAudioMirroring); |
| |
| if (!IsAudioTrackProcessingEnabled()) { |
| RecordProcessingState(AUDIO_PROCESSING_IN_WEBRTC); |
| return; |
| } |
| |
| #if defined(OS_IOS) |
| // On iOS, VPIO provides built-in AGC and AEC. |
| const bool echo_cancellation = false; |
| const bool goog_agc = false; |
| #else |
| const bool echo_cancellation = |
| audio_constraints.GetEchoCancellationProperty(); |
| const bool goog_agc = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogAutoGainControl); |
| #endif |
| |
| #if defined(OS_IOS) || defined(OS_ANDROID) |
| const bool goog_experimental_aec = false; |
| const bool goog_typing_detection = false; |
| #else |
| const bool goog_experimental_aec = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogExperimentalEchoCancellation); |
| const bool goog_typing_detection = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogTypingNoiseDetection); |
| #endif |
| |
| const bool goog_ns = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogNoiseSuppression); |
| const bool goog_experimental_ns = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogExperimentalNoiseSuppression); |
| const bool goog_high_pass_filter = audio_constraints.GetProperty( |
| MediaAudioConstraints::kGoogHighpassFilter); |
| |
| // Return immediately if no goog constraint is enabled. |
| if (!echo_cancellation && !goog_experimental_aec && !goog_ns && |
| !goog_high_pass_filter && !goog_typing_detection && |
| !goog_agc && !goog_experimental_ns) { |
| RecordProcessingState(AUDIO_PROCESSING_DISABLED); |
| return; |
| } |
| |
| // Create and configure the webrtc::AudioProcessing. |
| audio_processing_.reset(webrtc::AudioProcessing::Create()); |
| CHECK_EQ(0, audio_processing_->Initialize(kAudioProcessingSampleRate, |
| kAudioProcessingSampleRate, |
| kAudioProcessingSampleRate, |
| kAudioProcessingChannelLayout, |
| kAudioProcessingChannelLayout, |
| kAudioProcessingChannelLayout)); |
| |
| // Enable the audio processing components. |
| if (echo_cancellation) { |
| EnableEchoCancellation(audio_processing_.get()); |
| |
| if (goog_experimental_aec) |
| EnableExperimentalEchoCancellation(audio_processing_.get()); |
| |
| if (playout_data_source_) |
| playout_data_source_->AddPlayoutSink(this); |
| } |
| |
| if (goog_ns) |
| EnableNoiseSuppression(audio_processing_.get()); |
| |
| if (goog_experimental_ns) |
| EnableExperimentalNoiseSuppression(audio_processing_.get()); |
| |
| if (goog_high_pass_filter) |
| EnableHighPassFilter(audio_processing_.get()); |
| |
| if (goog_typing_detection) { |
| // TODO(xians): Remove this |typing_detector_| after the typing suppression |
| // is enabled by default. |
| typing_detector_.reset(new webrtc::TypingDetection()); |
| EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); |
| } |
| |
| if (goog_agc) |
| EnableAutomaticGainControl(audio_processing_.get()); |
| |
| RecordProcessingState(AUDIO_PROCESSING_ENABLED); |
| } |
| |
| void MediaStreamAudioProcessor::InitializeCaptureConverter( |
| const media::AudioParameters& source_params) { |
| DCHECK(main_thread_checker_.CalledOnValidThread()); |
| DCHECK(source_params.IsValid()); |
| |
| // Create and initialize audio converter for the source data. |
| // When the webrtc AudioProcessing is enabled, the sink format of the |
| // converter will be the same as the post-processed data format, which is |
| // 32k mono for desktops and 16k mono for Android. When the AudioProcessing |
| // is disabled, the sink format will be the same as the source format. |
| const int sink_sample_rate = audio_processing_ ? |
| kAudioProcessingSampleRate : source_params.sample_rate(); |
| const media::ChannelLayout sink_channel_layout = audio_processing_ ? |
| media::GuessChannelLayout(kAudioProcessingNumberOfChannels) : |
| source_params.channel_layout(); |
| |
| // WebRtc AudioProcessing requires 10ms as its packet size. We use this |
| // native size when processing is enabled. While processing is disabled, and |
| // the source is running with a buffer size smaller than 10ms buffer, we use |
| // same buffer size as the incoming format to avoid extra FIFO for WebAudio. |
| int sink_buffer_size = sink_sample_rate / 100; |
| if (!audio_processing_ && |
| source_params.frames_per_buffer() < sink_buffer_size) { |
| sink_buffer_size = source_params.frames_per_buffer(); |
| } |
| |
| media::AudioParameters sink_params( |
| media::AudioParameters::AUDIO_PCM_LOW_LATENCY, sink_channel_layout, |
| sink_sample_rate, 16, sink_buffer_size); |
| capture_converter_.reset( |
| new MediaStreamAudioConverter(source_params, sink_params)); |
| } |
| |
| void MediaStreamAudioProcessor::InitializeRenderConverterIfNeeded( |
| int sample_rate, int number_of_channels, int frames_per_buffer) { |
| DCHECK(render_thread_checker_.CalledOnValidThread()); |
| // TODO(xians): Figure out if we need to handle the buffer size change. |
| if (render_converter_.get() && |
| render_converter_->source_parameters().sample_rate() == sample_rate && |
| render_converter_->source_parameters().channels() == number_of_channels) { |
| // Do nothing if the |render_converter_| has been setup properly. |
| return; |
| } |
| |
| // Create and initialize audio converter for the render data. |
| // webrtc::AudioProcessing accepts the same format as what it uses to process |
| // capture data, which is 32k mono for desktops and 16k mono for Android. |
| media::AudioParameters source_params( |
| media::AudioParameters::AUDIO_PCM_LOW_LATENCY, |
| media::GuessChannelLayout(number_of_channels), sample_rate, 16, |
| frames_per_buffer); |
| media::AudioParameters sink_params( |
| media::AudioParameters::AUDIO_PCM_LOW_LATENCY, |
| media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16, |
| kAudioProcessingSampleRate / 100); |
| render_converter_.reset( |
| new MediaStreamAudioConverter(source_params, sink_params)); |
| render_data_bus_ = media::AudioBus::Create(number_of_channels, |
| frames_per_buffer); |
| } |
| |
| int MediaStreamAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame, |
| base::TimeDelta capture_delay, |
| int volume, |
| bool key_pressed) { |
| DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| if (!audio_processing_) |
| return 0; |
| |
| TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessData"); |
| DCHECK_EQ(audio_processing_->input_sample_rate_hz(), |
| capture_converter_->sink_parameters().sample_rate()); |
| DCHECK_EQ(audio_processing_->num_input_channels(), |
| capture_converter_->sink_parameters().channels()); |
| DCHECK_EQ(audio_processing_->num_output_channels(), |
| capture_converter_->sink_parameters().channels()); |
| |
| base::subtle::Atomic32 render_delay_ms = |
| base::subtle::Acquire_Load(&render_delay_ms_); |
| int64 capture_delay_ms = capture_delay.InMilliseconds(); |
| DCHECK_LT(capture_delay_ms, |
| std::numeric_limits<base::subtle::Atomic32>::max()); |
| int total_delay_ms = capture_delay_ms + render_delay_ms; |
| if (total_delay_ms > 300) { |
| LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms |
| << "ms; render delay: " << render_delay_ms << "ms"; |
| } |
| |
| audio_processing_->set_stream_delay_ms(total_delay_ms); |
| |
| DCHECK_LE(volume, WebRtcAudioDeviceImpl::kMaxVolumeLevel); |
| webrtc::GainControl* agc = audio_processing_->gain_control(); |
| int err = agc->set_stream_analog_level(volume); |
| DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; |
| |
| audio_processing_->set_stream_key_pressed(key_pressed); |
| |
| err = audio_processing_->ProcessStream(audio_frame); |
| DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; |
| |
| if (typing_detector_ && |
| audio_frame->vad_activity_ != webrtc::AudioFrame::kVadUnknown) { |
| bool vad_active = |
| (audio_frame->vad_activity_ == webrtc::AudioFrame::kVadActive); |
| bool typing_detected = typing_detector_->Process(key_pressed, vad_active); |
| base::subtle::Release_Store(&typing_detected_, typing_detected); |
| } |
| |
| // Return 0 if the volume has not been changed, otherwise return the new |
| // volume. |
| return (agc->stream_analog_level() == volume) ? |
| 0 : agc->stream_analog_level(); |
| } |
| |
| } // namespace content |