| // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/audio/audio_input_controller.h" |
| |
| #include "base/bind.h" |
| #include "base/strings/stringprintf.h" |
| #include "base/threading/thread_restrictions.h" |
| #include "base/time/time.h" |
| #include "media/base/limits.h" |
| #include "media/base/scoped_histogram_timer.h" |
| #include "media/base/user_input_monitor.h" |
| |
| using base::TimeDelta; |
| |
| namespace { |
| const int kMaxInputChannels = 3; |
| |
| // TODO(henrika): remove usage of timers and add support for proper |
| // notification of when the input device is removed. This was originally added |
| // to resolve http://crbug.com/79936 for Windows platforms. This then caused |
| // breakage (very hard to repro bugs!) on other platforms: See |
| // http://crbug.com/226327 and http://crbug.com/230972. |
| // See also that the timer has been disabled on Mac now due to |
| // crbug.com/357501. |
| const int kTimerResetIntervalSeconds = 1; |
| // We have received reports that the timer can be too trigger happy on some |
| // Mac devices and the initial timer interval has therefore been increased |
| // from 1 second to 5 seconds. |
| const int kTimerInitialIntervalSeconds = 5; |
| |
| #if defined(AUDIO_POWER_MONITORING) |
| // Time constant for AudioPowerMonitor. |
| // The utilized smoothing factor (alpha) in the exponential filter is given |
| // by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time |
| // constant given by |kPowerMeasurementTimeConstantMilliseconds|. |
| // Example: fs=44100, ts=10e-3 => alpha~0.022420 |
| // fs=44100, ts=20e-3 => alpha~0.165903 |
| // A large smoothing factor corresponds to a faster filter response to input |
| // changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input |
| // and y(n) is the output. |
| const int kPowerMeasurementTimeConstantMilliseconds = 10; |
| |
| // Time in seconds between two successive measurements of audio power levels. |
| const int kPowerMonitorLogIntervalSeconds = 5; |
| #endif |
| } |
| |
| namespace media { |
| |
| // static |
| AudioInputController::Factory* AudioInputController::factory_ = NULL; |
| |
| AudioInputController::AudioInputController(EventHandler* handler, |
| SyncWriter* sync_writer, |
| UserInputMonitor* user_input_monitor) |
| : creator_task_runner_(base::MessageLoopProxy::current()), |
| handler_(handler), |
| stream_(NULL), |
| data_is_active_(false), |
| state_(CLOSED), |
| sync_writer_(sync_writer), |
| max_volume_(0.0), |
| user_input_monitor_(user_input_monitor), |
| prev_key_down_count_(0) { |
| DCHECK(creator_task_runner_.get()); |
| } |
| |
| AudioInputController::~AudioInputController() { |
| DCHECK_EQ(state_, CLOSED); |
| } |
| |
| // static |
| scoped_refptr<AudioInputController> AudioInputController::Create( |
| AudioManager* audio_manager, |
| EventHandler* event_handler, |
| const AudioParameters& params, |
| const std::string& device_id, |
| UserInputMonitor* user_input_monitor) { |
| DCHECK(audio_manager); |
| |
| if (!params.IsValid() || (params.channels() > kMaxInputChannels)) |
| return NULL; |
| |
| if (factory_) { |
| return factory_->Create( |
| audio_manager, event_handler, params, user_input_monitor); |
| } |
| scoped_refptr<AudioInputController> controller( |
| new AudioInputController(event_handler, NULL, user_input_monitor)); |
| |
| controller->task_runner_ = audio_manager->GetTaskRunner(); |
| |
| // Create and open a new audio input stream from the existing |
| // audio-device thread. |
| if (!controller->task_runner_->PostTask(FROM_HERE, |
| base::Bind(&AudioInputController::DoCreate, controller, |
| base::Unretained(audio_manager), params, device_id))) { |
| controller = NULL; |
| } |
| |
| return controller; |
| } |
| |
| // static |
| scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency( |
| AudioManager* audio_manager, |
| EventHandler* event_handler, |
| const AudioParameters& params, |
| const std::string& device_id, |
| SyncWriter* sync_writer, |
| UserInputMonitor* user_input_monitor) { |
| DCHECK(audio_manager); |
| DCHECK(sync_writer); |
| |
| if (!params.IsValid() || (params.channels() > kMaxInputChannels)) |
| return NULL; |
| |
| // Create the AudioInputController object and ensure that it runs on |
| // the audio-manager thread. |
| scoped_refptr<AudioInputController> controller( |
| new AudioInputController(event_handler, sync_writer, user_input_monitor)); |
| controller->task_runner_ = audio_manager->GetTaskRunner(); |
| |
| // Create and open a new audio input stream from the existing |
| // audio-device thread. Use the provided audio-input device. |
| if (!controller->task_runner_->PostTask(FROM_HERE, |
| base::Bind(&AudioInputController::DoCreate, controller, |
| base::Unretained(audio_manager), params, device_id))) { |
| controller = NULL; |
| } |
| |
| return controller; |
| } |
| |
| // static |
| scoped_refptr<AudioInputController> AudioInputController::CreateForStream( |
| const scoped_refptr<base::SingleThreadTaskRunner>& task_runner, |
| EventHandler* event_handler, |
| AudioInputStream* stream, |
| SyncWriter* sync_writer, |
| UserInputMonitor* user_input_monitor) { |
| DCHECK(sync_writer); |
| DCHECK(stream); |
| |
| // Create the AudioInputController object and ensure that it runs on |
| // the audio-manager thread. |
| scoped_refptr<AudioInputController> controller( |
| new AudioInputController(event_handler, sync_writer, user_input_monitor)); |
| controller->task_runner_ = task_runner; |
| |
| // TODO(miu): See TODO at top of file. Until that's resolved, we need to |
| // disable the error auto-detection here (since the audio mirroring |
| // implementation will reliably report error and close events). Note, of |
| // course, that we're assuming CreateForStream() has been called for the audio |
| // mirroring use case only. |
| if (!controller->task_runner_->PostTask( |
| FROM_HERE, |
| base::Bind(&AudioInputController::DoCreateForStream, controller, |
| stream, false))) { |
| controller = NULL; |
| } |
| |
| return controller; |
| } |
| |
| void AudioInputController::Record() { |
| task_runner_->PostTask(FROM_HERE, base::Bind( |
| &AudioInputController::DoRecord, this)); |
| } |
| |
| void AudioInputController::Close(const base::Closure& closed_task) { |
| DCHECK(!closed_task.is_null()); |
| DCHECK(creator_task_runner_->BelongsToCurrentThread()); |
| |
| task_runner_->PostTaskAndReply( |
| FROM_HERE, base::Bind(&AudioInputController::DoClose, this), closed_task); |
| } |
| |
| void AudioInputController::SetVolume(double volume) { |
| task_runner_->PostTask(FROM_HERE, base::Bind( |
| &AudioInputController::DoSetVolume, this, volume)); |
| } |
| |
| void AudioInputController::SetAutomaticGainControl(bool enabled) { |
| task_runner_->PostTask(FROM_HERE, base::Bind( |
| &AudioInputController::DoSetAutomaticGainControl, this, enabled)); |
| } |
| |
| void AudioInputController::DoCreate(AudioManager* audio_manager, |
| const AudioParameters& params, |
| const std::string& device_id) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime"); |
| |
| #if defined(AUDIO_POWER_MONITORING) |
| // Create the audio (power) level meter given the provided audio parameters. |
| // An AudioBus is also needed to wrap the raw data buffer from the native |
| // layer to match AudioPowerMonitor::Scan(). |
| // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155. |
| audio_level_.reset(new media::AudioPowerMonitor( |
| params.sample_rate(), |
| TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds))); |
| audio_params_ = params; |
| #endif |
| |
| // TODO(miu): See TODO at top of file. Until that's resolved, assume all |
| // platform audio input requires the |no_data_timer_| be used to auto-detect |
| // errors. In reality, probably only Windows needs to be treated as |
| // unreliable here. |
| DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id), |
| true); |
| } |
| |
| void AudioInputController::DoCreateForStream( |
| AudioInputStream* stream_to_control, bool enable_nodata_timer) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| DCHECK(!stream_); |
| stream_ = stream_to_control; |
| |
| if (!stream_) { |
| if (handler_) |
| handler_->OnError(this, STREAM_CREATE_ERROR); |
| return; |
| } |
| |
| if (stream_ && !stream_->Open()) { |
| stream_->Close(); |
| stream_ = NULL; |
| if (handler_) |
| handler_->OnError(this, STREAM_OPEN_ERROR); |
| return; |
| } |
| |
| DCHECK(!no_data_timer_.get()); |
| |
| // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered |
| // from the timer must be ignored by the EventHandler. |
| // TODO(henrika): remove usage of timer when it has been verified on Canary |
| // that we are safe doing so. Goal is to get rid of |no_data_timer_| and |
| // everything that is tied to it. crbug.com/357569. |
| enable_nodata_timer = true; |
| |
| if (enable_nodata_timer) { |
| // Create the data timer which will call FirstCheckForNoData(). The timer |
| // is started in DoRecord() and restarted in each DoCheckForNoData() |
| // callback. |
| no_data_timer_.reset(new base::Timer( |
| FROM_HERE, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds), |
| base::Bind(&AudioInputController::FirstCheckForNoData, |
| base::Unretained(this)), false)); |
| } else { |
| DVLOG(1) << "Disabled: timer check for no data."; |
| } |
| |
| state_ = CREATED; |
| if (handler_) |
| handler_->OnCreated(this); |
| |
| if (user_input_monitor_) { |
| user_input_monitor_->EnableKeyPressMonitoring(); |
| prev_key_down_count_ = user_input_monitor_->GetKeyPressCount(); |
| } |
| } |
| |
| void AudioInputController::DoRecord() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime"); |
| |
| if (state_ != CREATED) |
| return; |
| |
| { |
| base::AutoLock auto_lock(lock_); |
| state_ = RECORDING; |
| } |
| |
| if (no_data_timer_) { |
| // Start the data timer. Once |kTimerResetIntervalSeconds| have passed, |
| // a callback to FirstCheckForNoData() is made. |
| no_data_timer_->Reset(); |
| } |
| |
| stream_->Start(this); |
| if (handler_) |
| handler_->OnRecording(this); |
| } |
| |
| void AudioInputController::DoClose() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime"); |
| |
| if (state_ == CLOSED) |
| return; |
| |
| // Delete the timer on the same thread that created it. |
| no_data_timer_.reset(); |
| |
| DoStopCloseAndClearStream(); |
| SetDataIsActive(false); |
| |
| if (SharedMemoryAndSyncSocketMode()) |
| sync_writer_->Close(); |
| |
| if (user_input_monitor_) |
| user_input_monitor_->DisableKeyPressMonitoring(); |
| |
| state_ = CLOSED; |
| } |
| |
| void AudioInputController::DoReportError() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| if (handler_) |
| handler_->OnError(this, STREAM_ERROR); |
| } |
| |
| void AudioInputController::DoSetVolume(double volume) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| DCHECK_GE(volume, 0); |
| DCHECK_LE(volume, 1.0); |
| |
| if (state_ != CREATED && state_ != RECORDING) |
| return; |
| |
| // Only ask for the maximum volume at first call and use cached value |
| // for remaining function calls. |
| if (!max_volume_) { |
| max_volume_ = stream_->GetMaxVolume(); |
| } |
| |
| if (max_volume_ == 0.0) { |
| DLOG(WARNING) << "Failed to access input volume control"; |
| return; |
| } |
| |
| // Set the stream volume and scale to a range matched to the platform. |
| stream_->SetVolume(max_volume_ * volume); |
| } |
| |
| void AudioInputController::DoSetAutomaticGainControl(bool enabled) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| DCHECK_NE(state_, RECORDING); |
| |
| // Ensure that the AGC state only can be modified before streaming starts. |
| if (state_ != CREATED) |
| return; |
| |
| stream_->SetAutomaticGainControl(enabled); |
| } |
| |
| void AudioInputController::FirstCheckForNoData() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| UMA_HISTOGRAM_BOOLEAN("Media.AudioInputControllerCaptureStartupSuccess", |
| GetDataIsActive()); |
| DoCheckForNoData(); |
| } |
| |
| void AudioInputController::DoCheckForNoData() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| if (!GetDataIsActive()) { |
| // The data-is-active marker will be false only if it has been more than |
| // one second since a data packet was recorded. This can happen if a |
| // capture device has been removed or disabled. |
| if (handler_) |
| handler_->OnError(this, NO_DATA_ERROR); |
| } |
| |
| // Mark data as non-active. The flag will be re-enabled in OnData() each |
| // time a data packet is received. Hence, under normal conditions, the |
| // flag will only be disabled during a very short period. |
| SetDataIsActive(false); |
| |
| // Restart the timer to ensure that we check the flag again in |
| // |kTimerResetIntervalSeconds|. |
| no_data_timer_->Start( |
| FROM_HERE, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds), |
| base::Bind(&AudioInputController::DoCheckForNoData, |
| base::Unretained(this))); |
| } |
| |
| void AudioInputController::OnData(AudioInputStream* stream, |
| const AudioBus* source, |
| uint32 hardware_delay_bytes, |
| double volume) { |
| // Mark data as active to ensure that the periodic calls to |
| // DoCheckForNoData() does not report an error to the event handler. |
| SetDataIsActive(true); |
| |
| { |
| base::AutoLock auto_lock(lock_); |
| if (state_ != RECORDING) |
| return; |
| } |
| |
| bool key_pressed = false; |
| if (user_input_monitor_) { |
| size_t current_count = user_input_monitor_->GetKeyPressCount(); |
| key_pressed = current_count != prev_key_down_count_; |
| prev_key_down_count_ = current_count; |
| DVLOG_IF(6, key_pressed) << "Detected keypress."; |
| } |
| |
| // Use SharedMemory and SyncSocket if the client has created a SyncWriter. |
| // Used by all low-latency clients except WebSpeech. |
| if (SharedMemoryAndSyncSocketMode()) { |
| sync_writer_->Write(source, volume, key_pressed); |
| sync_writer_->UpdateRecordedBytes(hardware_delay_bytes); |
| |
| #if defined(AUDIO_POWER_MONITORING) |
| // Only do power-level measurements if an AudioPowerMonitor object has |
| // been created. Done in DoCreate() but not DoCreateForStream(), hence |
| // logging will mainly be done for WebRTC and WebSpeech clients. |
| if (!audio_level_) |
| return; |
| |
| // Perform periodic audio (power) level measurements. |
| if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() > |
| kPowerMonitorLogIntervalSeconds) { |
| // Wrap data into an AudioBus to match AudioPowerMonitor::Scan. |
| // TODO(henrika): remove this section when capture side uses AudioBus. |
| // See http://crbug.com/375155 for details. |
| audio_level_->Scan(*source, source->frames()); |
| |
| // Get current average power level and add it to the log. |
| // Possible range is given by [-inf, 0] dBFS. |
| std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip(); |
| |
| // Use event handler on the audio thread to relay a message to the ARIH |
| // in content which does the actual logging on the IO thread. |
| task_runner_->PostTask( |
| FROM_HERE, |
| base::Bind( |
| &AudioInputController::DoLogAudioLevel, this, result.first)); |
| |
| last_audio_level_log_time_ = base::TimeTicks::Now(); |
| |
| // Reset the average power level (since we don't log continuously). |
| audio_level_->Reset(); |
| } |
| #endif |
| return; |
| } |
| |
| // TODO(henrika): Investigate if we can avoid the extra copy here. |
| // (see http://crbug.com/249316 for details). AFAIK, this scope is only |
| // active for WebSpeech clients. |
| scoped_ptr<AudioBus> audio_data = |
| AudioBus::Create(source->channels(), source->frames()); |
| source->CopyTo(audio_data.get()); |
| |
| // Ownership of the audio buffer will be with the callback until it is run, |
| // when ownership is passed to the callback function. |
| task_runner_->PostTask( |
| FROM_HERE, |
| base::Bind( |
| &AudioInputController::DoOnData, this, base::Passed(&audio_data))); |
| } |
| |
| void AudioInputController::DoOnData(scoped_ptr<AudioBus> data) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| if (handler_) |
| handler_->OnData(this, data.get()); |
| } |
| |
| void AudioInputController::DoLogAudioLevel(float level_dbfs) { |
| #if defined(AUDIO_POWER_MONITORING) |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| if (!handler_) |
| return; |
| |
| std::string log_string = base::StringPrintf( |
| "AIC::OnData: average audio level=%.2f dBFS", level_dbfs); |
| static const float kSilenceThresholdDBFS = -72.24719896f; |
| if (level_dbfs < kSilenceThresholdDBFS) |
| log_string += " <=> no audio input!"; |
| |
| handler_->OnLog(this, log_string); |
| #endif |
| } |
| |
| void AudioInputController::OnError(AudioInputStream* stream) { |
| // Handle error on the audio-manager thread. |
| task_runner_->PostTask(FROM_HERE, base::Bind( |
| &AudioInputController::DoReportError, this)); |
| } |
| |
| void AudioInputController::DoStopCloseAndClearStream() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| // Allow calling unconditionally and bail if we don't have a stream to close. |
| if (stream_ != NULL) { |
| stream_->Stop(); |
| stream_->Close(); |
| stream_ = NULL; |
| } |
| |
| // The event handler should not be touched after the stream has been closed. |
| handler_ = NULL; |
| } |
| |
| void AudioInputController::SetDataIsActive(bool enabled) { |
| base::subtle::Release_Store(&data_is_active_, enabled); |
| } |
| |
| bool AudioInputController::GetDataIsActive() { |
| return (base::subtle::Acquire_Load(&data_is_active_) != false); |
| } |
| |
| } // namespace media |