blob: 8282d697573c604c6b34cd37f72facb3dfc86fc9 [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/renderer/media/webrtc_local_audio_track.h"
#include "content/public/renderer/media_stream_audio_sink.h"
#include "content/renderer/media/media_stream_audio_sink_owner.h"
#include "content/renderer/media/media_stream_audio_track_sink.h"
#include "content/renderer/media/peer_connection_audio_sink_owner.h"
#include "content/renderer/media/webaudio_capturer_source.h"
#include "content/renderer/media/webrtc_audio_capturer.h"
#include "content/renderer/media/webrtc_local_audio_source_provider.h"
#include "media/base/audio_fifo.h"
#include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
namespace content {
static const size_t kMaxNumberOfBuffersInFifo = 2;
static const char kAudioTrackKind[] = "audio";
namespace {
using webrtc::MediaConstraintsInterface;
// This helper function checks if any audio constraints are set that require
// audio processing to be applied. Right now this is a big, single switch for
// all of the properties, but in the future they'll be handled one by one.
bool NeedsAudioProcessing(
const webrtc::MediaConstraintsInterface* constraints) {
if (!constraints)
return false;
static const char* kAudioProcessingProperties[] = {
MediaConstraintsInterface::kEchoCancellation,
MediaConstraintsInterface::kExperimentalEchoCancellation,
MediaConstraintsInterface::kAutoGainControl,
MediaConstraintsInterface::kExperimentalAutoGainControl,
MediaConstraintsInterface::kNoiseSuppression,
MediaConstraintsInterface::kHighpassFilter,
MediaConstraintsInterface::kTypingNoiseDetection,
};
for (size_t i = 0; i < arraysize(kAudioProcessingProperties); ++i) {
bool value = false;
if (webrtc::FindConstraint(constraints, kAudioProcessingProperties[i],
&value, NULL) &&
value) {
return true;
}
}
return false;
}
} // namespace.
// This is a temporary audio buffer with parameters used to send data to
// callbacks.
class WebRtcLocalAudioTrack::ConfiguredBuffer {
public:
ConfiguredBuffer() {}
virtual ~ConfiguredBuffer() {}
void Configure(const media::AudioParameters& params) {
DCHECK(params.IsValid());
// PeerConnection uses 10ms as the sink buffer size as its native packet
// size. We use the native PeerConnection buffer size to achieve the best
// performance when a PeerConnection is connected with a track.
int sink_buffer_size = params.sample_rate() / 100;
if (params.frames_per_buffer() < sink_buffer_size) {
// When the source is running with a buffer size smaller than the peer
// connection buffer size, that means no PeerConnection is connected
// to the track, use the same buffer size as the incoming format to
// avoid extra FIFO for WebAudio.
sink_buffer_size = params.frames_per_buffer();
}
params_.Reset(params.format(), params.channel_layout(), params.channels(),
params.input_channels(), params.sample_rate(),
params.bits_per_sample(), sink_buffer_size);
audio_wrapper_ = media::AudioBus::Create(params_.channels(),
params_.frames_per_buffer());
buffer_.reset(new int16[params_.frames_per_buffer() * params_.channels()]);
// The size of the FIFO should be at least twice of the source buffer size
// or twice of the sink buffer size.
int buffer_size = std::max(
kMaxNumberOfBuffersInFifo * params.frames_per_buffer(),
kMaxNumberOfBuffersInFifo * params_.frames_per_buffer());
fifo_.reset(new media::AudioFifo(params_.channels(), buffer_size));
}
void Push(media::AudioBus* audio_source) {
DCHECK(fifo_->frames() + audio_source->frames() <= fifo_->max_frames());
fifo_->Push(audio_source);
}
bool Consume() {
if (fifo_->frames() < audio_wrapper_->frames())
return false;
fifo_->Consume(audio_wrapper_.get(), 0, audio_wrapper_->frames());
audio_wrapper_->ToInterleaved(audio_wrapper_->frames(),
params_.bits_per_sample() / 8,
buffer());
return true;
}
int16* buffer() const { return buffer_.get(); }
// Format of the output audio buffer.
const media::AudioParameters& params() const { return params_; }
private:
media::AudioParameters params_;
scoped_ptr<media::AudioBus> audio_wrapper_;
scoped_ptr<media::AudioFifo> fifo_;
scoped_ptr<int16[]> buffer_;
};
scoped_refptr<WebRtcLocalAudioTrack> WebRtcLocalAudioTrack::Create(
const std::string& id,
const scoped_refptr<WebRtcAudioCapturer>& capturer,
WebAudioCapturerSource* webaudio_source,
webrtc::AudioSourceInterface* track_source,
const webrtc::MediaConstraintsInterface* constraints) {
talk_base::RefCountedObject<WebRtcLocalAudioTrack>* track =
new talk_base::RefCountedObject<WebRtcLocalAudioTrack>(
id, capturer, webaudio_source, track_source, constraints);
return track;
}
WebRtcLocalAudioTrack::WebRtcLocalAudioTrack(
const std::string& label,
const scoped_refptr<WebRtcAudioCapturer>& capturer,
WebAudioCapturerSource* webaudio_source,
webrtc::AudioSourceInterface* track_source,
const webrtc::MediaConstraintsInterface* constraints)
: webrtc::MediaStreamTrack<webrtc::AudioTrackInterface>(label),
capturer_(capturer),
webaudio_source_(webaudio_source),
track_source_(track_source),
need_audio_processing_(NeedsAudioProcessing(constraints)),
buffer_(new ConfiguredBuffer()) {
DCHECK(capturer.get() || webaudio_source);
if (!webaudio_source_) {
source_provider_.reset(new WebRtcLocalAudioSourceProvider());
AddSink(source_provider_.get());
}
DVLOG(1) << "WebRtcLocalAudioTrack::WebRtcLocalAudioTrack()";
}
WebRtcLocalAudioTrack::~WebRtcLocalAudioTrack() {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::~WebRtcLocalAudioTrack()";
// Users might not call Stop() on the track.
Stop();
}
void WebRtcLocalAudioTrack::Capture(media::AudioBus* audio_source,
int audio_delay_milliseconds,
int volume,
bool key_pressed) {
DCHECK(capture_thread_checker_.CalledOnValidThread());
scoped_refptr<WebRtcAudioCapturer> capturer;
std::vector<int> voe_channels;
SinkList::ItemList sinks;
SinkList::ItemList sinks_to_notify_format;
bool is_webaudio_source = false;
{
base::AutoLock auto_lock(lock_);
capturer = capturer_;
voe_channels = voe_channels_;
sinks = sinks_.Items();
sinks_.RetrieveAndClearTags(&sinks_to_notify_format);
is_webaudio_source = (webaudio_source_.get() != NULL);
}
// Notify the tracks on when the format changes. This will do nothing if
// |sinks_to_notify_format| is empty.
for (SinkList::ItemList::const_iterator it = sinks_to_notify_format.begin();
it != sinks_to_notify_format.end(); ++it) {
(*it)->OnSetFormat(buffer_->params());
}
// Push the data to the fifo.
buffer_->Push(audio_source);
// When the source is WebAudio, turn off the audio processing if the delay
// value is 0 even though the constraint is set to true. In such case, it
// indicates the data is not from microphone.
// TODO(xians): remove the flag when supporting one APM per audio track.
// See crbug/264611 for details.
bool need_audio_processing = need_audio_processing_;
if (is_webaudio_source && need_audio_processing)
need_audio_processing = (audio_delay_milliseconds != 0);
int current_volume = volume;
while (buffer_->Consume()) {
// Feed the data to the sinks.
// TODO (jiayl): we should not pass the real audio data down if the track is
// disabled. This is currently done so to feed input to WebRTC typing
// detection and should be changed when audio processing is moved from
// WebRTC to the track.
for (SinkList::ItemList::const_iterator it = sinks.begin();
it != sinks.end();
++it) {
int new_volume = (*it)->OnData(buffer_->buffer(),
buffer_->params().sample_rate(),
buffer_->params().channels(),
buffer_->params().frames_per_buffer(),
voe_channels,
audio_delay_milliseconds,
current_volume,
need_audio_processing,
key_pressed);
if (new_volume != 0 && capturer.get()) {
// Feed the new volume to WebRtc while changing the volume on the
// browser.
capturer->SetVolume(new_volume);
current_volume = new_volume;
}
}
}
}
void WebRtcLocalAudioTrack::OnSetFormat(
const media::AudioParameters& params) {
DVLOG(1) << "WebRtcLocalAudioTrack::OnSetFormat()";
// If the source is restarted, we might have changed to another capture
// thread.
capture_thread_checker_.DetachFromThread();
DCHECK(capture_thread_checker_.CalledOnValidThread());
DCHECK(params.IsValid());
buffer_->Configure(params);
base::AutoLock auto_lock(lock_);
// Remember to notify all sinks of the new format.
sinks_.TagAll();
}
void WebRtcLocalAudioTrack::AddChannel(int channel_id) {
DVLOG(1) << "WebRtcLocalAudioTrack::AddChannel(channel_id="
<< channel_id << ")";
base::AutoLock auto_lock(lock_);
if (std::find(voe_channels_.begin(), voe_channels_.end(), channel_id) !=
voe_channels_.end()) {
// We need to handle the case when the same channel is connected to the
// track more than once.
return;
}
voe_channels_.push_back(channel_id);
}
void WebRtcLocalAudioTrack::RemoveChannel(int channel_id) {
DVLOG(1) << "WebRtcLocalAudioTrack::RemoveChannel(channel_id="
<< channel_id << ")";
base::AutoLock auto_lock(lock_);
std::vector<int>::iterator iter =
std::find(voe_channels_.begin(), voe_channels_.end(), channel_id);
DCHECK(iter != voe_channels_.end());
voe_channels_.erase(iter);
}
// webrtc::AudioTrackInterface implementation.
webrtc::AudioSourceInterface* WebRtcLocalAudioTrack::GetSource() const {
return track_source_;
}
cricket::AudioRenderer* WebRtcLocalAudioTrack::GetRenderer() {
return this;
}
std::string WebRtcLocalAudioTrack::kind() const {
return kAudioTrackKind;
}
void WebRtcLocalAudioTrack::AddSink(MediaStreamAudioSink* sink) {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::AddSink()";
base::AutoLock auto_lock(lock_);
// Verify that |sink| is not already added to the list.
DCHECK(!sinks_.Contains(
MediaStreamAudioTrackSink::WrapsMediaStreamSink(sink)));
// Create (and add to the list) a new MediaStreamAudioTrackSink
// which owns the |sink| and delagates all calls to the
// MediaStreamAudioSink interface. It will be tagged in the list, so
// we remember to call OnSetFormat() on the new sink.
scoped_refptr<MediaStreamAudioTrackSink> sink_owner(
new MediaStreamAudioSinkOwner(sink));
sinks_.AddAndTag(sink_owner);
}
void WebRtcLocalAudioTrack::RemoveSink(MediaStreamAudioSink* sink) {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::RemoveSink()";
base::AutoLock auto_lock(lock_);
scoped_refptr<MediaStreamAudioTrackSink> removed_item = sinks_.Remove(
MediaStreamAudioTrackSink::WrapsMediaStreamSink(sink));
// Clear the delegate to ensure that no more capture callbacks will
// be sent to this sink. Also avoids a possible crash which can happen
// if this method is called while capturing is active.
if (removed_item.get())
removed_item->Reset();
}
void WebRtcLocalAudioTrack::AddSink(PeerConnectionAudioSink* sink) {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::AddSink()";
base::AutoLock auto_lock(lock_);
// Verify that |sink| is not already added to the list.
DCHECK(!sinks_.Contains(
MediaStreamAudioTrackSink::WrapsPeerConnectionSink(sink)));
// Create (and add to the list) a new MediaStreamAudioTrackSink
// which owns the |sink| and delagates all calls to the
// MediaStreamAudioSink interface. It will be tagged in the list, so
// we remember to call OnSetFormat() on the new sink.
scoped_refptr<MediaStreamAudioTrackSink> sink_owner(
new PeerConnectionAudioSinkOwner(sink));
sinks_.AddAndTag(sink_owner);
}
void WebRtcLocalAudioTrack::RemoveSink(PeerConnectionAudioSink* sink) {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::RemoveSink()";
base::AutoLock auto_lock(lock_);
scoped_refptr<MediaStreamAudioTrackSink> removed_item = sinks_.Remove(
MediaStreamAudioTrackSink::WrapsPeerConnectionSink(sink));
// Clear the delegate to ensure that no more capture callbacks will
// be sent to this sink. Also avoids a possible crash which can happen
// if this method is called while capturing is active.
if (removed_item.get())
removed_item->Reset();
}
void WebRtcLocalAudioTrack::Start() {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::Start()";
if (webaudio_source_.get()) {
// If the track is hooking up with WebAudio, do NOT add the track to the
// capturer as its sink otherwise two streams in different clock will be
// pushed through the same track.
webaudio_source_->Start(this, capturer_.get());
return;
}
if (capturer_.get())
capturer_->AddTrack(this);
}
void WebRtcLocalAudioTrack::Stop() {
DCHECK(main_render_thread_checker_.CalledOnValidThread());
DVLOG(1) << "WebRtcLocalAudioTrack::Stop()";
if (!capturer_.get() && !webaudio_source_.get())
return;
if (webaudio_source_.get()) {
// Called Stop() on the |webaudio_source_| explicitly so that
// |webaudio_source_| won't push more data to the track anymore.
// Also note that the track is not registered as a sink to the |capturer_|
// in such case and no need to call RemoveTrack().
webaudio_source_->Stop();
} else {
// It is necessary to call RemoveTrack on the |capturer_| to avoid getting
// audio callback after Stop().
capturer_->RemoveTrack(this);
}
// Protect the pointers using the lock when accessing |sinks_| and
// setting the |capturer_| to NULL.
SinkList::ItemList sinks;
{
base::AutoLock auto_lock(lock_);
sinks = sinks_.Items();
sinks_.Clear();
webaudio_source_ = NULL;
capturer_ = NULL;
}
for (SinkList::ItemList::const_iterator it = sinks.begin();
it != sinks.end();
++it)
(*it)->Reset();
}
} // namespace content