Support for unmixed remote audio into tracks.
BUG=chromium:121673
R=solenberg@webrtc.org
Review URL: https://codereview.webrtc.org/1505253004 .
Cr-Commit-Position: refs/heads/master@{#10995}
diff --git a/talk/app/webrtc/mediastreaminterface.h b/talk/app/webrtc/mediastreaminterface.h
index 00782cb..5327bd2 100644
--- a/talk/app/webrtc/mediastreaminterface.h
+++ b/talk/app/webrtc/mediastreaminterface.h
@@ -167,6 +167,8 @@
// TODO(xians): Makes all the interface pure virtual after Chrome has their
// implementations.
// Sets the volume to the source. |volume| is in the range of [0, 10].
+ // TODO(tommi): This method should be on the track and ideally volume should
+ // be applied in the track in a way that does not affect clones of the track.
virtual void SetVolume(double volume) {}
// Registers/unregisters observer to the audio source.
diff --git a/talk/app/webrtc/mediastreamprovider.h b/talk/app/webrtc/mediastreamprovider.h
index a78b55a..585d51b 100644
--- a/talk/app/webrtc/mediastreamprovider.h
+++ b/talk/app/webrtc/mediastreamprovider.h
@@ -29,6 +29,7 @@
#define TALK_APP_WEBRTC_MEDIASTREAMPROVIDER_H_
#include "webrtc/base/basictypes.h"
+#include "webrtc/base/scoped_ptr.h"
namespace cricket {
@@ -42,6 +43,8 @@
namespace webrtc {
+class AudioSinkInterface;
+
// TODO(deadbeef): Change the key from an ssrc to a "sender_id" or
// "receiver_id" string, which will be the MSID in the short term and MID in
// the long term.
@@ -67,6 +70,13 @@
// |volume| is in the range of [0, 10].
virtual void SetAudioPlayoutVolume(uint32_t ssrc, double volume) = 0;
+ // Allows for setting a direct audio sink for an incoming audio source.
+ // Only one audio sink is supported per ssrc and ownership of the sink is
+ // passed to the provider.
+ virtual void SetRawAudioSink(
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) = 0;
+
protected:
virtual ~AudioProviderInterface() {}
};
diff --git a/talk/app/webrtc/peerconnection.cc b/talk/app/webrtc/peerconnection.cc
index 933dc83..83e2919 100644
--- a/talk/app/webrtc/peerconnection.cc
+++ b/talk/app/webrtc/peerconnection.cc
@@ -39,6 +39,7 @@
#include "talk/app/webrtc/mediastreamproxy.h"
#include "talk/app/webrtc/mediastreamtrackproxy.h"
#include "talk/app/webrtc/remoteaudiosource.h"
+#include "talk/app/webrtc/remoteaudiotrack.h"
#include "talk/app/webrtc/remotevideocapturer.h"
#include "talk/app/webrtc/rtpreceiver.h"
#include "talk/app/webrtc/rtpsender.h"
@@ -448,10 +449,12 @@
MediaStream::Create(stream_label));
}
- AudioTrackInterface* AddAudioTrack(webrtc::MediaStreamInterface* stream,
+ AudioTrackInterface* AddAudioTrack(uint32_t ssrc,
+ AudioProviderInterface* provider,
+ webrtc::MediaStreamInterface* stream,
const std::string& track_id) {
- return AddTrack<AudioTrackInterface, AudioTrack, AudioTrackProxy>(
- stream, track_id, RemoteAudioSource::Create().get());
+ return AddTrack<AudioTrackInterface, RemoteAudioTrack, AudioTrackProxy>(
+ stream, track_id, RemoteAudioSource::Create(ssrc, provider));
}
VideoTrackInterface* AddVideoTrack(webrtc::MediaStreamInterface* stream,
@@ -467,7 +470,7 @@
template <typename TI, typename T, typename TP, typename S>
TI* AddTrack(MediaStreamInterface* stream,
const std::string& track_id,
- S* source) {
+ const S& source) {
rtc::scoped_refptr<TI> track(
TP::Create(signaling_thread_, T::Create(track_id, source)));
track->set_state(webrtc::MediaStreamTrackInterface::kLive);
@@ -1583,8 +1586,8 @@
MediaStreamInterface* stream = remote_streams_->find(stream_label);
if (media_type == cricket::MEDIA_TYPE_AUDIO) {
- AudioTrackInterface* audio_track =
- remote_stream_factory_->AddAudioTrack(stream, track_id);
+ AudioTrackInterface* audio_track = remote_stream_factory_->AddAudioTrack(
+ ssrc, session_.get(), stream, track_id);
CreateAudioReceiver(stream, audio_track, ssrc);
} else if (media_type == cricket::MEDIA_TYPE_VIDEO) {
VideoTrackInterface* video_track =
diff --git a/talk/app/webrtc/remoteaudiosource.cc b/talk/app/webrtc/remoteaudiosource.cc
index 41f3d87..e5af1e9 100644
--- a/talk/app/webrtc/remoteaudiosource.cc
+++ b/talk/app/webrtc/remoteaudiosource.cc
@@ -29,44 +29,143 @@
#include <algorithm>
#include <functional>
+#include <utility>
+#include "talk/app/webrtc/mediastreamprovider.h"
+#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
+#include "webrtc/base/thread.h"
namespace webrtc {
-rtc::scoped_refptr<RemoteAudioSource> RemoteAudioSource::Create() {
- return new rtc::RefCountedObject<RemoteAudioSource>();
+class RemoteAudioSource::MessageHandler : public rtc::MessageHandler {
+ public:
+ explicit MessageHandler(RemoteAudioSource* source) : source_(source) {}
+
+ private:
+ ~MessageHandler() override {}
+
+ void OnMessage(rtc::Message* msg) override {
+ source_->OnMessage(msg);
+ delete this;
+ }
+
+ const rtc::scoped_refptr<RemoteAudioSource> source_;
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(MessageHandler);
+};
+
+class RemoteAudioSource::Sink : public AudioSinkInterface {
+ public:
+ explicit Sink(RemoteAudioSource* source) : source_(source) {}
+ ~Sink() override { source_->OnAudioProviderGone(); }
+
+ private:
+ void OnData(const AudioSinkInterface::Data& audio) override {
+ if (source_)
+ source_->OnData(audio);
+ }
+
+ const rtc::scoped_refptr<RemoteAudioSource> source_;
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Sink);
+};
+
+rtc::scoped_refptr<RemoteAudioSource> RemoteAudioSource::Create(
+ uint32_t ssrc,
+ AudioProviderInterface* provider) {
+ rtc::scoped_refptr<RemoteAudioSource> ret(
+ new rtc::RefCountedObject<RemoteAudioSource>());
+ ret->Initialize(ssrc, provider);
+ return ret;
}
-RemoteAudioSource::RemoteAudioSource() {
+RemoteAudioSource::RemoteAudioSource()
+ : main_thread_(rtc::Thread::Current()),
+ state_(MediaSourceInterface::kLive) {
+ RTC_DCHECK(main_thread_);
}
RemoteAudioSource::~RemoteAudioSource() {
- ASSERT(audio_observers_.empty());
+ RTC_DCHECK(main_thread_->IsCurrent());
+ RTC_DCHECK(audio_observers_.empty());
+ RTC_DCHECK(sinks_.empty());
}
-MediaSourceInterface::SourceState RemoteAudioSource::state() const {
- return MediaSourceInterface::kLive;
-}
-
-void RemoteAudioSource::SetVolume(double volume) {
- ASSERT(volume >= 0 && volume <= 10);
- for (AudioObserverList::iterator it = audio_observers_.begin();
- it != audio_observers_.end(); ++it) {
- (*it)->OnSetVolume(volume);
+void RemoteAudioSource::Initialize(uint32_t ssrc,
+ AudioProviderInterface* provider) {
+ RTC_DCHECK(main_thread_->IsCurrent());
+ // To make sure we always get notified when the provider goes out of scope,
+ // we register for callbacks here and not on demand in AddSink.
+ if (provider) { // May be null in tests.
+ provider->SetRawAudioSink(
+ ssrc, std::move(rtc::scoped_ptr<AudioSinkInterface>(new Sink(this))));
}
}
+MediaSourceInterface::SourceState RemoteAudioSource::state() const {
+ RTC_DCHECK(main_thread_->IsCurrent());
+ return state_;
+}
+
+void RemoteAudioSource::SetVolume(double volume) {
+ RTC_DCHECK(volume >= 0 && volume <= 10);
+ for (auto* observer : audio_observers_)
+ observer->OnSetVolume(volume);
+}
+
void RemoteAudioSource::RegisterAudioObserver(AudioObserver* observer) {
- ASSERT(observer != NULL);
- ASSERT(std::find(audio_observers_.begin(), audio_observers_.end(),
- observer) == audio_observers_.end());
+ RTC_DCHECK(observer != NULL);
+ RTC_DCHECK(std::find(audio_observers_.begin(), audio_observers_.end(),
+ observer) == audio_observers_.end());
audio_observers_.push_back(observer);
}
void RemoteAudioSource::UnregisterAudioObserver(AudioObserver* observer) {
- ASSERT(observer != NULL);
+ RTC_DCHECK(observer != NULL);
audio_observers_.remove(observer);
}
+void RemoteAudioSource::AddSink(AudioTrackSinkInterface* sink) {
+ RTC_DCHECK(main_thread_->IsCurrent());
+ RTC_DCHECK(sink);
+
+ if (state_ != MediaSourceInterface::kLive) {
+ LOG(LS_ERROR) << "Can't register sink as the source isn't live.";
+ return;
+ }
+
+ rtc::CritScope lock(&sink_lock_);
+ RTC_DCHECK(std::find(sinks_.begin(), sinks_.end(), sink) == sinks_.end());
+ sinks_.push_back(sink);
+}
+
+void RemoteAudioSource::RemoveSink(AudioTrackSinkInterface* sink) {
+ RTC_DCHECK(main_thread_->IsCurrent());
+ RTC_DCHECK(sink);
+
+ rtc::CritScope lock(&sink_lock_);
+ sinks_.remove(sink);
+}
+
+void RemoteAudioSource::OnData(const AudioSinkInterface::Data& audio) {
+ // Called on the externally-owned audio callback thread, via/from webrtc.
+ rtc::CritScope lock(&sink_lock_);
+ for (auto* sink : sinks_) {
+ sink->OnData(audio.data, 16, audio.sample_rate, audio.channels,
+ audio.samples_per_channel);
+ }
+}
+
+void RemoteAudioSource::OnAudioProviderGone() {
+ // Called when the data provider is deleted. It may be the worker thread
+ // in libjingle or may be a different worker thread.
+ main_thread_->Post(new MessageHandler(this));
+}
+
+void RemoteAudioSource::OnMessage(rtc::Message* msg) {
+ RTC_DCHECK(main_thread_->IsCurrent());
+ sinks_.clear();
+ state_ = MediaSourceInterface::kEnded;
+ FireOnChanged();
+}
+
} // namespace webrtc
diff --git a/talk/app/webrtc/remoteaudiosource.h b/talk/app/webrtc/remoteaudiosource.h
index e49aca5..f518d9b 100644
--- a/talk/app/webrtc/remoteaudiosource.h
+++ b/talk/app/webrtc/remoteaudiosource.h
@@ -29,36 +29,65 @@
#define TALK_APP_WEBRTC_REMOTEAUDIOSOURCE_H_
#include <list>
+#include <string>
#include "talk/app/webrtc/mediastreaminterface.h"
#include "talk/app/webrtc/notifier.h"
+#include "talk/media/base/audiorenderer.h"
+#include "webrtc/audio/audio_sink.h"
+#include "webrtc/base/criticalsection.h"
+
+namespace rtc {
+struct Message;
+class Thread;
+} // namespace rtc
namespace webrtc {
-using webrtc::AudioSourceInterface;
+class AudioProviderInterface;
// This class implements the audio source used by the remote audio track.
class RemoteAudioSource : public Notifier<AudioSourceInterface> {
public:
// Creates an instance of RemoteAudioSource.
- static rtc::scoped_refptr<RemoteAudioSource> Create();
-
- protected:
- RemoteAudioSource();
- virtual ~RemoteAudioSource();
-
- private:
- typedef std::list<AudioObserver*> AudioObserverList;
+ static rtc::scoped_refptr<RemoteAudioSource> Create(
+ uint32_t ssrc,
+ AudioProviderInterface* provider);
// MediaSourceInterface implementation.
MediaSourceInterface::SourceState state() const override;
+ void AddSink(AudioTrackSinkInterface* sink);
+ void RemoveSink(AudioTrackSinkInterface* sink);
+
+ protected:
+ RemoteAudioSource();
+ ~RemoteAudioSource() override;
+
+ // Post construction initialize where we can do things like save a reference
+ // to ourselves (need to be fully constructed).
+ void Initialize(uint32_t ssrc, AudioProviderInterface* provider);
+
+ private:
+ typedef std::list<AudioObserver*> AudioObserverList;
+
// AudioSourceInterface implementation.
void SetVolume(double volume) override;
void RegisterAudioObserver(AudioObserver* observer) override;
void UnregisterAudioObserver(AudioObserver* observer) override;
+ class Sink;
+ void OnData(const AudioSinkInterface::Data& audio);
+ void OnAudioProviderGone();
+
+ class MessageHandler;
+ void OnMessage(rtc::Message* msg);
+
AudioObserverList audio_observers_;
+ rtc::CriticalSection sink_lock_;
+ std::list<AudioTrackSinkInterface*> sinks_;
+ rtc::Thread* const main_thread_;
+ SourceState state_;
};
} // namespace webrtc
diff --git a/talk/app/webrtc/remoteaudiotrack.cc b/talk/app/webrtc/remoteaudiotrack.cc
index df27f05..2c4481c 100644
--- a/talk/app/webrtc/remoteaudiotrack.cc
+++ b/talk/app/webrtc/remoteaudiotrack.cc
@@ -26,3 +26,70 @@
*/
#include "talk/app/webrtc/remoteaudiotrack.h"
+
+#include "talk/app/webrtc/remoteaudiosource.h"
+
+using rtc::scoped_refptr;
+
+namespace webrtc {
+
+// static
+scoped_refptr<RemoteAudioTrack> RemoteAudioTrack::Create(
+ const std::string& id,
+ const scoped_refptr<RemoteAudioSource>& source) {
+ return new rtc::RefCountedObject<RemoteAudioTrack>(id, source);
+}
+
+RemoteAudioTrack::RemoteAudioTrack(
+ const std::string& label,
+ const scoped_refptr<RemoteAudioSource>& source)
+ : MediaStreamTrack<AudioTrackInterface>(label), audio_source_(source) {
+ audio_source_->RegisterObserver(this);
+ TrackState new_state = kInitializing;
+ switch (audio_source_->state()) {
+ case MediaSourceInterface::kLive:
+ case MediaSourceInterface::kMuted:
+ new_state = kLive;
+ break;
+ case MediaSourceInterface::kEnded:
+ new_state = kEnded;
+ break;
+ case MediaSourceInterface::kInitializing:
+ default:
+ // kInitializing;
+ break;
+ }
+ set_state(new_state);
+}
+
+RemoteAudioTrack::~RemoteAudioTrack() {
+ set_state(MediaStreamTrackInterface::kEnded);
+ audio_source_->UnregisterObserver(this);
+}
+
+std::string RemoteAudioTrack::kind() const {
+ return MediaStreamTrackInterface::kAudioKind;
+}
+
+AudioSourceInterface* RemoteAudioTrack::GetSource() const {
+ return audio_source_.get();
+}
+
+void RemoteAudioTrack::AddSink(AudioTrackSinkInterface* sink) {
+ audio_source_->AddSink(sink);
+}
+
+void RemoteAudioTrack::RemoveSink(AudioTrackSinkInterface* sink) {
+ audio_source_->RemoveSink(sink);
+}
+
+bool RemoteAudioTrack::GetSignalLevel(int* level) {
+ return false;
+}
+
+void RemoteAudioTrack::OnChanged() {
+ if (audio_source_->state() == MediaSourceInterface::kEnded)
+ set_state(MediaStreamTrackInterface::kEnded);
+}
+
+} // namespace webrtc
diff --git a/talk/app/webrtc/remoteaudiotrack.h b/talk/app/webrtc/remoteaudiotrack.h
index 4ce4e79..c9240eb 100644
--- a/talk/app/webrtc/remoteaudiotrack.h
+++ b/talk/app/webrtc/remoteaudiotrack.h
@@ -28,4 +28,50 @@
#ifndef TALK_APP_WEBRTC_REMOTEAUDIOTRACK_H_
#define TALK_APP_WEBRTC_REMOTEAUDIOTRACK_H_
+#include <string>
+
+#include "talk/app/webrtc/mediastreaminterface.h"
+#include "talk/app/webrtc/mediastreamtrack.h"
+#include "talk/app/webrtc/notifier.h"
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/base/scoped_ref_ptr.h"
+
+namespace webrtc {
+
+class RemoteAudioSource;
+
+class RemoteAudioTrack : public MediaStreamTrack<AudioTrackInterface>,
+ public ObserverInterface {
+ protected:
+ // Protected ctor to force use of factory method.
+ RemoteAudioTrack(const std::string& label,
+ const rtc::scoped_refptr<RemoteAudioSource>& source);
+ ~RemoteAudioTrack() override;
+
+ public:
+ static rtc::scoped_refptr<RemoteAudioTrack> Create(
+ const std::string& id,
+ const rtc::scoped_refptr<RemoteAudioSource>& source);
+
+ private:
+ // MediaStreamTrack implementation.
+ std::string kind() const override;
+
+ // AudioTrackInterface implementation.
+ AudioSourceInterface* GetSource() const override;
+
+ void AddSink(AudioTrackSinkInterface* sink) override;
+ void RemoveSink(AudioTrackSinkInterface* sink) override;
+ bool GetSignalLevel(int* level) override;
+
+ // ObserverInterface implementation.
+ void OnChanged() override;
+
+ private:
+ const rtc::scoped_refptr<RemoteAudioSource> audio_source_;
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RemoteAudioTrack);
+};
+
+} // namespace webrtc
+
#endif // TALK_APP_WEBRTC_REMOTEAUDIOTRACK_H_
diff --git a/talk/app/webrtc/rtpreceiver.h b/talk/app/webrtc/rtpreceiver.h
index a93ccbc..db021ba 100644
--- a/talk/app/webrtc/rtpreceiver.h
+++ b/talk/app/webrtc/rtpreceiver.h
@@ -68,10 +68,10 @@
private:
void Reconfigure();
- std::string id_;
- rtc::scoped_refptr<AudioTrackInterface> track_;
- uint32_t ssrc_;
- AudioProviderInterface* provider_;
+ const std::string id_;
+ const rtc::scoped_refptr<AudioTrackInterface> track_;
+ const uint32_t ssrc_;
+ AudioProviderInterface* provider_; // Set to null in Stop().
bool cached_track_enabled_;
};
diff --git a/talk/app/webrtc/rtpsenderreceiver_unittest.cc b/talk/app/webrtc/rtpsenderreceiver_unittest.cc
index c987186..3f61504 100644
--- a/talk/app/webrtc/rtpsenderreceiver_unittest.cc
+++ b/talk/app/webrtc/rtpsenderreceiver_unittest.cc
@@ -26,10 +26,12 @@
*/
#include <string>
+#include <utility>
#include "talk/app/webrtc/audiotrack.h"
#include "talk/app/webrtc/mediastream.h"
#include "talk/app/webrtc/remoteaudiosource.h"
+#include "talk/app/webrtc/remoteaudiotrack.h"
#include "talk/app/webrtc/rtpreceiver.h"
#include "talk/app/webrtc/rtpsender.h"
#include "talk/app/webrtc/streamcollection.h"
@@ -57,7 +59,8 @@
// Helper class to test RtpSender/RtpReceiver.
class MockAudioProvider : public AudioProviderInterface {
public:
- virtual ~MockAudioProvider() {}
+ ~MockAudioProvider() override {}
+
MOCK_METHOD2(SetAudioPlayout,
void(uint32_t ssrc,
bool enable));
@@ -67,6 +70,14 @@
const cricket::AudioOptions& options,
cricket::AudioRenderer* renderer));
MOCK_METHOD2(SetAudioPlayoutVolume, void(uint32_t ssrc, double volume));
+
+ void SetRawAudioSink(uint32_t,
+ rtc::scoped_ptr<AudioSinkInterface> sink) override {
+ sink_ = std::move(sink);
+ }
+
+ private:
+ rtc::scoped_ptr<AudioSinkInterface> sink_;
};
// Helper class to test RtpSender/RtpReceiver.
@@ -151,8 +162,8 @@
}
void CreateAudioRtpReceiver() {
- audio_track_ =
- AudioTrack::Create(kAudioTrackId, RemoteAudioSource::Create().get());
+ audio_track_ = RemoteAudioTrack::Create(
+ kAudioTrackId, RemoteAudioSource::Create(kAudioSsrc, NULL));
EXPECT_TRUE(stream_->AddTrack(audio_track_));
EXPECT_CALL(audio_provider_, SetAudioPlayout(kAudioSsrc, true));
audio_rtp_receiver_ = new AudioRtpReceiver(stream_->GetAudioTracks()[0],
diff --git a/talk/app/webrtc/webrtcsession.cc b/talk/app/webrtc/webrtcsession.cc
index 015531d..5e946a2 100644
--- a/talk/app/webrtc/webrtcsession.cc
+++ b/talk/app/webrtc/webrtcsession.cc
@@ -30,8 +30,9 @@
#include <limits.h>
#include <algorithm>
-#include <vector>
#include <set>
+#include <utility>
+#include <vector>
#include "talk/app/webrtc/jsepicecandidate.h"
#include "talk/app/webrtc/jsepsessiondescription.h"
@@ -44,6 +45,7 @@
#include "talk/session/media/channel.h"
#include "talk/session/media/channelmanager.h"
#include "talk/session/media/mediasession.h"
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/basictypes.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/helpers.h"
@@ -1318,6 +1320,15 @@
}
}
+void WebRtcSession::SetRawAudioSink(uint32_t ssrc,
+ rtc::scoped_ptr<AudioSinkInterface> sink) {
+ ASSERT(signaling_thread()->IsCurrent());
+ if (!voice_channel_)
+ return;
+
+ voice_channel_->SetRawAudioSink(ssrc, std::move(sink));
+}
+
bool WebRtcSession::SetCaptureDevice(uint32_t ssrc,
cricket::VideoCapturer* camera) {
ASSERT(signaling_thread()->IsCurrent());
diff --git a/talk/app/webrtc/webrtcsession.h b/talk/app/webrtc/webrtcsession.h
index d9c40d1..dd5ca18 100644
--- a/talk/app/webrtc/webrtcsession.h
+++ b/talk/app/webrtc/webrtcsession.h
@@ -38,11 +38,11 @@
#include "talk/app/webrtc/peerconnectioninterface.h"
#include "talk/app/webrtc/statstypes.h"
#include "talk/media/base/mediachannel.h"
-#include "webrtc/p2p/base/transportcontroller.h"
#include "talk/session/media/mediasession.h"
#include "webrtc/base/sigslot.h"
#include "webrtc/base/sslidentity.h"
#include "webrtc/base/thread.h"
+#include "webrtc/p2p/base/transportcontroller.h"
namespace cricket {
@@ -250,6 +250,8 @@
const cricket::AudioOptions& options,
cricket::AudioRenderer* renderer) override;
void SetAudioPlayoutVolume(uint32_t ssrc, double volume) override;
+ void SetRawAudioSink(uint32_t ssrc,
+ rtc::scoped_ptr<AudioSinkInterface> sink) override;
// Implements VideoMediaProviderInterface.
bool SetCaptureDevice(uint32_t ssrc, cricket::VideoCapturer* camera) override;
diff --git a/talk/media/base/fakemediaengine.h b/talk/media/base/fakemediaengine.h
index 9dbb649..f5b2174 100644
--- a/talk/media/base/fakemediaengine.h
+++ b/talk/media/base/fakemediaengine.h
@@ -38,9 +38,10 @@
#include "talk/media/base/mediaengine.h"
#include "talk/media/base/rtputils.h"
#include "talk/media/base/streamparams.h"
-#include "webrtc/p2p/base/sessiondescription.h"
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/buffer.h"
#include "webrtc/base/stringutils.h"
+#include "webrtc/p2p/base/sessiondescription.h"
namespace cricket {
@@ -346,6 +347,12 @@
virtual bool GetStats(VoiceMediaInfo* info) { return false; }
+ virtual void SetRawAudioSink(
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) {
+ sink_ = std::move(sink);
+ }
+
private:
class VoiceChannelAudioSink : public AudioRenderer::Sink {
public:
@@ -418,6 +425,7 @@
int time_since_last_typing_;
AudioOptions options_;
std::map<uint32_t, VoiceChannelAudioSink*> local_renderers_;
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink_;
};
// A helper function to compare the FakeVoiceMediaChannel::DtmfInfo.
diff --git a/talk/media/base/mediachannel.h b/talk/media/base/mediachannel.h
index fe223bb..44b9d4f 100644
--- a/talk/media/base/mediachannel.h
+++ b/talk/media/base/mediachannel.h
@@ -31,6 +31,7 @@
#include <string>
#include <vector>
+#include "talk/media/base/audiorenderer.h"
#include "talk/media/base/codec.h"
#include "talk/media/base/constants.h"
#include "talk/media/base/streamparams.h"
@@ -51,9 +52,12 @@
class Timing;
}
+namespace webrtc {
+class AudioSinkInterface;
+}
+
namespace cricket {
-class AudioRenderer;
struct RtpHeader;
class ScreencastId;
struct VideoFormat;
@@ -1028,6 +1032,10 @@
virtual bool InsertDtmf(uint32_t ssrc, int event, int duration) = 0;
// Gets quality stats for the channel.
virtual bool GetStats(VoiceMediaInfo* info) = 0;
+
+ virtual void SetRawAudioSink(
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) = 0;
};
struct VideoSendParameters : RtpSendParameters<VideoCodec, VideoOptions> {
diff --git a/talk/media/webrtc/fakewebrtccall.cc b/talk/media/webrtc/fakewebrtccall.cc
index bf51fb3..d50a53c 100644
--- a/talk/media/webrtc/fakewebrtccall.cc
+++ b/talk/media/webrtc/fakewebrtccall.cc
@@ -28,10 +28,12 @@
#include "talk/media/webrtc/fakewebrtccall.h"
#include <algorithm>
+#include <utility>
#include "talk/media/base/rtputils.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/gunit.h"
+#include "webrtc/audio/audio_sink.h"
namespace cricket {
FakeAudioSendStream::FakeAudioSendStream(
@@ -90,6 +92,11 @@
return stats_;
}
+void FakeAudioReceiveStream::SetSink(
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) {
+ sink_ = std::move(sink);
+}
+
FakeVideoSendStream::FakeVideoSendStream(
const webrtc::VideoSendStream::Config& config,
const webrtc::VideoEncoderConfig& encoder_config)
diff --git a/talk/media/webrtc/fakewebrtccall.h b/talk/media/webrtc/fakewebrtccall.h
index 024c50d..3528c7a 100644
--- a/talk/media/webrtc/fakewebrtccall.h
+++ b/talk/media/webrtc/fakewebrtccall.h
@@ -106,10 +106,12 @@
// webrtc::AudioReceiveStream implementation.
webrtc::AudioReceiveStream::Stats GetStats() const override;
+ void SetSink(rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) override;
webrtc::AudioReceiveStream::Config config_;
webrtc::AudioReceiveStream::Stats stats_;
int received_packets_;
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink_;
};
class FakeVideoSendStream final : public webrtc::VideoSendStream,
diff --git a/talk/media/webrtc/webrtcvoiceengine.cc b/talk/media/webrtc/webrtcvoiceengine.cc
index 0b5bed1..2634a24 100644
--- a/talk/media/webrtc/webrtcvoiceengine.cc
+++ b/talk/media/webrtc/webrtcvoiceengine.cc
@@ -44,6 +44,7 @@
#include "talk/media/base/streamparams.h"
#include "talk/media/webrtc/webrtcmediaengine.h"
#include "talk/media/webrtc/webrtcvoe.h"
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/arraysize.h"
#include "webrtc/base/base64.h"
#include "webrtc/base/byteorder.h"
@@ -1248,6 +1249,11 @@
return config_.voe_channel_id;
}
+ void SetRawAudioSink(rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) {
+ RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
+ stream_->SetSink(std::move(sink));
+ }
+
private:
void RecreateAudioReceiveStream(bool use_combined_bwe,
const std::vector<webrtc::RtpExtension>& extensions) {
@@ -2032,6 +2038,7 @@
// Clean up and delete the receive stream+channel.
LOG(LS_INFO) << "Removing audio receive stream " << ssrc
<< " with VoiceEngine channel #" << channel << ".";
+ it->second->SetRawAudioSink(nullptr);
delete it->second;
recv_streams_.erase(it);
return DeleteVoEChannel(channel);
@@ -2408,6 +2415,19 @@
return true;
}
+void WebRtcVoiceMediaChannel::SetRawAudioSink(
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) {
+ RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
+ LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::SetRawAudioSink";
+ const auto it = recv_streams_.find(ssrc);
+ if (it == recv_streams_.end()) {
+ LOG(LS_WARNING) << "SetRawAudioSink: no recv stream" << ssrc;
+ return;
+ }
+ it->second->SetRawAudioSink(std::move(sink));
+}
+
int WebRtcVoiceMediaChannel::GetOutputLevel(int channel) {
unsigned int ulevel = 0;
int ret = engine()->voe()->volume()->GetSpeechOutputLevel(channel, ulevel);
diff --git a/talk/media/webrtc/webrtcvoiceengine.h b/talk/media/webrtc/webrtcvoiceengine.h
index 1de4fb9..0f2f59e 100644
--- a/talk/media/webrtc/webrtcvoiceengine.h
+++ b/talk/media/webrtc/webrtcvoiceengine.h
@@ -196,6 +196,10 @@
void OnReadyToSend(bool ready) override {}
bool GetStats(VoiceMediaInfo* info) override;
+ void SetRawAudioSink(
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) override;
+
// implements Transport interface
bool SendRtp(const uint8_t* data,
size_t len,
diff --git a/talk/session/media/channel.cc b/talk/session/media/channel.cc
index 588a036..9180852 100644
--- a/talk/session/media/channel.cc
+++ b/talk/session/media/channel.cc
@@ -30,6 +30,7 @@
#include "talk/media/base/constants.h"
#include "talk/media/base/rtputils.h"
#include "talk/session/media/channelmanager.h"
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/bind.h"
#include "webrtc/base/buffer.h"
#include "webrtc/base/byteorder.h"
@@ -40,9 +41,18 @@
#include "webrtc/p2p/base/transportchannel.h"
namespace cricket {
-
using rtc::Bind;
+namespace {
+// See comment below for why we need to use a pointer to a scoped_ptr.
+bool SetRawAudioSink_w(VoiceMediaChannel* channel,
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface>* sink) {
+ channel->SetRawAudioSink(ssrc, std::move(*sink));
+ return true;
+}
+} // namespace
+
enum {
MSG_EARLYMEDIATIMEOUT = 1,
MSG_SCREENCASTWINDOWEVENT,
@@ -1376,6 +1386,15 @@
media_channel(), ssrc, volume));
}
+void VoiceChannel::SetRawAudioSink(
+ uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink) {
+ // We need to work around Bind's lack of support for scoped_ptr and ownership
+ // passing. So we invoke to our own little routine that gets a pointer to
+ // our local variable. This is OK since we're synchronously invoking.
+ InvokeOnWorker(Bind(&SetRawAudioSink_w, media_channel(), ssrc, &sink));
+}
+
bool VoiceChannel::GetStats(VoiceMediaInfo* stats) {
return InvokeOnWorker(Bind(&VoiceMediaChannel::GetStats,
media_channel(), stats));
diff --git a/talk/session/media/channel.h b/talk/session/media/channel.h
index 1140954..8faefe6 100644
--- a/talk/session/media/channel.h
+++ b/talk/session/media/channel.h
@@ -38,19 +38,24 @@
#include "talk/media/base/mediaengine.h"
#include "talk/media/base/streamparams.h"
#include "talk/media/base/videocapturer.h"
-#include "webrtc/p2p/base/transportcontroller.h"
-#include "webrtc/p2p/client/socketmonitor.h"
#include "talk/session/media/audiomonitor.h"
#include "talk/session/media/bundlefilter.h"
#include "talk/session/media/mediamonitor.h"
#include "talk/session/media/mediasession.h"
#include "talk/session/media/rtcpmuxfilter.h"
#include "talk/session/media/srtpfilter.h"
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/asyncudpsocket.h"
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/network.h"
#include "webrtc/base/sigslot.h"
#include "webrtc/base/window.h"
+#include "webrtc/p2p/base/transportcontroller.h"
+#include "webrtc/p2p/client/socketmonitor.h"
+
+namespace webrtc {
+class AudioSinkInterface;
+} // namespace webrtc
namespace cricket {
@@ -367,6 +372,9 @@
// event 0-9, *, #, A-D.
bool InsertDtmf(uint32_t ssrc, int event_code, int duration);
bool SetOutputVolume(uint32_t ssrc, double volume);
+ void SetRawAudioSink(uint32_t ssrc,
+ rtc::scoped_ptr<webrtc::AudioSinkInterface> sink);
+
// Get statistics about the current media session.
bool GetStats(VoiceMediaInfo* stats);
diff --git a/talk/session/media/channel_unittest.cc b/talk/session/media/channel_unittest.cc
index 08d83b5..35e7142 100644
--- a/talk/session/media/channel_unittest.cc
+++ b/talk/session/media/channel_unittest.cc
@@ -34,7 +34,6 @@
#include "talk/media/base/screencastid.h"
#include "talk/media/base/testutils.h"
#include "talk/session/media/channel.h"
-#include "webrtc/p2p/base/faketransportcontroller.h"
#include "webrtc/base/arraysize.h"
#include "webrtc/base/fileutils.h"
#include "webrtc/base/gunit.h"
@@ -45,6 +44,7 @@
#include "webrtc/base/ssladapter.h"
#include "webrtc/base/sslidentity.h"
#include "webrtc/base/window.h"
+#include "webrtc/p2p/base/faketransportcontroller.h"
#define MAYBE_SKIP_TEST(feature) \
if (!(rtc::SSLStreamAdapter::feature())) { \
diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn
index abae434..5a9902e 100644
--- a/webrtc/audio/BUILD.gn
+++ b/webrtc/audio/BUILD.gn
@@ -14,6 +14,7 @@
"audio_receive_stream.h",
"audio_send_stream.cc",
"audio_send_stream.h",
+ "audio_sink.h",
"audio_state.cc",
"audio_state.h",
"conversion.h",
@@ -31,7 +32,7 @@
deps = [
"..:webrtc_common",
- "../voice_engine",
"../system_wrappers",
+ "../voice_engine",
]
}
diff --git a/webrtc/audio/audio_receive_stream.cc b/webrtc/audio/audio_receive_stream.cc
index 87cb215..dfad79f 100644
--- a/webrtc/audio/audio_receive_stream.cc
+++ b/webrtc/audio/audio_receive_stream.cc
@@ -11,7 +11,9 @@
#include "webrtc/audio/audio_receive_stream.h"
#include <string>
+#include <utility>
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/audio/audio_state.h"
#include "webrtc/audio/conversion.h"
#include "webrtc/base/checks.h"
@@ -201,6 +203,11 @@
return stats;
}
+void AudioReceiveStream::SetSink(rtc::scoped_ptr<AudioSinkInterface> sink) {
+ RTC_DCHECK(thread_checker_.CalledOnValidThread());
+ channel_proxy_->SetSink(std::move(sink));
+}
+
const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
return config_;
diff --git a/webrtc/audio/audio_receive_stream.h b/webrtc/audio/audio_receive_stream.h
index 08e65cd..42286af 100644
--- a/webrtc/audio/audio_receive_stream.h
+++ b/webrtc/audio/audio_receive_stream.h
@@ -24,6 +24,7 @@
} // namespace voe
namespace internal {
+
class AudioReceiveStream final : public webrtc::AudioReceiveStream {
public:
AudioReceiveStream(RemoteBitrateEstimator* remote_bitrate_estimator,
@@ -43,6 +44,8 @@
// webrtc::AudioReceiveStream implementation.
webrtc::AudioReceiveStream::Stats GetStats() const override;
+ void SetSink(rtc::scoped_ptr<AudioSinkInterface> sink) override;
+
const webrtc::AudioReceiveStream::Config& config() const;
private:
diff --git a/webrtc/audio/audio_sink.h b/webrtc/audio/audio_sink.h
new file mode 100644
index 0000000..d022b32
--- /dev/null
+++ b/webrtc/audio/audio_sink.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_AUDIO_AUDIO_SINK_H_
+#define WEBRTC_AUDIO_AUDIO_SINK_H_
+
+#if defined(WEBRTC_POSIX) && !defined(__STDC_FORMAT_MACROS)
+// Avoid conflict with format_macros.h.
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <stddef.h>
+
+namespace webrtc {
+
+// Represents a simple push audio sink.
+class AudioSinkInterface {
+ public:
+ virtual ~AudioSinkInterface() {}
+
+ struct Data {
+ Data(int16_t* data,
+ size_t samples_per_channel,
+ int sample_rate,
+ int channels,
+ uint32_t timestamp)
+ : data(data),
+ samples_per_channel(samples_per_channel),
+ sample_rate(sample_rate),
+ channels(channels),
+ timestamp(timestamp) {}
+
+ int16_t* data; // The actual 16bit audio data.
+ size_t samples_per_channel; // Number of frames in the buffer.
+ int sample_rate; // Sample rate in Hz.
+ int channels; // Number of channels in the audio data.
+ uint32_t timestamp; // The RTP timestamp of the first sample.
+ };
+
+ virtual void OnData(const Data& audio) = 0;
+};
+
+} // namespace webrtc
+
+#endif // WEBRTC_AUDIO_AUDIO_SINK_H_
diff --git a/webrtc/audio/webrtc_audio.gypi b/webrtc/audio/webrtc_audio.gypi
index 9b4879a..53b7d16 100644
--- a/webrtc/audio/webrtc_audio.gypi
+++ b/webrtc/audio/webrtc_audio.gypi
@@ -18,6 +18,7 @@
'audio/audio_receive_stream.h',
'audio/audio_send_stream.cc',
'audio/audio_send_stream.h',
+ 'audio/audio_sink.h',
'audio/audio_state.cc',
'audio/audio_state.h',
'audio/conversion.h',
diff --git a/webrtc/audio_receive_stream.h b/webrtc/audio_receive_stream.h
index 356a3a3..daf4598 100644
--- a/webrtc/audio_receive_stream.h
+++ b/webrtc/audio_receive_stream.h
@@ -15,6 +15,7 @@
#include <string>
#include <vector>
+#include "webrtc/base/scoped_ptr.h"
#include "webrtc/config.h"
#include "webrtc/stream.h"
#include "webrtc/transport.h"
@@ -23,6 +24,7 @@
namespace webrtc {
class AudioDecoder;
+class AudioSinkInterface;
// WORK IN PROGRESS
// This class is under development and is not yet intended for for use outside
@@ -100,6 +102,16 @@
};
virtual Stats GetStats() const = 0;
+
+ // Sets an audio sink that receives unmixed audio from the receive stream.
+ // Ownership of the sink is passed to the stream and can be used by the
+ // caller to do lifetime management (i.e. when the sink's dtor is called).
+ // Only one sink can be set and passing a null sink, clears an existing one.
+ // NOTE: Audio must still somehow be pulled through AudioTransport for audio
+ // to stream through this sink. In practice, this happens if mixed audio
+ // is being pulled+rendered and/or if audio is being pulled for the purposes
+ // of feeding to the AEC.
+ virtual void SetSink(rtc::scoped_ptr<AudioSinkInterface> sink) = 0;
};
} // namespace webrtc
diff --git a/webrtc/base/format_macros.h b/webrtc/base/format_macros.h
index 5d7dcc3..90f86a6 100644
--- a/webrtc/base/format_macros.h
+++ b/webrtc/base/format_macros.h
@@ -73,6 +73,8 @@
#else // WEBRTC_WIN
+#include <inttypes.h>
+
#if !defined(PRId64)
#define PRId64 "I64d"
#endif
diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc
index 18f3408..48111db 100644
--- a/webrtc/voice_engine/channel.cc
+++ b/webrtc/voice_engine/channel.cc
@@ -11,6 +11,7 @@
#include "webrtc/voice_engine/channel.h"
#include <algorithm>
+#include <utility>
#include "webrtc/base/checks.h"
#include "webrtc/base/format_macros.h"
@@ -560,6 +561,21 @@
}
}
+ {
+ // Pass the audio buffers to an optional sink callback, before applying
+ // scaling/panning, as that applies to the mix operation.
+ // External recipients of the audio (e.g. via AudioTrack), will do their
+ // own mixing/dynamic processing.
+ CriticalSectionScoped cs(&_callbackCritSect);
+ if (audio_sink_) {
+ AudioSinkInterface::Data data(
+ &audioFrame->data_[0],
+ audioFrame->samples_per_channel_, audioFrame->sample_rate_hz_,
+ audioFrame->num_channels_, audioFrame->timestamp_);
+ audio_sink_->OnData(data);
+ }
+ }
+
float output_gain = 1.0f;
float left_pan = 1.0f;
float right_pan = 1.0f;
@@ -608,13 +624,10 @@
const bool isStereo = (audioFrame->num_channels_ == 2);
if (_outputExternalMediaCallbackPtr)
{
- _outputExternalMediaCallbackPtr->Process(
- _channelId,
- kPlaybackPerChannel,
- (int16_t*)audioFrame->data_,
- audioFrame->samples_per_channel_,
- audioFrame->sample_rate_hz_,
- isStereo);
+ _outputExternalMediaCallbackPtr->Process(
+ _channelId, kPlaybackPerChannel, (int16_t*)audioFrame->data_,
+ audioFrame->samples_per_channel_, audioFrame->sample_rate_hz_,
+ isStereo);
}
}
@@ -1172,6 +1185,11 @@
return 0;
}
+void Channel::SetSink(rtc::scoped_ptr<AudioSinkInterface> sink) {
+ CriticalSectionScoped cs(&_callbackCritSect);
+ audio_sink_ = std::move(sink);
+}
+
int32_t
Channel::StartPlayout()
{
diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h
index d3b1b93..9184b93 100644
--- a/webrtc/voice_engine/channel.h
+++ b/webrtc/voice_engine/channel.h
@@ -11,6 +11,7 @@
#ifndef WEBRTC_VOICE_ENGINE_CHANNEL_H_
#define WEBRTC_VOICE_ENGINE_CHANNEL_H_
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/common_audio/resampler/include/push_resampler.h"
@@ -192,6 +193,8 @@
CriticalSectionWrapper* callbackCritSect);
int32_t UpdateLocalTimeStamp();
+ void SetSink(rtc::scoped_ptr<AudioSinkInterface> sink);
+
// API methods
// VoEBase
@@ -508,6 +511,7 @@
TelephoneEventHandler* telephone_event_handler_;
rtc::scoped_ptr<RtpRtcp> _rtpRtcpModule;
rtc::scoped_ptr<AudioCodingModule> audio_coding_;
+ rtc::scoped_ptr<AudioSinkInterface> audio_sink_;
AudioLevel _outputAudioLevel;
bool _externalTransport;
AudioFrame _audioFrame;
diff --git a/webrtc/voice_engine/channel_proxy.cc b/webrtc/voice_engine/channel_proxy.cc
index 68fbf38..f54c81e 100644
--- a/webrtc/voice_engine/channel_proxy.cc
+++ b/webrtc/voice_engine/channel_proxy.cc
@@ -10,6 +10,9 @@
#include "webrtc/voice_engine/channel_proxy.h"
+#include <utility>
+
+#include "webrtc/audio/audio_sink.h"
#include "webrtc/base/checks.h"
#include "webrtc/voice_engine/channel.h"
@@ -22,6 +25,8 @@
RTC_CHECK(channel_owner_.channel());
}
+ChannelProxy::~ChannelProxy() {}
+
void ChannelProxy::SetRTCPStatus(bool enable) {
channel()->SetRTCPStatus(enable);
}
@@ -134,6 +139,11 @@
channel()->SendTelephoneEventOutband(event, duration_ms, 10, false) == 0;
}
+void ChannelProxy::SetSink(rtc::scoped_ptr<AudioSinkInterface> sink) {
+ RTC_DCHECK(thread_checker_.CalledOnValidThread());
+ channel()->SetSink(std::move(sink));
+}
+
Channel* ChannelProxy::channel() const {
RTC_DCHECK(channel_owner_.channel());
return channel_owner_.channel();
diff --git a/webrtc/voice_engine/channel_proxy.h b/webrtc/voice_engine/channel_proxy.h
index fa33e6c..b990d91 100644
--- a/webrtc/voice_engine/channel_proxy.h
+++ b/webrtc/voice_engine/channel_proxy.h
@@ -20,6 +20,7 @@
namespace webrtc {
+class AudioSinkInterface;
class PacketRouter;
class RtpPacketSender;
class TransportFeedbackObserver;
@@ -39,7 +40,7 @@
public:
ChannelProxy();
explicit ChannelProxy(const ChannelOwner& channel_owner);
- virtual ~ChannelProxy() {}
+ virtual ~ChannelProxy();
virtual void SetRTCPStatus(bool enable);
virtual void SetLocalSSRC(uint32_t ssrc);
@@ -64,6 +65,8 @@
virtual bool SetSendTelephoneEventPayloadType(int payload_type);
virtual bool SendTelephoneEventOutband(uint8_t event, uint32_t duration_ms);
+ virtual void SetSink(rtc::scoped_ptr<AudioSinkInterface> sink);
+
private:
Channel* channel() const;
