Remove additional channel constraints when Beamforming is enabled in AudioProcessing
The general constraints on number of channels for AudioProcessing is:
num_in_channels == num_out_channels || num_out_channels == 1
When Beamforming is enabled and additional constraint was added forcing:
num_out_channels == 1
This artificial constraint was removed by adding upmixing support in CopyTo, since it was already supported for the AudioFrame interface using InterleaveTo.
Review URL: https://codereview.webrtc.org/1571013002
Cr-Commit-Position: refs/heads/master@{#11215}
diff --git a/talk/media/webrtc/fakewebrtcvoiceengine.h b/talk/media/webrtc/fakewebrtcvoiceengine.h
index bf22a29..eb3b4d3 100644
--- a/talk/media/webrtc/fakewebrtcvoiceengine.h
+++ b/talk/media/webrtc/fakewebrtcvoiceengine.h
@@ -78,6 +78,7 @@
WEBRTC_STUB_CONST(proc_sample_rate_hz, ());
WEBRTC_STUB_CONST(proc_split_sample_rate_hz, ());
WEBRTC_STUB_CONST(num_input_channels, ());
+ WEBRTC_STUB_CONST(num_proc_channels, ());
WEBRTC_STUB_CONST(num_output_channels, ());
WEBRTC_STUB_CONST(num_reverse_channels, ());
WEBRTC_VOID_STUB(set_output_will_be_muted, (bool muted));
diff --git a/webrtc/modules/audio_processing/audio_buffer.cc b/webrtc/modules/audio_processing/audio_buffer.cc
index c1c4061..77bda79 100644
--- a/webrtc/modules/audio_processing/audio_buffer.cc
+++ b/webrtc/modules/audio_processing/audio_buffer.cc
@@ -150,7 +150,7 @@
void AudioBuffer::CopyTo(const StreamConfig& stream_config,
float* const* data) {
assert(stream_config.num_frames() == output_num_frames_);
- assert(stream_config.num_channels() == num_channels_);
+ assert(stream_config.num_channels() == num_channels_ || num_channels_ == 1);
// Convert to the float range.
float* const* data_ptr = data;
@@ -173,6 +173,11 @@
output_num_frames_);
}
}
+
+ // Upmix.
+ for (int i = num_channels_; i < stream_config.num_channels(); ++i) {
+ memcpy(data[i], data[0], output_num_frames_ * sizeof(**data));
+ }
}
void AudioBuffer::InitForNewData() {
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc
index 67709b2..fea5785 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.cc
+++ b/webrtc/modules/audio_processing/audio_processing_impl.cc
@@ -226,9 +226,9 @@
#else
capture_(config.Get<ExperimentalNs>().enabled,
#endif
- config.Get<Beamforming>().enabled,
config.Get<Beamforming>().array_geometry,
- config.Get<Beamforming>().target_direction)
+ config.Get<Beamforming>().target_direction),
+ capture_nonlocked_(config.Get<Beamforming>().enabled)
{
{
rtc::CritScope cs_render(&crit_render_);
@@ -345,7 +345,7 @@
int AudioProcessingImpl::InitializeLocked() {
const int fwd_audio_buffer_channels =
- capture_.beamformer_enabled
+ capture_nonlocked_.beamformer_enabled
? formats_.api_format.input_stream().num_channels()
: formats_.api_format.output_stream().num_channels();
const int rev_audio_buffer_out_num_frames =
@@ -428,9 +428,8 @@
return kBadNumberChannelsError;
}
- if (capture_.beamformer_enabled &&
- (static_cast<size_t>(num_in_channels) != capture_.array_geometry.size() ||
- num_out_channels > 1)) {
+ if (capture_nonlocked_.beamformer_enabled &&
+ static_cast<size_t>(num_in_channels) != capture_.array_geometry.size()) {
return kBadNumberChannelsError;
}
@@ -500,8 +499,9 @@
}
#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
- if (capture_.beamformer_enabled != config.Get<Beamforming>().enabled) {
- capture_.beamformer_enabled = config.Get<Beamforming>().enabled;
+ if (capture_nonlocked_.beamformer_enabled !=
+ config.Get<Beamforming>().enabled) {
+ capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled;
if (config.Get<Beamforming>().array_geometry.size() > 1) {
capture_.array_geometry = config.Get<Beamforming>().array_geometry;
}
@@ -537,6 +537,11 @@
return formats_.api_format.input_stream().num_channels();
}
+int AudioProcessingImpl::num_proc_channels() const {
+ // Used as callback from submodules, hence locking is not allowed.
+ return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels();
+}
+
int AudioProcessingImpl::num_output_channels() const {
// Used as callback from submodules, hence locking is not allowed.
return formats_.api_format.output_stream().num_channels();
@@ -771,7 +776,7 @@
ca->num_channels());
}
- if (capture_.beamformer_enabled) {
+ if (capture_nonlocked_.beamformer_enabled) {
private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(),
ca->split_data_f());
ca->set_num_channels(1);
@@ -793,7 +798,7 @@
if (constants_.use_new_agc &&
public_submodules_->gain_control->is_enabled() &&
- (!capture_.beamformer_enabled ||
+ (!capture_nonlocked_.beamformer_enabled ||
private_submodules_->beamformer->is_target_present())) {
private_submodules_->agc_manager->Process(
ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(),
@@ -1183,7 +1188,7 @@
}
bool AudioProcessingImpl::is_data_processed() const {
- if (capture_.beamformer_enabled) {
+ if (capture_nonlocked_.beamformer_enabled) {
return true;
}
@@ -1293,12 +1298,12 @@
public_submodules_->transient_suppressor->Initialize(
capture_nonlocked_.fwd_proc_format.sample_rate_hz(),
capture_nonlocked_.split_rate,
- formats_.api_format.output_stream().num_channels());
+ num_proc_channels());
}
}
void AudioProcessingImpl::InitializeBeamformer() {
- if (capture_.beamformer_enabled) {
+ if (capture_nonlocked_.beamformer_enabled) {
if (!private_submodules_->beamformer) {
private_submodules_->beamformer.reset(new NonlinearBeamformer(
capture_.array_geometry, capture_.target_direction));
@@ -1320,12 +1325,12 @@
}
void AudioProcessingImpl::InitializeHighPassFilter() {
- public_submodules_->high_pass_filter->Initialize(num_output_channels(),
+ public_submodules_->high_pass_filter->Initialize(num_proc_channels(),
proc_sample_rate_hz());
}
void AudioProcessingImpl::InitializeNoiseSuppression() {
- public_submodules_->noise_suppression->Initialize(num_output_channels(),
+ public_submodules_->noise_suppression->Initialize(num_proc_channels(),
proc_sample_rate_hz());
}
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h
index 39f87ac..6cb9e8c 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.h
+++ b/webrtc/modules/audio_processing/audio_processing_impl.h
@@ -102,6 +102,7 @@
int proc_sample_rate_hz() const override;
int proc_split_sample_rate_hz() const override;
int num_input_channels() const override;
+ int num_proc_channels() const override;
int num_output_channels() const override;
int num_reverse_channels() const override;
int stream_delay_ms() const override;
@@ -280,7 +281,6 @@
struct ApmCaptureState {
ApmCaptureState(bool transient_suppressor_enabled,
- bool beamformer_enabled,
const std::vector<Point>& array_geometry,
SphericalPointf target_direction)
: aec_system_delay_jumps(-1),
@@ -292,7 +292,6 @@
output_will_be_muted(false),
key_pressed(false),
transient_suppressor_enabled(transient_suppressor_enabled),
- beamformer_enabled(beamformer_enabled),
array_geometry(array_geometry),
target_direction(target_direction),
fwd_proc_format(kSampleRate16kHz),
@@ -306,7 +305,6 @@
bool output_will_be_muted;
bool key_pressed;
bool transient_suppressor_enabled;
- bool beamformer_enabled;
std::vector<Point> array_geometry;
SphericalPointf target_direction;
rtc::scoped_ptr<AudioBuffer> capture_audio;
@@ -318,16 +316,18 @@
} capture_ GUARDED_BY(crit_capture_);
struct ApmCaptureNonLockedState {
- ApmCaptureNonLockedState()
+ ApmCaptureNonLockedState(bool beamformer_enabled)
: fwd_proc_format(kSampleRate16kHz),
split_rate(kSampleRate16kHz),
- stream_delay_ms(0) {}
+ stream_delay_ms(0),
+ beamformer_enabled(beamformer_enabled) {}
// Only the rate and samples fields of fwd_proc_format_ are used because the
// forward processing number of channels is mutable and is tracked by the
// capture_audio_.
StreamConfig fwd_proc_format;
int split_rate;
int stream_delay_ms;
+ bool beamformer_enabled;
} capture_nonlocked_;
struct ApmRenderState {
diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc
index bdcad20..13e71bc 100644
--- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc
+++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc
@@ -174,7 +174,7 @@
}
assert(audio->num_frames_per_band() <= 160);
- assert(audio->num_channels() == apm_->num_output_channels());
+ assert(audio->num_channels() == apm_->num_proc_channels());
int err = AudioProcessing::kNoError;
diff --git a/webrtc/modules/audio_processing/gain_control_impl.cc b/webrtc/modules/audio_processing/gain_control_impl.cc
index b9b3564..7b284e8 100644
--- a/webrtc/modules/audio_processing/gain_control_impl.cc
+++ b/webrtc/modules/audio_processing/gain_control_impl.cc
@@ -435,7 +435,7 @@
int GainControlImpl::num_handles_required() const {
// Not locked as it only relies on APM public API which is threadsafe.
- return apm_->num_output_channels();
+ return apm_->num_proc_channels();
}
int GainControlImpl::GetHandleError(void* handle) const {
diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h
index 5fcc4d4..d39d27e 100644
--- a/webrtc/modules/audio_processing/include/audio_processing.h
+++ b/webrtc/modules/audio_processing/include/audio_processing.h
@@ -288,6 +288,7 @@
virtual int proc_sample_rate_hz() const = 0;
virtual int proc_split_sample_rate_hz() const = 0;
virtual int num_input_channels() const = 0;
+ virtual int num_proc_channels() const = 0;
virtual int num_output_channels() const = 0;
virtual int num_reverse_channels() const = 0;
