Add a BWE fairness test.
Also moves the BWE perf tests to webrtc_perf_tests for tracking.
BUG=4548
R=pbos@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/45189004
Cr-Commit-Position: refs/heads/master@{#9022}
diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp
index 1bf8466..dbf0187 100644
--- a/webrtc/modules/modules.gyp
+++ b/webrtc/modules/modules.gyp
@@ -60,6 +60,7 @@
'audio_device' ,
'audio_processing',
'bitrate_controller',
+ 'bwe_simulator',
'CNG',
'desktop_capture',
'iSACFix',
@@ -216,33 +217,6 @@
'remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc',
'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc',
'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h',
- 'remote_bitrate_estimator/remote_bitrate_estimators_test.cc',
- 'remote_bitrate_estimator/test/bwe.cc',
- 'remote_bitrate_estimator/test/bwe.h',
- 'remote_bitrate_estimator/test/bwe_test.cc',
- 'remote_bitrate_estimator/test/bwe_test.h',
- 'remote_bitrate_estimator/test/bwe_test_baselinefile.cc',
- 'remote_bitrate_estimator/test/bwe_test_baselinefile.h',
- 'remote_bitrate_estimator/test/bwe_test_fileutils.cc',
- 'remote_bitrate_estimator/test/bwe_test_fileutils.h',
- 'remote_bitrate_estimator/test/bwe_test_framework.cc',
- 'remote_bitrate_estimator/test/bwe_test_framework.h',
- 'remote_bitrate_estimator/test/bwe_test_framework_unittest.cc',
- 'remote_bitrate_estimator/test/bwe_test_logging.cc',
- 'remote_bitrate_estimator/test/bwe_test_logging.h',
- 'remote_bitrate_estimator/test/packet_receiver.cc',
- 'remote_bitrate_estimator/test/packet_receiver.h',
- 'remote_bitrate_estimator/test/packet_sender.cc',
- 'remote_bitrate_estimator/test/packet_sender.h',
- 'remote_bitrate_estimator/test/packet.h',
- 'remote_bitrate_estimator/test/estimators/nada.cc',
- 'remote_bitrate_estimator/test/estimators/nada.h',
- 'remote_bitrate_estimator/test/estimators/remb.cc',
- 'remote_bitrate_estimator/test/estimators/remb.h',
- 'remote_bitrate_estimator/test/estimators/send_side.cc',
- 'remote_bitrate_estimator/test/estimators/send_side.h',
- 'remote_bitrate_estimator/test/estimators/tcp.cc',
- 'remote_bitrate_estimator/test/estimators/tcp.h',
'rtp_rtcp/source/mock/mock_rtp_payload_strategy.h',
'rtp_rtcp/source/byte_io_unittest.cc',
'rtp_rtcp/source/fec_receiver_unittest.cc',
diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi
index 21d3bd5..c8a48c1 100644
--- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi
+++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi
@@ -56,6 +56,41 @@
['include_tests==1', {
'targets': [
{
+ 'target_name': 'bwe_simulator',
+ 'type': 'static_library',
+ 'dependencies': [
+ '<(DEPTH)/testing/gtest.gyp:gtest',
+ ],
+ 'sources': [
+ 'test/bwe.cc',
+ 'test/bwe.h',
+ 'test/bwe_test.cc',
+ 'test/bwe_test.h',
+ 'test/bwe_test_baselinefile.cc',
+ 'test/bwe_test_baselinefile.h',
+ 'test/bwe_test_fileutils.cc',
+ 'test/bwe_test_fileutils.h',
+ 'test/bwe_test_framework.cc',
+ 'test/bwe_test_framework.h',
+ 'test/bwe_test_framework_unittest.cc',
+ 'test/bwe_test_logging.cc',
+ 'test/bwe_test_logging.h',
+ 'test/packet_receiver.cc',
+ 'test/packet_receiver.h',
+ 'test/packet_sender.cc',
+ 'test/packet_sender.h',
+ 'test/packet.h',
+ 'test/estimators/nada.cc',
+ 'test/estimators/nada.h',
+ 'test/estimators/remb.cc',
+ 'test/estimators/remb.h',
+ 'test/estimators/send_side.cc',
+ 'test/estimators/send_side.h',
+ 'test/estimators/tcp.cc',
+ 'test/estimators/tcp.h',
+ ],
+ },
+ {
'target_name': 'bwe_tools_util',
'type': 'static_library',
'dependencies': [
diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc
index b64033e..837c4dd 100644
--- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc
+++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc
@@ -241,8 +241,10 @@
BweFeedbackTest() : BweTest() {}
virtual ~BweFeedbackTest() {}
- void PrintResults(double max_throughput_kbps, Stats<double> throughput_kbps,
- Stats<double> delay_ms) {
+ void PrintResults(double max_throughput_kbps,
+ Stats<double> throughput_kbps,
+ Stats<double> delay_ms,
+ std::vector<Stats<double>> flow_throughput_kbps) {
double utilization = throughput_kbps.GetMean() / max_throughput_kbps;
webrtc::test::PrintResult("BwePerformance",
GetTestName(),
@@ -264,6 +266,16 @@
delay_ms.AsString(),
"ms",
false);
+ double fairness_index = 0.0;
+ double squared_bitrate_sum = 0.0;
+ for (Stats<double> flow : flow_throughput_kbps) {
+ squared_bitrate_sum += flow.GetMean() * flow.GetMean();
+ fairness_index += flow.GetMean();
+ }
+ fairness_index *= fairness_index;
+ fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum;
+ webrtc::test::PrintResult("BwePerformance", GetTestName(), "Fairness",
+ fairness_index * 100, "%", false);
}
protected:
@@ -294,7 +306,8 @@
filter.SetCapacity(kHighCapacityKbps);
RunFor(60 * 1000);
PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
- counter.GetBitrateStats(), filter.GetDelayStats());
+ counter.GetBitrateStats(), filter.GetDelayStats(),
+ std::vector<Stats<double>>());
}
TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) {
@@ -314,7 +327,8 @@
RunFor(60 * 1000);
PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
- counter.GetBitrateStats(), filter.GetDelayStats());
+ counter.GetBitrateStats(), filter.GetDelayStats(),
+ std::vector<Stats<double>>());
}
TEST_P(BweFeedbackTest, Verizon4gDownlinkTest) {
@@ -327,7 +341,7 @@
ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
RunFor(22 * 60 * 1000);
PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
- filter.GetDelayStats());
+ filter.GetDelayStats(), std::vector<Stats<double>>());
}
// webrtc:3277
@@ -342,7 +356,50 @@
ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
RunFor(300 * 1000);
PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
- filter.GetDelayStats());
+ filter.GetDelayStats(), std::vector<Stats<double>>());
+}
+
+TEST_P(BweFeedbackTest, PacedSelfFairnessTest) {
+ srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
+ const int kAllFlowIds[] = {0, 1, 2, 3};
+ const size_t kNumFlows = sizeof(kAllFlowIds) / sizeof(kAllFlowIds[0]);
+ rtc::scoped_ptr<VideoSource> sources[kNumFlows];
+ rtc::scoped_ptr<PacketSender> senders[kNumFlows];
+
+ for (size_t i = 0; i < kNumFlows; ++i) {
+ // Streams started woth ramdp, pffsets tp give them different advantage when
+ // competing for the bandwidth.
+ sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
+ i * (rand() % 40000)));
+ senders[i].reset(
+ new PacedVideoSender(&uplink_, sources[i].get(), GetParam()));
+ }
+
+ ChokeFilter choke(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
+ choke.SetCapacity(3000);
+ choke.SetMaxDelay(1000);
+
+ rtc::scoped_ptr<RateCounterFilter> rate_counters[kNumFlows];
+ for (size_t i = 0; i < kNumFlows; ++i) {
+ rate_counters[i].reset(new RateCounterFilter(
+ &uplink_, CreateFlowIds(&kAllFlowIds[i], 1), "receiver_input"));
+ }
+
+ RateCounterFilter total_utilization(
+ &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "total_utilization");
+
+ rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
+ for (size_t i = 0; i < kNumFlows; ++i) {
+ receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], GetParam(),
+ i == 0, false));
+ }
+ RunFor(15 * 60 * 1000);
+
+ std::vector<Stats<double>> flow_throughput_kbps;
+ for (size_t i = 0; i < kNumFlows; ++i)
+ flow_throughput_kbps.push_back(rate_counters[i]->GetBitrateStats());
+ PrintResults(3000, total_utilization.GetBitrateStats(), choke.GetDelayStats(),
+ flow_throughput_kbps);
}
} // namespace bwe
} // namespace testing
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc
index 1b1117d..b92b39e 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc
@@ -615,7 +615,7 @@
void TestChoke(PacketProcessor* filter,
int64_t run_for_ms,
uint32_t packets_to_generate,
- uint32_t expected_kbit_transmitted) {
+ size_t expected_kbit_transmitted) {
// Generate a bunch of packets, apply choke, verify output is ordered.
Packets packets;
RTPHeader header;
@@ -634,7 +634,7 @@
ASSERT_TRUE(IsSequenceNumberSorted(output_packets_));
// Sum up the transmitted bytes up until the current time.
- uint32_t bytes_transmitted = 0;
+ size_t bytes_transmitted = 0;
while (!output_packets_.empty()) {
const Packet* packet = output_packets_.front();
if (packet->send_time_us() > now_ms_ * 1000) {
@@ -769,7 +769,7 @@
int64_t run_for_ms,
uint32_t expected_packets,
uint32_t expected_payload_size,
- uint32_t expected_total_payload_size) {
+ size_t expected_total_payload_size) {
assert(sender);
Packets packets;
sender->RunFor(run_for_ms, &packets);
@@ -777,7 +777,7 @@
ASSERT_TRUE(IsSequenceNumberSorted(packets));
EXPECT_EQ(expected_packets, packets.size());
int64_t send_time_us = -1;
- uint32_t total_payload_size = 0;
+ size_t total_payload_size = 0;
uint32_t absolute_send_time = 0;
uint32_t absolute_send_time_wraps = 0;
uint32_t rtp_timestamp = 0;
diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi
index c0b3c03..08f3ae8 100644
--- a/webrtc/webrtc_tests.gypi
+++ b/webrtc/webrtc_tests.gypi
@@ -178,6 +178,8 @@
'type': '<(gtest_target_type)',
'sources': [
'modules/audio_coding/neteq/test/neteq_performance_unittest.cc',
+ 'modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc',
+
'tools/agc/agc_manager_integrationtest.cc',
'video/call_perf_tests.cc',
'video/full_stack.cc',
@@ -190,7 +192,8 @@
'<(webrtc_root)/modules/modules.gyp:video_capture',
'<(webrtc_root)/test/test.gyp:channel_transport',
'<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine',
- 'modules/modules.gyp:neteq_test_support', # Needed by neteq_performance_unittest.
+ 'modules/modules.gyp:neteq_test_support',
+ 'modules/modules.gyp:bwe_simulator',
'modules/modules.gyp:rtp_rtcp',
'test/test.gyp:test_main',
'test/webrtc_test_common.gyp:webrtc_test_common',
