Add Nearby Share stress test
1) Add Nearby Share stress test
2) Refactor existing codes by removing the prefix of first_/sencond_ as much as possible to avoid the confusion to Nearby Share and eSIM transfer test which only needs one connection.
3) Rename esim_transfer_only_stress_test to esim_trasfer_stress_test.
Bug: 308484461
Test: manual
Change-Id: Ie68238a3588e990f05da83d6f92b837fef209265
diff --git a/tests/bettertogether/quickstart/performance_test/Android.bp b/tests/bettertogether/quickstart/performance_test/Android.bp
index f68c58c..cd9fb8c 100644
--- a/tests/bettertogether/quickstart/performance_test/Android.bp
+++ b/tests/bettertogether/quickstart/performance_test/Android.bp
@@ -98,9 +98,9 @@
}
python_test_host {
- name: "esim_transfer_only_stress_test",
- main: "esim_transfer_only_stress_test.py",
- srcs: ["esim_transfer_only_stress_test.py",],
+ name: "esim_transfer_stress_test",
+ main: "esim_transfer_stress_test.py",
+ srcs: ["esim_transfer_stress_test.py",],
libs: [
"nc_constants",
"setup_utils",
@@ -123,3 +123,30 @@
},
},
}
+
+python_test_host {
+ name: "nearby_share_stress_test",
+ main: "nearby_share_stress_test.py",
+ srcs: ["nearby_share_stress_test.py",],
+ libs: [
+ "nc_constants",
+ "setup_utils",
+ "nearby_connection_wrapper",
+ "nc_base_test",
+ "mobly",],
+ test_suites: [],
+ test_options: {
+ unit_test: false, // as Mobly tests require device(s)
+ // This tag is used to enable the ATest Mobly runner
+ tags: ["mobly"],
+ },
+ data: [
+ "local_nearby_share_testbed.yml",
+ // package the snippes for atest
+ ":nearby_snippet",],
+ version: {
+ py3: {
+ embedded_launcher: false,
+ },
+ },
+}
diff --git a/tests/bettertogether/quickstart/performance_test/esim_transfer_only_stress_test.py b/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py
similarity index 84%
rename from tests/bettertogether/quickstart/performance_test/esim_transfer_only_stress_test.py
rename to tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py
index c691c50..aab7192 100644
--- a/tests/bettertogether/quickstart/performance_test/esim_transfer_only_stress_test.py
+++ b/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py
@@ -1,3 +1,17 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
"""Esim transfer stress test, only use Bluetooth connection."""
import dataclasses
@@ -36,7 +50,7 @@
_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR = 10
-class EsimTransferOnlyStressTest(nc_base_test.NCBaseTestClass):
+class EsimTransferStressTest(nc_base_test.NCBaseTestClass):
"""Esim transfer stress test."""
@dataclasses.dataclass(frozen=False)
@@ -76,6 +90,10 @@
time.sleep(_DELAY_BETWEEN_EACH_TEST_CYCLE.total_seconds())
def _mimic_esim_transfer(self):
+ if self.test_parameters.toggle_airplane_mode_target_side:
+ setup_utils.toggle_airplane_mode(self.advertiser)
+ if self.test_parameters.reset_wifi_connection:
+ self._reset_wifi_connection()
self._test_result = nc_constants.SingleTestResult()
# 1. connect to wifi
wifi_ssid = self.test_parameters.wifi_ssid
@@ -110,23 +128,23 @@
connection_medium=nc_constants.NearbyMedium.BT_ONLY,
upgrade_medium=nc_constants.NearbyMedium.BT_ONLY,
)
- first_connection_setup_timeouts = nc_constants.ConnectionSetupTimeouts(
+ connection_setup_timeouts = nc_constants.ConnectionSetupTimeouts(
nc_constants.FIRST_DISCOVERY_TIMEOUT,
nc_constants.FIRST_CONNECTION_INIT_TIMEOUT,
nc_constants.FIRST_CONNECTION_RESULT_TIMEOUT)
try:
nearby_snippet_1.start_nearby_connection(
- timeouts=first_connection_setup_timeouts,
+ timeouts=connection_setup_timeouts,
medium_upgrade_type=nc_constants.MediumUpgradeType.NON_DISRUPTIVE)
finally:
- self._test_result.first_connection_setup_quality_info = (
+ self._test_result.connection_setup_quality_info = (
nearby_snippet_1.connection_quality_info
)
# 3. transfer file through bluetooth
file_1_mb = _TRANSFER_FILE_SIZE_1MB
- self._test_result.first_bt_transfer_throughput_kbps = (
+ self._test_result.bt_transfer_throughput_kbps = (
nearby_snippet_1.transfer_file(
file_1_mb, nc_constants.FILE_1M_PAYLOAD_TRANSFER_TIMEOUT,
nc_constants.PayloadType.FILE))
@@ -138,8 +156,8 @@
quality_info = {
'bt connection': (
- self._test_result.first_connection_setup_quality_info.get_dict()),
- 'bt_kBps': self._test_result.first_bt_transfer_throughput_kbps,
+ self._test_result.connection_setup_quality_info.get_dict()),
+ 'bt_kBps': self._test_result.bt_transfer_throughput_kbps,
}
test_report = {'quality_info': quality_info}
@@ -153,13 +171,13 @@
def _collect_current_test_metrics(self):
"""Collects test result metrics for each iteration."""
self._esim_transfer_test_metrics.discovery_latencies.append(
- self._test_result.first_connection_setup_quality_info.discovery_latency
+ self._test_result.connection_setup_quality_info.discovery_latency
)
self._esim_transfer_test_metrics.connection_latencies.append(
- self._test_result.first_connection_setup_quality_info.connection_latency
+ self._test_result.connection_setup_quality_info.connection_latency
)
self._esim_transfer_test_metrics.bt_transfer_throughputs_kbps.append(
- self._test_result.first_bt_transfer_throughput_kbps
+ self._test_result.bt_transfer_throughput_kbps
)
# @typing.override
diff --git a/tests/bettertogether/quickstart/performance_test/local_dev_testbed.yml b/tests/bettertogether/quickstart/performance_test/local_dev_testbed.yml
index 9bc1525..c6ea27a 100644
--- a/tests/bettertogether/quickstart/performance_test/local_dev_testbed.yml
+++ b/tests/bettertogether/quickstart/performance_test/local_dev_testbed.yml
@@ -2,6 +2,8 @@
wifi_country_code: "US"
wifi_ssid: ""
wifi_password: ""
+ upgrade_medium: 7 # WIFIAWARE_ONLY = 4, UPGRADE_TO_ALL_WIFI = 9, MEDIUM_UPGRADE_TO_WIFIDIRECT= 7
+ allow_unrooted_device: True
x-controllers: &controllers
Controllers:
diff --git a/tests/bettertogether/quickstart/performance_test/local_esim_testbed.yml b/tests/bettertogether/quickstart/performance_test/local_esim_testbed.yml
index 8f5fbfd..c47e50a 100644
--- a/tests/bettertogether/quickstart/performance_test/local_esim_testbed.yml
+++ b/tests/bettertogether/quickstart/performance_test/local_esim_testbed.yml
@@ -1,6 +1,7 @@
x-test-params: &test-params
wifi_ssid: ""
wifi_password: ""
+ advertising_discovery_medium: 2 # 0 - auto, 2 - BLE_ONLY
x-controllers: &controllers
Controllers:
@@ -23,7 +24,6 @@
test_report_alias_name: "Wi-Fi_2G"
wifi_ssid: "AP-2G"
wifi_password: "AP-2G"
- advertising_discovery_medium: 2 # 0 - auto, 2 - BLE_ONLY
bt_transfer_throughput_median_benchmark_kbps: 20
- Name: Local5GTestbed
@@ -33,5 +33,4 @@
test_report_alias_name: "Wi-Fi_5G"
wifi_ssid: "AP-5G"
wifi_password: "AP-5G"
- advertising_discovery_medium: 2 # 0 - auto, 2 - BLE_ONLY
bt_transfer_throughput_median_benchmark_kbps: 40
diff --git a/tests/bettertogether/quickstart/performance_test/local_nearby_share_testbed.yml b/tests/bettertogether/quickstart/performance_test/local_nearby_share_testbed.yml
new file mode 100644
index 0000000..c6ea27a
--- /dev/null
+++ b/tests/bettertogether/quickstart/performance_test/local_nearby_share_testbed.yml
@@ -0,0 +1,62 @@
+x-test-params: &test-params
+ wifi_country_code: "US"
+ wifi_ssid: ""
+ wifi_password: ""
+ upgrade_medium: 7 # WIFIAWARE_ONLY = 4, UPGRADE_TO_ALL_WIFI = 9, MEDIUM_UPGRADE_TO_WIFIDIRECT= 7
+ allow_unrooted_device: True
+
+x-controllers: &controllers
+ Controllers:
+ AndroidDevice:
+ - serial: "123456ABCDEF"
+ role: "source_device"
+ - serial: "ABCDEF123456"
+ role: "target_device"
+
+TestBeds:
+- Name: LocalCustomizedTestbed
+ <<: *controllers
+ TestParams:
+ <<: *test-params
+
+- Name: LocalUS2GTestbed
+ <<: *controllers
+ TestParams:
+ <<: *test-params
+ test_report_alias_name: "US-2G"
+ wifi_ssid: "AP-2437"
+ wifi_password: "AP-2437"
+ bt_transfer_throughput_median_benchmark_kbps: 20
+ wifi_transfer_throughput_median_benchmark_kbps: 10240
+
+- Name: LocalUS5GTestbed
+ <<: *controllers
+ TestParams:
+ <<: *test-params
+ test_report_alias_name: "US-5G"
+ wifi_ssid: "AP-5180"
+ wifi_password: "AP-5180"
+ bt_transfer_throughput_median_benchmark_kbps: 40
+ wifi_transfer_throughput_median_benchmark_kbps: 20480
+
+- Name: LocalJP5GTestbed
+ <<: *controllers
+ TestParams:
+ <<: *test-params
+ test_report_alias_name: "JP-5G"
+ wifi_country_code: "JP"
+ wifi_ssid: "AP-5180"
+ wifi_password: "AP-5180"
+ bt_transfer_throughput_median_benchmark_kbps: 40
+ wifi_transfer_throughput_median_benchmark_kbps: 10240
+
+- Name: LocalGB5GTestbed
+ <<: *controllers
+ TestParams:
+ <<: *test-params
+ test_report_alias_name: "GB-5G"
+ wifi_country_code: "GB"
+ wifi_ssid: "AP-5260"
+ wifi_password: "AP-5260"
+ bt_transfer_throughput_median_benchmark_kbps: 40
+ wifi_transfer_throughput_median_benchmark_kbps: 10240
diff --git a/tests/bettertogether/quickstart/performance_test/nc_base_test.py b/tests/bettertogether/quickstart/performance_test/nc_base_test.py
index a440c5d..81537d1 100644
--- a/tests/bettertogether/quickstart/performance_test/nc_base_test.py
+++ b/tests/bettertogether/quickstart/performance_test/nc_base_test.py
@@ -103,9 +103,6 @@
setup_utils.enable_auto_reconnect(ad)
def setup_test(self):
- if self.test_parameters.toggle_airplane_mode_target_side:
- setup_utils.toggle_airplane_mode(self.advertiser)
- self._reset_wifi_connection()
self._reset_nearby_connection()
def _reset_wifi_connection(self) -> None:
diff --git a/tests/bettertogether/quickstart/performance_test/nc_constants.py b/tests/bettertogether/quickstart/performance_test/nc_constants.py
index 1ac4265..0fa6efb 100644
--- a/tests/bettertogether/quickstart/performance_test/nc_constants.py
+++ b/tests/bettertogether/quickstart/performance_test/nc_constants.py
@@ -29,7 +29,7 @@
SECOND_CONNECTION_INIT_TIMEOUT = datetime.timedelta(seconds=10)
SECOND_CONNECTION_RESULT_TIMEOUT = datetime.timedelta(seconds=25)
CONNECTION_BANDWIDTH_CHANGED_TIMEOUT = datetime.timedelta(seconds=25)
-FILE_1G_PAYLOAD_TRANSFER_TIMEOUT = datetime.timedelta(seconds=400)
+FILE_1G_PAYLOAD_TRANSFER_TIMEOUT = datetime.timedelta(seconds=210)
WIFI_WLAN_CONNECTING_TIME_OUT = datetime.timedelta(seconds=25)
DISCONNECTION_TIMEOUT = datetime.timedelta(seconds=15)
@@ -86,6 +86,8 @@
wifi_ssid: str = ''
wifi_password: str = ''
toggle_airplane_mode_target_side: bool = True
+ reset_wifi_connection: bool = True
+ disconnect_bt_after_test: bool = False
disconnect_wifi_after_test: bool = False
bt_transfer_throughput_median_benchmark_kbps: float = (
BT_TRANSFER_THROUGHPUT_MEDIAN_BENCHMARK_KBPS
@@ -171,15 +173,15 @@
class SingleTestResult:
"""The test result of a single iteration."""
- first_connection_setup_quality_info: ConnectionSetupQualityInfo = (
+ connection_setup_quality_info: ConnectionSetupQualityInfo = (
dataclasses.field(default_factory=ConnectionSetupQualityInfo)
)
- first_bt_transfer_throughput_kbps: float = UNSET_THROUGHPUT_KBPS
+ bt_transfer_throughput_kbps: float = UNSET_THROUGHPUT_KBPS
discoverer_wifi_wlan_latency: datetime.timedelta = UNSET_LATENCY
second_connection_setup_quality_info: ConnectionSetupQualityInfo = (
dataclasses.field(default_factory=ConnectionSetupQualityInfo)
)
- second_wifi_transfer_throughput_kbps: float = UNSET_THROUGHPUT_KBPS
+ wifi_transfer_throughput_kbps: float = UNSET_THROUGHPUT_KBPS
advertiser_wifi_wlan_latency: datetime.timedelta = UNSET_LATENCY
discoverer_wifi_wlan_expected: bool = False
advertiser_wifi_wlan_expected: bool = False
@@ -229,7 +231,7 @@
)
second_connection_latencies: list[datetime.timedelta] = dataclasses.field(
default_factory=list[datetime.timedelta])
- second_medium_upgrade_latencies: list[
+ medium_upgrade_latencies: list[
datetime.timedelta] = dataclasses.field(
default_factory=list[datetime.timedelta])
advertiser_wifi_wlan_latencies: list[
diff --git a/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py b/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py
new file mode 100644
index 0000000..20e0819
--- /dev/null
+++ b/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py
@@ -0,0 +1,267 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Stress tests for Neaby Share/Nearby Connection flow."""
+
+import dataclasses
+import datetime
+import logging
+import os
+import sys
+import time
+
+# check the python version
+if sys.version_info < (3,10):
+ logging.error('The test only can run on python 3.10 and above')
+ exit()
+
+from mobly import asserts
+from mobly import base_test
+from mobly import test_runner
+
+# Allows local imports to be resolved via relative path, so the test can be run
+# without building.
+_performance_test_dir = os.path.dirname(os.path.dirname(__file__))
+if _performance_test_dir not in sys.path:
+ sys.path.append(_performance_test_dir)
+
+from performance_test import nc_base_test
+from performance_test import nc_constants
+from performance_test import nearby_connection_wrapper
+from performance_test import setup_utils
+
+_TEST_SCRIPT_VERSTION = '1.5'
+
+_DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5)
+_TRANSFER_FILE_SIZE_1GB = 1024 * 1024
+
+_PERFORMANCE_TEST_REPEAT_COUNT = 100
+_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR = 10
+
+
+class NearbyShareStressTest(nc_base_test.NCBaseTestClass):
+ """Nearby Share stress test."""
+
+ @dataclasses.dataclass(frozen=False)
+ class NearbyShareTestMetrics:
+ """Metrics data for Nearby Share test."""
+
+ discovery_latencies: list[datetime.timedelta] = dataclasses.field(
+ default_factory=list[datetime.timedelta]
+ )
+ connection_latencies: list[datetime.timedelta] = dataclasses.field(
+ default_factory=list[datetime.timedelta]
+ )
+ medium_upgrade_latencies: list[datetime.timedelta] = dataclasses.field(
+ default_factory=list[datetime.timedelta]
+ )
+ wifi_transfer_throughputs_kbps: list[float] = dataclasses.field(
+ default_factory=list[float]
+ )
+
+ # @typing.override
+ def __init__(self, configs):
+ super().__init__(configs)
+ self._test_result = nc_constants.SingleTestResult()
+ self._nearby_share_test_metrics = self.NearbyShareTestMetrics()
+
+ # @typing.override
+ def setup_class(self):
+ super().setup_class()
+ wifi_ssid = self.test_parameters.wifi_ssid
+ wifi_password = self.test_parameters.wifi_password
+ if wifi_ssid:
+ discoverer_wifi_latency = setup_utils.connect_to_wifi_wlan_till_success(
+ self.discoverer, wifi_ssid, wifi_password
+ )
+ self.discoverer.log.info(
+ 'connecting to wifi in '
+ f'{round(discoverer_wifi_latency.total_seconds())} s'
+ )
+ advertiser_wlan_latency = setup_utils.connect_to_wifi_wlan_till_success(
+ self.advertiser, wifi_ssid, wifi_password)
+ self.advertiser.log.info(
+ 'connecting to wifi in '
+ f'{round(advertiser_wlan_latency.total_seconds())} s')
+ self.advertiser.log.info(
+ self.advertiser.nearby.wifiGetConnectionInfo().get('mFrequency')
+ )
+
+ self.performance_test_iterations = getattr(
+ self.test_nearby_share_performance, base_test.ATTR_REPEAT_CNT)
+ logging.info('performance test iterations: %s',
+ self.performance_test_iterations)
+
+ @base_test.repeat(
+ count=_PERFORMANCE_TEST_REPEAT_COUNT,
+ max_consecutive_error=_PERFORMANCE_TEST_MAX_CONSECUTIVE_ERROR)
+ def test_nearby_share_performance(self):
+ """Nearby Share stress test, which only transfer data through WiFi."""
+ try:
+ self._mimic_nearby_share()
+ finally:
+ self._write_current_test_report()
+ self._collect_current_test_metrics()
+ time.sleep(_DELAY_BETWEEN_EACH_TEST_CYCLE.total_seconds())
+
+ def _mimic_nearby_share(self):
+ """Mimics Nearby Share stress test, which only transfer data through WiFi."""
+ self._test_result = nc_constants.SingleTestResult()
+
+ # 1. set up BT and WiFi connection
+ advertising_discovery_medium = nc_constants.NearbyMedium(
+ self.test_parameters.advertising_discovery_medium
+ )
+ nearby_snippet_1 = nearby_connection_wrapper.NearbyConnectionWrapper(
+ self.advertiser,
+ self.discoverer,
+ self.advertiser.nearby,
+ self.discoverer.nearby,
+ advertising_discovery_medium=advertising_discovery_medium,
+ connection_medium=nc_constants.NearbyMedium.BT_ONLY,
+ upgrade_medium=nc_constants.NearbyMedium(
+ self.test_parameters.upgrade_medium
+ ),
+ )
+ connection_setup_timeouts = nc_constants.ConnectionSetupTimeouts(
+ nc_constants.FIRST_DISCOVERY_TIMEOUT,
+ nc_constants.FIRST_CONNECTION_INIT_TIMEOUT,
+ nc_constants.FIRST_CONNECTION_RESULT_TIMEOUT)
+
+ try:
+ nearby_snippet_1.start_nearby_connection(
+ timeouts=connection_setup_timeouts,
+ medium_upgrade_type=nc_constants.MediumUpgradeType.NON_DISRUPTIVE)
+ finally:
+ self._test_result.connection_setup_quality_info = (
+ nearby_snippet_1.connection_quality_info
+ )
+ self._test_result.connection_setup_quality_info.medium_upgrade_expected = (
+ True
+ )
+
+ # 2. transfer file through WiFi
+ file_1_gb = _TRANSFER_FILE_SIZE_1GB
+ self._test_result.wifi_transfer_throughput_kbps = (
+ nearby_snippet_1.transfer_file(
+ file_1_gb, nc_constants.FILE_1G_PAYLOAD_TRANSFER_TIMEOUT,
+ nc_constants.PayloadType.FILE))
+ # 3. disconnect
+ nearby_snippet_1.disconnect_endpoint()
+
+ def _write_current_test_report(self):
+ """Writes test report for each iteration."""
+
+ quality_info = {
+ 'Latency (sec)': (
+ self._test_result.connection_setup_quality_info.get_dict()),
+ 'Speed (kByte/sec)': self._test_result.wifi_transfer_throughput_kbps,
+ }
+ test_report = {'quality_info': quality_info}
+
+ self.discoverer.log.info(test_report)
+ self.record_data({
+ 'Test Class': self.TAG,
+ 'Test Name': self.current_test_info.name,
+ 'sponge_properties': test_report,
+ })
+
+ def _collect_current_test_metrics(self):
+ """Collects test result metrics for each iteration."""
+ self._nearby_share_test_metrics.discovery_latencies.append(
+ self._test_result.connection_setup_quality_info.discovery_latency
+ )
+ self._nearby_share_test_metrics.connection_latencies.append(
+ self._test_result.connection_setup_quality_info.connection_latency
+ )
+ self._nearby_share_test_metrics.medium_upgrade_latencies.append(
+ self._test_result.connection_setup_quality_info.medium_upgrade_latency
+ )
+ self._nearby_share_test_metrics.wifi_transfer_throughputs_kbps.append(
+ self._test_result.wifi_transfer_throughput_kbps
+ )
+
+ # @typing.override
+ def _summary_test_results(self):
+ """Summarizes test results of all iterations."""
+ wifi_transfer_stats = self._stats_throughput_result(
+ 'WiFi',
+ self._nearby_share_test_metrics.wifi_transfer_throughputs_kbps,
+ nc_constants.BT_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE,
+ self.test_parameters.wifi_transfer_throughput_median_benchmark_kbps)
+
+ discovery_stats = self._stats_latency_result(
+ self._nearby_share_test_metrics.discovery_latencies)
+ connection_stats = self._stats_latency_result(
+ self._nearby_share_test_metrics.connection_latencies)
+ medium_upgrade_stats = self._stats_latency_result(
+ self._nearby_share_test_metrics.medium_upgrade_latencies
+ )
+
+ passed = True
+ result_message = 'Passed'
+ fail_message = ''
+ if wifi_transfer_stats.fail_targets:
+ fail_message += self._generate_target_fail_message(
+ wifi_transfer_stats.fail_targets)
+ if fail_message:
+ passed = False
+ result_message = 'Test Failed due to:\n' + fail_message
+
+ detailed_stats = {
+ '0 test iterations': self.performance_test_iterations,
+ '1 Completed WiFi transfer': f'{wifi_transfer_stats.success_count}',
+ '2 failure counts': {
+ 'discovery': discovery_stats.failure_count,
+ 'connection': connection_stats.failure_count,
+ 'upgrade': medium_upgrade_stats.failure_count,
+ 'transfer': self.performance_test_iterations - (
+ wifi_transfer_stats.success_count),
+ },
+ '3 50% and 95% of WiFi transfer speed (KBps)': (
+ f'{wifi_transfer_stats.percentile_50_kbps}'
+ f' / {wifi_transfer_stats.percentile_95_kbps}'),
+ '4 50% and 95% of discovery latency(sec)': (
+ f'{discovery_stats.percentile_50}'
+ f' / {discovery_stats.percentile_95}'),
+ '5 50% and 95% of connection latency(sec)': (
+ f'{connection_stats.percentile_50}'
+ f' / {connection_stats.percentile_95}'),
+ '6 50% and 95% of upgrade latency(sec)': (
+ f'{medium_upgrade_stats.percentile_50}'
+ f' / {medium_upgrade_stats.percentile_95}'),
+ }
+
+ self.record_data({
+ 'Test Class': self.TAG,
+ 'sponge_properties': {
+ 'test_script_verion': _TEST_SCRIPT_VERSTION,
+ '00_test_report_alias_name': (
+ self.test_parameters.test_report_alias_name),
+ '01_test_result': result_message,
+ '02_source_device_serial': self.discoverer.serial,
+ '03_target_device_serial': self.advertiser.serial,
+ '04_source_GMS_version': setup_utils.dump_gms_version(
+ self.discoverer),
+ '05_target_GMS_version': setup_utils.dump_gms_version(
+ self.advertiser),
+ '06_detailed_stats': detailed_stats
+ }
+ })
+
+ asserts.assert_true(passed, result_message)
+
+
+if __name__ == '__main__':
+ test_runner.main()
diff --git a/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py b/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py
index ebe0fdc..7b81d6a 100644
--- a/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py
+++ b/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py
@@ -145,6 +145,10 @@
wifi_password: str = '',
) -> None:
"""Mimics quick start flow test with 2 nearby connections."""
+ if self.test_parameters.toggle_airplane_mode_target_side:
+ setup_utils.toggle_airplane_mode(self.advertiser)
+ if self.test_parameters.reset_wifi_connection:
+ self._reset_wifi_connection()
# 1. discoverer connect to wifi wlan
self._test_result = nc_constants.SingleTestResult()
if wifi_ssid:
@@ -181,13 +185,13 @@
timeouts=first_connection_setup_timeouts,
medium_upgrade_type=nc_constants.MediumUpgradeType.NON_DISRUPTIVE)
finally:
- self._test_result.first_connection_setup_quality_info = (
+ self._test_result.connection_setup_quality_info = (
nearby_snippet_1.connection_quality_info
)
# 3. transfer file through bluetooth
file_1_mb = _TRANSFER_FILE_SIZE_1MB
- self._test_result.first_bt_transfer_throughput_kbps = (
+ self._test_result.bt_transfer_throughput_kbps = (
nearby_snippet_1.transfer_file(
file_1_mb, nc_constants.FILE_1M_PAYLOAD_TRANSFER_TIMEOUT,
nc_constants.PayloadType.FILE))
@@ -238,7 +242,7 @@
# 6. transfer file through wifi
file_1_gb = _TRANSFER_FILE_SIZE_1GB
- self._test_result.second_wifi_transfer_throughput_kbps = (
+ self._test_result.wifi_transfer_throughput_kbps = (
nearby_snippet_2.transfer_file(
file_1_gb, nc_constants.FILE_1G_PAYLOAD_TRANSFER_TIMEOUT,
self.test_parameters.payload_type))
@@ -253,11 +257,11 @@
quality_info = {
'1st connection': (
- self._test_result.first_connection_setup_quality_info.get_dict()),
- 'bt_kBps': self._test_result.first_bt_transfer_throughput_kbps,
+ self._test_result.connection_setup_quality_info.get_dict()),
+ 'bt_kBps': self._test_result.bt_transfer_throughput_kbps,
'2nd connection': (
self._test_result.second_connection_setup_quality_info.get_dict()),
- 'wifi_kBps': self._test_result.second_wifi_transfer_throughput_kbps,
+ 'wifi_kBps': self._test_result.wifi_transfer_throughput_kbps,
}
if self._test_result.discoverer_wifi_wlan_expected:
@@ -280,13 +284,13 @@
def _collect_current_test_metrics(self) -> None:
"""Collects test result metrics for each iteration."""
self._quick_start_test_metrics.first_discovery_latencies.append(
- self._test_result.first_connection_setup_quality_info.discovery_latency
+ self._test_result.connection_setup_quality_info.discovery_latency
)
self._quick_start_test_metrics.first_connection_latencies.append(
- self._test_result.first_connection_setup_quality_info.connection_latency
+ self._test_result.connection_setup_quality_info.connection_latency
)
self._quick_start_test_metrics.bt_transfer_throughputs_kbps.append(
- self._test_result.first_bt_transfer_throughput_kbps
+ self._test_result.bt_transfer_throughput_kbps
)
self._quick_start_test_metrics.second_discovery_latencies.append(
@@ -295,13 +299,13 @@
self._quick_start_test_metrics.second_connection_latencies.append(
self._test_result.second_connection_setup_quality_info.connection_latency
)
- self._quick_start_test_metrics.second_medium_upgrade_latencies.append(
+ self._quick_start_test_metrics.medium_upgrade_latencies.append(
self._test_result.second_connection_setup_quality_info.medium_upgrade_latency
)
self._quick_start_test_metrics.upgraded_wifi_transfer_mediums.append(
self._test_result.second_connection_setup_quality_info.upgrade_medium)
self._quick_start_test_metrics.wifi_transfer_throughputs_kbps.append(
- self._test_result.second_wifi_transfer_throughput_kbps
+ self._test_result.wifi_transfer_throughput_kbps
)
self._quick_start_test_metrics.discoverer_wifi_wlan_latencies.append(
self._test_result.discoverer_wifi_wlan_latency)
@@ -320,13 +324,13 @@
# @typing.override
def _summary_test_results(self) -> None:
"""Summarizes test results of all iterations."""
- first_bt_transfer_stats = self._stats_throughput_result(
+ bt_transfer_stats = self._stats_throughput_result(
'BT',
self._quick_start_test_metrics.bt_transfer_throughputs_kbps,
nc_constants.BT_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE,
self.test_parameters.bt_transfer_throughput_median_benchmark_kbps)
- second_wifi_transfer_stats = self._stats_throughput_result(
+ wifi_transfer_stats = self._stats_throughput_result(
'Wi-Fi',
self._quick_start_test_metrics.wifi_transfer_throughputs_kbps,
nc_constants.WIFI_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE,
@@ -340,18 +344,18 @@
self._quick_start_test_metrics.second_discovery_latencies)
second_connection_stats = self._stats_latency_result(
self._quick_start_test_metrics.second_connection_latencies)
- second_medium_upgrade_stats = self._stats_latency_result(
- self._quick_start_test_metrics.second_medium_upgrade_latencies)
+ medium_upgrade_stats = self._stats_latency_result(
+ self._quick_start_test_metrics.medium_upgrade_latencies)
passed = True
result_message = 'Passed'
fail_message = ''
- if first_bt_transfer_stats.fail_targets:
+ if bt_transfer_stats.fail_targets:
fail_message += self._generate_target_fail_message(
- first_bt_transfer_stats.fail_targets)
- if second_wifi_transfer_stats.fail_targets:
+ bt_transfer_stats.fail_targets)
+ if wifi_transfer_stats.fail_targets:
fail_message += self._generate_target_fail_message(
- second_wifi_transfer_stats.fail_targets)
+ wifi_transfer_stats.fail_targets)
if fail_message:
passed = False
result_message = 'Test Failed due to:\n' + fail_message
@@ -359,29 +363,29 @@
detailed_stats = {
'0 test iterations': self.performance_test_iterations,
'1 Completed BT/Wi-Fi transfer': (
- f'{first_bt_transfer_stats.success_count}'
- f' / {second_wifi_transfer_stats.success_count}'),
+ f'{bt_transfer_stats.success_count}'
+ f' / {wifi_transfer_stats.success_count}'),
'2 BT transfer failures': {
'discovery': first_discovery_stats.failure_count,
'connection': first_connection_stats.failure_count,
'transfer': self.performance_test_iterations - (
- first_bt_transfer_stats.success_count),
+ bt_transfer_stats.success_count),
},
'3 Wi-Fi transfer failures': {
'discovery': second_discovery_stats.failure_count,
'connection': second_connection_stats.failure_count,
- 'upgrade': second_medium_upgrade_stats.failure_count,
+ 'upgrade': medium_upgrade_stats.failure_count,
'transfer': self.performance_test_iterations - (
- second_wifi_transfer_stats.success_count),
+ wifi_transfer_stats.success_count),
},
'4 Medium upgrade count': (
self._summary_upgraded_wifi_transfer_mediums()),
'5 50% and 95% of BT transfer speed (KBps)': (
- f'{first_bt_transfer_stats.percentile_50_kbps}'
- f' / {first_bt_transfer_stats.percentile_95_kbps}'),
+ f'{bt_transfer_stats.percentile_50_kbps}'
+ f' / {bt_transfer_stats.percentile_95_kbps}'),
'6 50% and 95% of Wi-Fi transfer speed(KBps)': (
- f'{second_wifi_transfer_stats.percentile_50_kbps}'
- f' / {second_wifi_transfer_stats.percentile_95_kbps}'),
+ f'{wifi_transfer_stats.percentile_50_kbps}'
+ f' / {wifi_transfer_stats.percentile_95_kbps}'),
'7 50% and 95% of discovery latency(sec)': (
f'{first_discovery_stats.percentile_50}'
f' / {first_discovery_stats.percentile_95} (1st), '
@@ -393,8 +397,8 @@
f'{second_connection_stats.percentile_50}'
f' / {second_connection_stats.percentile_95} (2nd)'),
'9 50% and 95% of medium upgrade latency(sec)': (
- f'{second_medium_upgrade_stats.percentile_50}'
- f' / {second_medium_upgrade_stats.percentile_95} (2nd)'),
+ f'{medium_upgrade_stats.percentile_50}'
+ f' / {medium_upgrade_stats.percentile_95}'),
}
self.record_data({