cheets_CTS_R: More retry for CtsWindowManagerDeviceTestCases
The tests are flakier than others. To ensure stably
generating passing results, we enlarge the max retry
count as a temporary solution.
BUG=b:181731092
TEST=cheets_CTS_R.11_r3.arm.CtsWindowManager
Change-Id: I2d841ae2a6162b72b76791932482c5cb3024a5f7
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2730380
Tested-by: Kazuhiro Inaba <kinaba@chromium.org>
Auto-Submit: Kazuhiro Inaba <kinaba@chromium.org>
Commit-Queue: Kazuhiro Inaba <kinaba@chromium.org>
Commit-Queue: Jiyoun Ha <jiyounha@chromium.org>
Reviewed-by: Jiyoun Ha <jiyounha@chromium.org>
diff --git a/server/cros/tradefed/generate_controlfiles_CTS_R.py b/server/cros/tradefed/generate_controlfiles_CTS_R.py
index 34f1b92..fa92a6a 100755
--- a/server/cros/tradefed/generate_controlfiles_CTS_R.py
+++ b/server/cros/tradefed/generate_controlfiles_CTS_R.py
@@ -50,7 +50,10 @@
CONFIG['CTS_JOB_RETRIES_IN_PUBLIC'] = 1
CONFIG['CTS_QUAL_RETRIES'] = 9
-CONFIG['CTS_MAX_RETRIES'] = {}
+CONFIG['CTS_MAX_RETRIES'] = {
+ # TODO(b/181543065): Remove once the flakiness is fixed.
+ 'CtsWindowManagerDeviceTestCases': 10,
+}
# Timeout in hours.
CONFIG['CTS_TIMEOUT_DEFAULT'] = 1.0
@@ -198,7 +201,10 @@
# This information is changed based on regular analysis of the failure rate on
# partner moblabs.
-CONFIG['PUBLIC_MODULE_RETRY_COUNT'] = {}
+CONFIG['PUBLIC_MODULE_RETRY_COUNT'] = {
+ # TODO(b/181543065): Remove once the flakiness is fixed.
+ 'CtsWindowManagerDeviceTestCases': 10,
+}
# This information is changed based on regular analysis of the job run time on
# partner moblabs.
diff --git a/server/site_tests/cheets_CTS_R/control.11_r3.arm.CtsWindowManager b/server/site_tests/cheets_CTS_R/control.11_r3.arm.CtsWindowManager
index 78df94b..3628301 100644
--- a/server/site_tests/cheets_CTS_R/control.11_r3.arm.CtsWindowManager
+++ b/server/site_tests/cheets_CTS_R/control.11_r3.arm.CtsWindowManager
@@ -20,6 +20,7 @@
'cheets_CTS_R',
hosts=host_list,
iterations=1,
+ max_retry=10,
tag='11_r3.arm.CtsWindowManager',
test_name='cheets_CTS_R.11_r3.arm.CtsWindowManager',
run_template=['run', 'commandAndExit', 'cts', '--include-filter', 'CtsWindowManagerDeviceTestCases', '--include-filter', 'CtsWindowManagerDeviceTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerJetpackTestCases', '--include-filter', 'CtsWindowManagerJetpackTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk25TestCases', '--include-filter', 'CtsWindowManagerSdk25TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk28TestCases', '--include-filter', 'CtsWindowManagerSdk28TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk29TestCases', '--include-filter', 'CtsWindowManagerSdk29TestCases[secondary_user]', '--logcat-on-failure'],
diff --git a/server/site_tests/cheets_CTS_R/control.11_r3.arm.all.CtsVoiceInteractionTestCases_-_vm-tests-tf b/server/site_tests/cheets_CTS_R/control.11_r3.arm.all.CtsVoiceInteractionTestCases_-_vm-tests-tf
index 42f2e34..c18bbbf 100644
--- a/server/site_tests/cheets_CTS_R/control.11_r3.arm.all.CtsVoiceInteractionTestCases_-_vm-tests-tf
+++ b/server/site_tests/cheets_CTS_R/control.11_r3.arm.all.CtsVoiceInteractionTestCases_-_vm-tests-tf
@@ -20,7 +20,7 @@
'cheets_CTS_R',
hosts=host_list,
iterations=1,
- max_retry=9,
+ max_retry=10,
tag='11_r3.arm.all.CtsVoiceInteractionTestCases_-_vm-tests-tf',
test_name='cheets_CTS_R.11_r3.arm.all.CtsVoiceInteractionTestCases_-_vm-tests-tf',
run_template=['run', 'commandAndExit', 'cts', '--include-filter', 'CtsVoiceInteractionTestCases', '--include-filter', 'CtsVoiceInteractionTestCases[instant]', '--include-filter', 'CtsVoiceInteractionTestCases[secondary_user]', '--include-filter', 'CtsVoiceSettingsTestCases', '--include-filter', 'CtsVoiceSettingsTestCases[instant]', '--include-filter', 'CtsVoiceSettingsTestCases[secondary_user]', '--include-filter', 'CtsVrTestCases', '--include-filter', 'CtsVrTestCases[secondary_user]', '--include-filter', 'CtsWebkitTestCases', '--include-filter', 'CtsWebkitTestCases[instant]', '--include-filter', 'CtsWebkitTestCases[secondary_user]', '--include-filter', 'CtsWidgetTestCases', '--include-filter', 'CtsWidgetTestCases29', '--include-filter', 'CtsWidgetTestCases29[instant]', '--include-filter', 'CtsWidgetTestCases29[secondary_user]', '--include-filter', 'CtsWidgetTestCases[instant]', '--include-filter', 'CtsWidgetTestCases[secondary_user]', '--include-filter', 'CtsWifiBroadcastsHostTestCases', '--include-filter', 'CtsWifiBroadcastsHostTestCases[instant]', '--include-filter', 'CtsWifiBroadcastsHostTestCases[secondary_user]', '--include-filter', 'CtsWifiTestCases', '--include-filter', 'CtsWifiTestCases[instant]', '--include-filter', 'CtsWifiTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerDeviceTestCases', '--include-filter', 'CtsWindowManagerDeviceTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerJetpackTestCases', '--include-filter', 'CtsWindowManagerJetpackTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk25TestCases', '--include-filter', 'CtsWindowManagerSdk25TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk28TestCases', '--include-filter', 'CtsWindowManagerSdk28TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk29TestCases', '--include-filter', 'CtsWindowManagerSdk29TestCases[secondary_user]', '--include-filter', 'CtsWrapNoWrapTestCases', '--include-filter', 'CtsWrapNoWrapTestCases[secondary_user]', '--include-filter', 'CtsWrapWrapDebugMallocDebugTestCases', '--include-filter', 'CtsWrapWrapDebugMallocDebugTestCases[secondary_user]', '--include-filter', 'CtsWrapWrapDebugTestCases', '--include-filter', 'CtsWrapWrapDebugTestCases[secondary_user]', '--include-filter', 'CtsWrapWrapNoDebugTestCases', '--include-filter', 'CtsWrapWrapNoDebugTestCases[secondary_user]', '--include-filter', 'LegacyStorageTest', '--include-filter', 'LegacyStorageTest[instant]', '--include-filter', 'ScopedStorageTest', '--include-filter', 'ScopedStorageTest[instant]', '--include-filter', 'signed-CtsOmapiTestCases', '--include-filter', 'signed-CtsOmapiTestCases[secondary_user]', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases1', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases1[secondary_user]', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases2', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases2[secondary_user]', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases3', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases3[secondary_user]', '--include-filter', 'vm-tests-tf', '--include-filter', 'vm-tests-tf[secondary_user]', '--logcat-on-failure'],
diff --git a/server/site_tests/cheets_CTS_R/control.11_r3.x86.CtsWindowManager b/server/site_tests/cheets_CTS_R/control.11_r3.x86.CtsWindowManager
index f5bdd41..bef8149 100644
--- a/server/site_tests/cheets_CTS_R/control.11_r3.x86.CtsWindowManager
+++ b/server/site_tests/cheets_CTS_R/control.11_r3.x86.CtsWindowManager
@@ -20,6 +20,7 @@
'cheets_CTS_R',
hosts=host_list,
iterations=1,
+ max_retry=10,
tag='11_r3.x86.CtsWindowManager',
test_name='cheets_CTS_R.11_r3.x86.CtsWindowManager',
run_template=['run', 'commandAndExit', 'cts', '--include-filter', 'CtsWindowManagerDeviceTestCases', '--include-filter', 'CtsWindowManagerDeviceTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerJetpackTestCases', '--include-filter', 'CtsWindowManagerJetpackTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk25TestCases', '--include-filter', 'CtsWindowManagerSdk25TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk28TestCases', '--include-filter', 'CtsWindowManagerSdk28TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk29TestCases', '--include-filter', 'CtsWindowManagerSdk29TestCases[secondary_user]', '--logcat-on-failure'],
diff --git a/server/site_tests/cheets_CTS_R/control.11_r3.x86.all.CtsVoiceInteractionTestCases_-_vm-tests-tf b/server/site_tests/cheets_CTS_R/control.11_r3.x86.all.CtsVoiceInteractionTestCases_-_vm-tests-tf
index b0e48a5..421bc5e 100644
--- a/server/site_tests/cheets_CTS_R/control.11_r3.x86.all.CtsVoiceInteractionTestCases_-_vm-tests-tf
+++ b/server/site_tests/cheets_CTS_R/control.11_r3.x86.all.CtsVoiceInteractionTestCases_-_vm-tests-tf
@@ -20,7 +20,7 @@
'cheets_CTS_R',
hosts=host_list,
iterations=1,
- max_retry=9,
+ max_retry=10,
tag='11_r3.x86.all.CtsVoiceInteractionTestCases_-_vm-tests-tf',
test_name='cheets_CTS_R.11_r3.x86.all.CtsVoiceInteractionTestCases_-_vm-tests-tf',
run_template=['run', 'commandAndExit', 'cts', '--include-filter', 'CtsVoiceInteractionTestCases', '--include-filter', 'CtsVoiceInteractionTestCases[instant]', '--include-filter', 'CtsVoiceInteractionTestCases[secondary_user]', '--include-filter', 'CtsVoiceSettingsTestCases', '--include-filter', 'CtsVoiceSettingsTestCases[instant]', '--include-filter', 'CtsVoiceSettingsTestCases[secondary_user]', '--include-filter', 'CtsVrTestCases', '--include-filter', 'CtsVrTestCases[secondary_user]', '--include-filter', 'CtsWebkitTestCases', '--include-filter', 'CtsWebkitTestCases[instant]', '--include-filter', 'CtsWebkitTestCases[secondary_user]', '--include-filter', 'CtsWidgetTestCases', '--include-filter', 'CtsWidgetTestCases29', '--include-filter', 'CtsWidgetTestCases29[instant]', '--include-filter', 'CtsWidgetTestCases29[secondary_user]', '--include-filter', 'CtsWidgetTestCases[instant]', '--include-filter', 'CtsWidgetTestCases[secondary_user]', '--include-filter', 'CtsWifiBroadcastsHostTestCases', '--include-filter', 'CtsWifiBroadcastsHostTestCases[instant]', '--include-filter', 'CtsWifiBroadcastsHostTestCases[secondary_user]', '--include-filter', 'CtsWifiTestCases', '--include-filter', 'CtsWifiTestCases[instant]', '--include-filter', 'CtsWifiTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerDeviceTestCases', '--include-filter', 'CtsWindowManagerDeviceTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerJetpackTestCases', '--include-filter', 'CtsWindowManagerJetpackTestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk25TestCases', '--include-filter', 'CtsWindowManagerSdk25TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk28TestCases', '--include-filter', 'CtsWindowManagerSdk28TestCases[secondary_user]', '--include-filter', 'CtsWindowManagerSdk29TestCases', '--include-filter', 'CtsWindowManagerSdk29TestCases[secondary_user]', '--include-filter', 'CtsWrapNoWrapTestCases', '--include-filter', 'CtsWrapNoWrapTestCases[secondary_user]', '--include-filter', 'CtsWrapWrapDebugMallocDebugTestCases', '--include-filter', 'CtsWrapWrapDebugMallocDebugTestCases[secondary_user]', '--include-filter', 'CtsWrapWrapDebugTestCases', '--include-filter', 'CtsWrapWrapDebugTestCases[secondary_user]', '--include-filter', 'CtsWrapWrapNoDebugTestCases', '--include-filter', 'CtsWrapWrapNoDebugTestCases[secondary_user]', '--include-filter', 'LegacyStorageTest', '--include-filter', 'LegacyStorageTest[instant]', '--include-filter', 'ScopedStorageTest', '--include-filter', 'ScopedStorageTest[instant]', '--include-filter', 'signed-CtsOmapiTestCases', '--include-filter', 'signed-CtsOmapiTestCases[secondary_user]', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases1', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases1[secondary_user]', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases2', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases2[secondary_user]', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases3', '--include-filter', 'signed-CtsSecureElementAccessControlTestCases3[secondary_user]', '--include-filter', 'vm-tests-tf', '--include-filter', 'vm-tests-tf[secondary_user]', '--logcat-on-failure'],
diff --git a/server/site_tests/cheets_CTS_R/control.arm.CtsWindowManagerDeviceTestCases b/server/site_tests/cheets_CTS_R/control.arm.CtsWindowManagerDeviceTestCases
index bf582f4..c455d61 100644
--- a/server/site_tests/cheets_CTS_R/control.arm.CtsWindowManagerDeviceTestCases
+++ b/server/site_tests/cheets_CTS_R/control.arm.CtsWindowManagerDeviceTestCases
@@ -20,6 +20,7 @@
'cheets_CTS_R',
hosts=host_list,
iterations=1,
+ max_retry=10,
tag='arm.CtsWindowManagerDeviceTestCases',
test_name='cheets_CTS_R.arm.CtsWindowManagerDeviceTestCases',
run_template=['run', 'commandAndExit', 'cts', '--module', 'CtsWindowManagerDeviceTestCases'],
diff --git a/server/site_tests/cheets_CTS_R/control.x86.CtsWindowManagerDeviceTestCases b/server/site_tests/cheets_CTS_R/control.x86.CtsWindowManagerDeviceTestCases
index 785f5a8..b318dbd 100644
--- a/server/site_tests/cheets_CTS_R/control.x86.CtsWindowManagerDeviceTestCases
+++ b/server/site_tests/cheets_CTS_R/control.x86.CtsWindowManagerDeviceTestCases
@@ -20,6 +20,7 @@
'cheets_CTS_R',
hosts=host_list,
iterations=1,
+ max_retry=10,
tag='x86.CtsWindowManagerDeviceTestCases',
test_name='cheets_CTS_R.x86.CtsWindowManagerDeviceTestCases',
run_template=['run', 'commandAndExit', 'cts', '--module', 'CtsWindowManagerDeviceTestCases'],
