Test: Clean up CTS test plans
Disable some new tests by adding to knownfailures
Re-enable bluetooth tests.
Bug: 20224949
Bug: 23238984
Bug: 23604254
Bug: 23776083
Bug: 23776099
Bug: 23776893
Bug: 23779168
Bug: 23779020
Change-Id: I7012afc31aa9125472b3f0ceb36ce767314f5b04
diff --git a/tests/expectations/knownfailures.txt b/tests/expectations/knownfailures.txt
index 754e38a..29c5035 100644
--- a/tests/expectations/knownfailures.txt
+++ b/tests/expectations/knownfailures.txt
@@ -1,5 +1,14 @@
[
{
+ description: "some AlarmClockTests are not robust across different device types",
+ names: [
+ "android.alarmclock.cts.DismissAlarmTest#testAll",
+ "android.alarmclock.cts.SetAlarmTest#testAll",
+ "android.alarmclock.cts.SnoozeAlarmTest#testAll"
+ ],
+ bug: 23776083
+},
+{
description: "the UsageStats is not yet stable enough",
names: [
"android.app.usage.cts.UsageStatsTest"
@@ -15,6 +24,21 @@
bug: 17595050
},
{
+ description: "test fails on devices with no telephony",
+ names: [
+ "android.calllog.cts.CallLogBackupTest#testSingleCallBackup"
+ ],
+ bug: 23776099
+},
+{
+ description: "test fails on some devices",
+ names: [
+ "android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput",
+ "android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats"
+ ],
+ bug: 23776893
+},
+{
description: "the SSLCertificateSocketFactoryTest often fails because of lack of live internet or short timeout, it should be refactored to do a local server testing",
names: [
"android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket",
@@ -39,6 +63,21 @@
bug: 18461670
},
{
+ description: "test not robust",
+ names: [
+ "android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect",
+ "android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections"
+ ],
+ bug: 23604254
+},
+{
+ description: "tests too flaky",
+ names: [
+ "android.transition.cts.ChangeScrollTest#testChangeScroll"
+ ],
+ bug: 23779020
+},
+{
description: "Not all jdwp features are currently supported. These tests will fail",
names: [
"org.apache.harmony.jpda.tests.jdwp.DebuggerOnDemand.OnthrowDebuggerLaunchTest#testDebuggerLaunch001",
@@ -119,6 +158,13 @@
bug: 17508787
},
{
+ description: "Test is not yet properly implemented",
+ names: [
+ "android.voicesettings.cts.ZenModeTest#testAll"
+ ],
+ bug: 23238984
+},
+{
description: "These tests fail on some devices.",
names: [
"android.uirendering.cts.testclasses.ExactCanvasTests#testBlueRect",
@@ -234,5 +280,12 @@
"android.hardware.cts.SingleSensorTests#testOrientation_5hz"
],
bug: 22922206
+},
+{
+ description: "tests are not yet ready",
+ names: [
+ "com.android.cts.app.os.OsHostTests#testNonExportedActivities"
+ ],
+ bug: 23779168
}
]
diff --git a/tools/utils/buildCts.py b/tools/utils/buildCts.py
index 2b7e093..375b1a0 100755
--- a/tools/utils/buildCts.py
+++ b/tools/utils/buildCts.py
@@ -191,8 +191,6 @@
for package, test_list in small_tests.iteritems():
plan.Include(package+'$')
plan.Exclude(r'com\.android\.cts\.browserbench')
- for package, test_list in temporarily_known_failure_tests.iteritems():
- plan.ExcludeTests(package, test_list)
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
@@ -273,9 +271,6 @@
plan.Exclude(package+'$')
for package, tests_list in new_test_packages.iteritems():
plan.Exclude(package+'$')
- for package, test_list in temporarily_known_failure_tests.iteritems():
- plan.Include(package+'$')
- plan.IncludeTests(package, test_list)
plan.Exclude(r'com\.drawelements\.')
plan.Exclude(r'android\.hardware$')
plan.Exclude(r'android\.media$')
@@ -286,6 +281,15 @@
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
+ self.__WritePlan(plan, 'CTS-m-tests')
+
+
+ # CTS - sub plan for new test packages added for staging
+ plan = tools.TestPlan(packages)
+ plan.Exclude('.*')
+ for package, test_list in temporarily_known_failure_tests.iteritems():
+ plan.Include(package+'$')
+ plan.IncludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-staging')
plan = tools.TestPlan(packages)
@@ -461,11 +465,31 @@
""" Construct a defaultdict that maps package name to a list of tests
that are known failures during dev cycle but expected to be fixed before launch """
return {
- 'android.bluetooth' : [
- 'android.bluetooth.cts.BluetoothLeScanTest#testBasicBleScan',
- 'android.bluetooth.cts.BluetoothLeScanTest#testBatchScan',
- 'android.bluetooth.cts.BluetoothLeScanTest#testOpportunisticScan',
- 'android.bluetooth.cts.BluetoothLeScanTest#testScanFilter',],
+ 'android.alarmclock' : [
+ 'android.alarmclock.cts.DismissAlarmTest#testAll',
+ 'android.alarmclock.cts.SetAlarmTest#testAll',
+ 'android.alarmclock.cts.SnoozeAlarmTest#testAll',
+ ],
+ 'android.calllog' : [
+ 'android.calllog.cts.CallLogBackupTest#testSingleCallBackup',
+ ],
+ 'android.dumpsys' : [
+ 'android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput',
+ 'android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats',
+ ],
+ 'android.telecom' : [
+ 'android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect',
+ 'android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections',
+ ],
+ 'android.transition' : [
+ 'android.transition.cts.ChangeScrollTest#testChangeScroll',
+ ],
+ 'android.voicesettings' : [
+ 'android.voicesettings.cts.ZenModeTest#testAll',
+ ],
+ 'com.android.cts.app.os' : [
+ 'com.android.cts.app.os.OsHostTests#testNonExportedActivities',
+ ],
'' : []}
def LogGenerateDescription(name):
