Revise sub test plans for lab testing
move tests using SIM cards to CTS-kitkat-medium plan, move out now stable tests
from CTS-flaky etc.
Bug: 17599198
Bug: 17679062
Change-Id: Id80939a169e813d640bc5dbf2c2a14f39fce1576
diff --git a/tools/utils/buildCts.py b/tools/utils/buildCts.py
index f78952d..4a1b2f3 100755
--- a/tools/utils/buildCts.py
+++ b/tools/utils/buildCts.py
@@ -158,6 +158,7 @@
# CTS Stable plan
plan = tools.TestPlan(packages)
plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-stable')
@@ -166,6 +167,7 @@
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'com\.android\.cts\.browserbench')
+ plan.Include(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.Include(package+'$')
plan.IncludeTests(package, test_list)
@@ -180,6 +182,8 @@
plan.Exclude('.*')
for package, test_list in small_tests.iteritems():
plan.Include(package+'$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-kitkat-small')
@@ -189,6 +193,8 @@
plan.Exclude('.*')
for package, test_list in medium_tests.iteritems():
plan.Include(package+'$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-kitkat-medium')
@@ -197,6 +203,8 @@
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.hardware$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-hardware')
@@ -205,6 +213,8 @@
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.media$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-media')
@@ -213,6 +223,8 @@
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.mediastress$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-mediastress')
@@ -222,6 +234,8 @@
plan.Exclude('.*')
for package, test_list in new_test_packages.iteritems():
plan.Include(package+'$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-l-tests')
@@ -237,6 +251,8 @@
plan.Exclude(r'android\.hardware$')
plan.Exclude(r'android\.media$')
plan.Exclude(r'android\.mediastress$')
+ plan.Exclude(r'com\.android\.cts\.browserbench')
+ plan.Exclude(r'com\.android\.cts\.filesystemperf\.RandomRWTest$')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-staging')
@@ -270,6 +286,7 @@
'android.host.security' : [],
'android.net' : [],
'android.os' : [],
+ 'android.permission2' : [],
'android.security' : [],
'android.telephony' : [],
'android.webkit' : [],
@@ -313,7 +330,6 @@
'android.opengl' : [],
'android.openglperf' : [],
'android.permission' : [],
- 'android.permission2' : [],
'android.preference' : [],
'android.preference2' : [],
'android.provider' : [],
@@ -375,17 +391,10 @@
that are known to be flaky. """
return {
'android.app' : [
- 'cts.ActivityManagerTest#testIsRunningInTestHarness',
- 'cts.AlertDialogTest#testAlertDialogCancelable',
- 'cts.ExpandableListActivityTest#testCallback',],
+ 'cts.ActivityManagerTest#testIsRunningInTestHarness',],
'android.dpi' : [
'cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',],
'android.hardware' : [
- 'camera2.cts.CameraDeviceTest#testCameraDeviceRepeatingRequest',
- 'camera2.cts.ImageReaderTest#testImageReaderFromCameraJpeg',
- 'cts.CameraTest#testImmediateZoom',
- 'cts.CameraTest#testPreviewCallback',
- 'cts.CameraTest#testSmoothZoom',
'cts.CameraTest#testVideoSnapshot',
'cts.CameraGLTest#testCameraToSurfaceTextureMetadata',
'cts.CameraGLTest#testSetPreviewTextureBothCallbacks',
@@ -393,8 +402,6 @@
'android.media' : [
'cts.DecoderTest#testCodecResetsH264WithSurface',
'cts.StreamingMediaPlayerTest#testHLS',],
- 'android.mediastress' : [
- 'cts.NativeMediaTest#test480pPlay',],
'android.net' : [
'cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri',
'cts.DnsTest#testDnsWorks',
@@ -403,9 +410,7 @@
'cts.SSLCertificateSocketFactoryTest#test_createSocket_simple',
'cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping',
'cts.TrafficStatsTest#testTrafficStatsForLocalhost',
- 'wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',
- 'wifi.cts.ScanResultTest#testAndroidTestCaseSetupProperly',
- 'wifi.cts.ScanResultTest#testScanResultTimeStamp',],
+ 'wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',],
'android.os' : [
'cts.BuildVersionTest#testReleaseVersion',
'cts.BuildTest#testIsSecureUserBuild',],
@@ -418,14 +423,7 @@
'cts.SELinuxDomainTest#testSuDomain',
'cts.SELinuxHostTest#testAllEnforcing',],
'android.webkit' : [
- 'cts.WebViewClientTest#testDoUpdateVisitedHistory',
- 'cts.WebViewClientTest#testLoadPage',
- 'cts.WebViewClientTest#testOnFormResubmission',
- 'cts.WebViewClientTest#testOnReceivedError',
- 'cts.WebViewClientTest#testOnReceivedHttpAuthRequest',
- 'cts.WebViewClientTest#testOnScaleChanged',
- 'cts.WebViewClientTest#testOnUnhandledKeyEvent',
- 'cts.WebViewTest#testSetInitialScale',]}
+ 'cts.WebViewClientTest#testOnUnhandledKeyEvent',]}
def LogGenerateDescription(name):
print 'Generating test description for package %s' % name
